From 9af0a33d96d19aaea0fed9b97e38d50ec0bbef51 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 16:23:01 -0300 Subject: [PATCH 01/25] feat: extract inbox subscription into dedicated module --- src/inbox.rs | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 9 ++- 2 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 src/inbox.rs diff --git a/src/inbox.rs b/src/inbox.rs new file mode 100644 index 00000000..0c530522 --- /dev/null +++ b/src/inbox.rs @@ -0,0 +1,163 @@ +//! The daemon's Nostr inbox subscription. +//! +//! Every user action Mostro reacts to — `TakeSell`, `AddInvoice`, `FiatSent`, +//! `Release`, `Dispute`, … — arrives over a **single** long-lived subscription +//! opened once at startup. That subscription is the node's only ear, so this +//! module gives it an identity: a stable id plus the filter that defines it. +//! +//! The id matters because the pieces that keep the inbox alive have to be able +//! to *name* it. A relay's `CLOSED` frame carries a subscription id and nothing +//! else; recognising one as "our inbox just died" — and re-issuing the REQ +//! under the same id — is only possible if the daemon decided the name instead +//! of letting the SDK generate a fresh random one per call. +//! +//! Note that the subscription is deliberately built with `.limit(0)`: it wants +//! live traffic, never stored history. The event loop discards anything whose +//! `created_at` is older than ten seconds anyway (see `accept_event` in +//! `src/app.rs`), so asking a relay for a backlog would only pay for frames +//! that are rejected on arrival. + +use nostr_sdk::prelude::*; +use tracing::{error, info, warn}; + +/// Subscription id used for the daemon inbox. +/// +/// Fixed rather than the SDK's per-call random id, so a `CLOSED` frame can be +/// attributed to the inbox and the REQ re-issued under the same name. It is +/// visible to every relay, which costs nothing in privacy: the filter's `#p` +/// tag already names this node. +const INBOX_SUBSCRIPTION_ID: &str = "mostro-inbox"; + +/// The daemon's inbox: the subscription every trade message arrives on. +#[derive(Debug, Clone)] +pub struct InboxSubscription { + id: SubscriptionId, + filter: Filter, +} + +impl InboxSubscription { + /// Build the inbox subscription for `mostro_pubkey` on the configured + /// transport's `event_kind` (1059 for protocol v1 gift wraps, 14 for the + /// v2 NIP-44 direct messages — see `docs/TRANSPORT_V2_SPEC.md`). + pub fn new(mostro_pubkey: PublicKey, event_kind: Kind) -> Self { + Self { + id: SubscriptionId::new(INBOX_SUBSCRIPTION_ID), + filter: Filter::new() + .pubkey(mostro_pubkey) + .kind(event_kind) + .limit(0), + } + } + + /// The subscription id relays echo back in `EVENT`, `EOSE` and `CLOSED`. + pub fn id(&self) -> &SubscriptionId { + &self.id + } + + /// The filter defining what the inbox listens for. + pub fn filter(&self) -> &Filter { + &self.filter + } + + /// Send the inbox REQ to every connected relay and report the outcome. + /// + /// The SDK's `Output` marks each relay individually, and a relay that + /// refuses the REQ is not an error for the call as a whole — so a node can + /// come up with a dead ear on some (or every) relay and still look healthy. + /// That verdict is logged here rather than discarded. + pub async fn subscribe(&self, client: &Client) -> Result<(), Error> { + let output = client + .subscribe(self.filter.clone()) + .with_id(self.id.clone()) + .await?; + self.report(&output); + Ok(()) + } + + /// Log which relays took the inbox REQ and which refused it. + fn report(&self, output: &Output) { + for (url, err) in output.failed.iter() { + warn!("Inbox subscription refused by relay {url}: {err}"); + } + + if output.success.is_empty() { + // Not fatal — relays reconnect, and the watchdog retries — but the + // node is deaf until one of them accepts, and that must be said out + // loud. The SDK logs its side at `debug`, which release builds + // filter out entirely (`RUST_LOG=none,mostro=info`). + error!( + "Inbox subscription '{}' was accepted by NO relay: Mostro cannot receive any \ + trade message until this recovers", + self.id + ); + } else { + info!( + "Inbox subscription '{}' active on {} relay(s)", + self.id, + output.success.len() + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pubkey() -> PublicKey { + Keys::generate().public_key() + } + + #[test] + fn filter_matches_the_subscription_the_daemon_has_always_used() { + let key = pubkey(); + + for kind in [Kind::GiftWrap, Kind::PrivateDirectMessage] { + let inbox = InboxSubscription::new(key, kind); + // The pre-existing `main.rs` filter, spelled out: p-tagged to this + // node, one transport kind, no stored history. + let expected = Filter::new().pubkey(key).kind(kind).limit(0); + assert_eq!( + inbox.filter(), + &expected, + "inbox filter drifted from the daemon's historical subscription" + ); + } + } + + #[test] + fn id_is_stable_across_instances() { + // A CLOSED frame can only be attributed to the inbox if the id is the + // same one the REQ went out under — including after a re-subscribe, + // which builds a fresh `InboxSubscription`. + let key = pubkey(); + let first = InboxSubscription::new(key, Kind::PrivateDirectMessage); + let second = InboxSubscription::new(key, Kind::PrivateDirectMessage); + + assert_eq!(first.id(), second.id()); + assert_eq!(first.id().to_string(), INBOX_SUBSCRIPTION_ID); + } + + #[test] + fn transport_kind_selects_what_the_inbox_hears() { + let key = pubkey(); + + let v1 = InboxSubscription::new(key, Kind::GiftWrap); + let v2 = InboxSubscription::new(key, Kind::PrivateDirectMessage); + + assert_ne!(v1.filter(), v2.filter()); + assert_eq!(v1.filter().kinds.as_ref().unwrap().len(), 1); + assert!(v1 + .filter() + .kinds + .as_ref() + .unwrap() + .contains(&Kind::GiftWrap)); + assert!(v2 + .filter() + .kinds + .as_ref() + .unwrap() + .contains(&Kind::PrivateDirectMessage)); + } +} diff --git a/src/main.rs b/src/main.rs index 6a636497..6becd38e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ pub mod config; pub mod db; pub mod escrow; pub mod flow; +pub mod inbox; pub mod lightning; pub mod lnurl; pub mod messages; @@ -28,6 +29,7 @@ use crate::config::{ get_db_pool, Settings, DB_POOL, LN_STATUS, MESSAGE_QUEUES, MOSTRO_CONFIG, NOSTR_CLIENT, }; use crate::db::find_held_invoices; +use crate::inbox::InboxSubscription; use crate::lightning::LnStatus; use crate::lightning::LndConnector; use crate::rpc::RpcServer; @@ -104,10 +106,7 @@ async fn main() -> Result<()> { support protocol v2. See https://github.com/MostroP2P/mostro/issues/786" ); } - let subscription = Filter::new() - .pubkey(mostro_keys.public_key()) - .kind(transport.event_kind()) - .limit(0); + let inbox = InboxSubscription::new(mostro_keys.public_key(), transport.event_kind()); let client = match get_nostr_client() { Ok(client) => client, @@ -119,7 +118,7 @@ async fn main() -> Result<()> { }; // Client subscription - client.subscribe(subscription).await?; + inbox.subscribe(client).await?; // Publish NIP-01 kind 0 metadata event let mostro_settings = Settings::get_mostro(); From 1744b5876718ea8d772b68a72c030ae08eb80a3d Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 16:41:22 -0300 Subject: [PATCH 02/25] feat: auto-recover inbox subscription after relay-initiated closures --- src/app.rs | 99 +++++++----- src/inbox.rs | 420 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 476 insertions(+), 43 deletions(-) diff --git a/src/app.rs b/src/app.rs index fd9a9b38..8767e322 100644 --- a/src/app.rs +++ b/src/app.rs @@ -50,6 +50,7 @@ use crate::app::trade_pubkey::trade_pubkey_action; // Core functionality imports use crate::db::add_new_user; use crate::db::is_user_present; +use crate::inbox::{InboxKeeper, InboxSubscription}; use crate::lightning::LndConnector; use crate::util::enqueue_cant_do_msg; use crate::Result; @@ -457,35 +458,44 @@ pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { // gate is meaningless for v1 (gift wraps are signed by throwaway keys). let pow_first_contact = ctx.settings().mostro.effective_pow_first_contact(); let is_v2 = accepted_kind.as_u16() == crate::config::constants::DM_EVENT_KIND; + // Same id and filter `main.rs` subscribed with — the inbox identity is + // derived, not passed around (see `crate::inbox`). + let mut keeper = InboxKeeper::new(InboxSubscription::new(my_keys.public_key(), accepted_kind)); loop { let mut notifications = client.notifications(); while let Some(notification) = notifications.next().await { - if let ClientNotification::Event { event, .. } = notification { - let Some((action, message, unwrapped)) = accept_event( - &ctx, - &event, - my_keys, - pow, - pow_first_contact, - accepted_kind, - is_v2, - ) - .await - else { - continue; - }; - let result = handle_message_action( - &action, - message.clone(), - &unwrapped, - my_keys, - ln_client, - &ctx, - ) - .await; - finalize_dispatch(result, message, unwrapped, &action).await; + match notification { + ClientNotification::Event { event, .. } => { + let Some((action, message, unwrapped)) = accept_event( + &ctx, + &event, + my_keys, + pow, + pow_first_contact, + accepted_kind, + is_v2, + ) + .await + else { + continue; + }; + let result = handle_message_action( + &action, + message.clone(), + &unwrapped, + my_keys, + ln_client, + &ctx, + ) + .await; + finalize_dispatch(result, message, unwrapped, &action).await; + } + ClientNotification::Message { relay_url, message } => { + keeper.on_relay_message(client, &relay_url, &message).await; + } + ClientNotification::Shutdown => return Ok(()), } } } @@ -508,28 +518,35 @@ pub async fn run_cashu(ctx: AppContext) -> Result<()> { let accepted_kind = ctx.settings().mostro.transport.event_kind(); let pow_first_contact = ctx.settings().mostro.effective_pow_first_contact(); let is_v2 = accepted_kind.as_u16() == crate::config::constants::DM_EVENT_KIND; + let mut keeper = InboxKeeper::new(InboxSubscription::new(my_keys.public_key(), accepted_kind)); loop { let mut notifications = client.notifications(); while let Some(notification) = notifications.next().await { - if let ClientNotification::Event { event, .. } = notification { - let Some((action, message, unwrapped)) = accept_event( - &ctx, - &event, - my_keys, - pow, - pow_first_contact, - accepted_kind, - is_v2, - ) - .await - else { - continue; - }; - let result = - dispatch_cashu(&action, message.clone(), &unwrapped, my_keys, &ctx).await; - finalize_dispatch(result, message, unwrapped, &action).await; + match notification { + ClientNotification::Event { event, .. } => { + let Some((action, message, unwrapped)) = accept_event( + &ctx, + &event, + my_keys, + pow, + pow_first_contact, + accepted_kind, + is_v2, + ) + .await + else { + continue; + }; + let result = + dispatch_cashu(&action, message.clone(), &unwrapped, my_keys, &ctx).await; + finalize_dispatch(result, message, unwrapped, &action).await; + } + ClientNotification::Message { relay_url, message } => { + keeper.on_relay_message(client, &relay_url, &message).await; + } + ClientNotification::Shutdown => return Ok(()), } } } diff --git a/src/inbox.rs b/src/inbox.rs index 0c530522..62cee585 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -15,10 +15,33 @@ //! live traffic, never stored history. The event loop discards anything whose //! `created_at` is older than ten seconds anyway (see `accept_event` in //! `src/app.rs`), so asking a relay for a backlog would only pay for frames -//! that are rejected on arrival. +//! that are rejected on arrival. The same ten-second window is why a +//! re-subscribe cannot recover what was missed: whatever a user sent while the +//! inbox was down is already too old to be accepted by the time it could be +//! replayed. Losing the ear loses those messages for good — hence +//! [`InboxKeeper`], which exists to make the outage as short as possible. +//! +//! # Keeping the ear open +//! +//! Relays end subscriptions on their own initiative, and say so with a +//! `CLOSED` frame. The SDK's reaction is unforgiving: for nearly every reason +//! prefix — and for no prefix at all — it *removes* the subscription outright, +//! and removed subscriptions are never re-REQ'd, not even across a reconnect. +//! A single frame from a single relay therefore ends the daemon's ability to +//! hear anything, permanently and without a word: the SDK logs it at `debug`, +//! which release builds filter out (`RUST_LOG=none,mostro=info`). +//! +//! [`InboxKeeper`] closes that hole. It watches the control-plane traffic the +//! event loop used to discard, recognises a `CLOSED` aimed at the inbox, and +//! re-issues the REQ to the relay that sent it — under per-relay backoff, so a +//! relay that refuses the inbox on principle is retried at a decreasing rate +//! instead of being hammered. + +use std::collections::HashMap; +use std::time::{Duration, Instant}; use nostr_sdk::prelude::*; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; /// Subscription id used for the daemon inbox. /// @@ -28,6 +51,31 @@ use tracing::{error, info, warn}; /// tag already names this node. const INBOX_SUBSCRIPTION_ID: &str = "mostro-inbox"; +/// Delay before a *second* consecutive re-subscribe to the same relay. +/// +/// The first `CLOSED` is answered immediately — the common case is a transient +/// refusal, and every second of delay is a second of deaf node. Backoff only +/// starts mattering when a relay keeps closing the inbox. +const RESUBSCRIBE_INITIAL_BACKOFF: Duration = Duration::from_secs(2); + +/// Ceiling for the per-relay re-subscribe delay. +/// +/// A relay that has refused the inbox for five minutes straight is not having +/// a hiccup — it is configured to refuse us (NIP-42, a pubkey allowlist, a ban) +/// and the operator has to intervene. Retrying every five minutes keeps the +/// door open for a config change on their side without generating traffic that +/// looks like an attack. +const RESUBSCRIBE_MAX_BACKOFF: Duration = Duration::from_secs(300); + +/// Per-relay re-subscribe pacing. +#[derive(Debug)] +struct RelayBackoff { + /// Earliest instant at which another REQ may go out to this relay. + next_attempt_at: Instant, + /// Delay applied after the next attempt; doubles up to the ceiling. + delay: Duration, +} + /// The daemon's inbox: the subscription every trade message arrives on. #[derive(Debug, Clone)] pub struct InboxSubscription { @@ -100,6 +148,121 @@ impl InboxSubscription { } } +/// Keeps the inbox subscription alive across relay-initiated closures. +/// +/// Lives in the event loop, which is the only consumer of the notification +/// stream — hence the plain `&mut self` state rather than a lock. +pub struct InboxKeeper { + subscription: InboxSubscription, + /// Only holds relays that are currently failing; a relay that accepts the + /// REQ is dropped from the map, so the steady state is empty. + backoff: HashMap, +} + +impl InboxKeeper { + pub fn new(subscription: InboxSubscription) -> Self { + Self { + subscription, + backoff: HashMap::new(), + } + } + + /// React to one control-plane frame from `relay_url`. + /// + /// Two frames matter for the inbox: `CLOSED`, which means the ear on that + /// relay is gone and has to be re-opened, and `EOSE`, which is a relay + /// confirming it accepted the REQ and is the signal used to clear the + /// backoff. Everything else (`OK`, `NOTICE`, other subscriptions' frames) + /// is not this module's business. + pub async fn on_relay_message( + &mut self, + client: &Client, + relay_url: &RelayUrl, + message: &RelayMessage<'_>, + ) { + match message { + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id.as_ref() == self.subscription.id() => { + warn!("Relay {relay_url} closed the Mostro inbox subscription: \"{message}\""); + self.resubscribe(client, relay_url).await; + } + RelayMessage::EndOfStoredEvents(subscription_id) + if subscription_id.as_ref() == self.subscription.id() => + { + // The relay answered the REQ, so whatever made it fail before + // is over and the next failure deserves a prompt retry again. + if self.backoff.remove(relay_url).is_some() { + info!("Inbox subscription re-established on relay {relay_url}"); + } + } + _ => {} + } + } + + /// Re-issue the inbox REQ to a single relay, subject to backoff. + async fn resubscribe(&mut self, client: &Client, relay_url: &RelayUrl) { + if !self.allow_attempt(relay_url, Instant::now()) { + debug!("Skipping inbox re-subscribe on relay {relay_url}: backing off"); + return; + } + + let relay = match client.relay(relay_url).await { + Ok(Some(relay)) => relay, + Ok(None) => { + warn!("Relay {relay_url} closed the inbox but is no longer in the pool"); + return; + } + Err(e) => { + warn!("Cannot reach relay {relay_url} to re-subscribe the inbox: {e}"); + return; + } + }; + + // A `CLOSED` does not always remove the subscription: rate-limited and + // auth-required closures only *mark* it, and a marked subscription is + // re-REQ'd no earlier than the next reconnect — which may never come on + // a healthy connection. Dropping the registration first makes the REQ + // below unconditional, instead of being refused as a duplicate id. + let _ = relay.unsubscribe(self.subscription.id()).await; + + match relay + .subscribe(self.subscription.filter().clone()) + .with_id(self.subscription.id().clone()) + .await + { + Ok(_) => info!("Re-sent the inbox subscription to relay {relay_url}"), + Err(e) => warn!("Failed to re-subscribe the inbox on relay {relay_url}: {e}"), + } + } + + /// Whether a re-subscribe to `relay` may go out at `now`, arming the next + /// delay when it may. The first failure for a relay always passes. + fn allow_attempt(&mut self, relay: &RelayUrl, now: Instant) -> bool { + match self.backoff.get_mut(relay) { + None => { + self.backoff.insert( + relay.clone(), + RelayBackoff { + next_attempt_at: now + RESUBSCRIBE_INITIAL_BACKOFF, + delay: RESUBSCRIBE_INITIAL_BACKOFF, + }, + ); + true + } + Some(state) => { + if now < state.next_attempt_at { + return false; + } + state.delay = (state.delay * 2).min(RESUBSCRIBE_MAX_BACKOFF); + state.next_attempt_at = now + state.delay; + true + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -108,6 +271,14 @@ mod tests { Keys::generate().public_key() } + fn keeper() -> InboxKeeper { + InboxKeeper::new(InboxSubscription::new(pubkey(), Kind::GiftWrap)) + } + + fn relay_url(url: &str) -> RelayUrl { + RelayUrl::parse(url).expect("valid relay url") + } + #[test] fn filter_matches_the_subscription_the_daemon_has_always_used() { let key = pubkey(); @@ -160,4 +331,249 @@ mod tests { .unwrap() .contains(&Kind::PrivateDirectMessage)); } + + // ───────────────────────────── backoff pacing ───────────────────────────── + + #[test] + fn first_closure_from_a_relay_retries_immediately() { + let mut keeper = keeper(); + let relay = relay_url("ws://relay.example"); + + assert!( + keeper.allow_attempt(&relay, Instant::now()), + "a first CLOSED must be answered at once: every delay is deaf time" + ); + } + + #[test] + fn repeat_closures_are_paced_and_back_off() { + let mut keeper = keeper(); + let relay = relay_url("ws://relay.example"); + let start = Instant::now(); + + assert!(keeper.allow_attempt(&relay, start)); + // A relay that closes again right away must not pull a second REQ. + assert!(!keeper.allow_attempt(&relay, start)); + assert!(!keeper.allow_attempt(&relay, start + Duration::from_secs(1))); + + // Past the first delay it retries, and the next wait is longer. + assert!(keeper.allow_attempt(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF)); + assert!(!keeper.allow_attempt(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 2)); + assert!(keeper.allow_attempt(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 3)); + } + + #[test] + fn backoff_is_capped() { + let mut keeper = keeper(); + let relay = relay_url("ws://relay.example"); + let mut now = Instant::now(); + + // Drive it well past the ceiling. + for _ in 0..20 { + assert!(keeper.allow_attempt(&relay, now)); + now += RESUBSCRIBE_MAX_BACKOFF * 2; + } + + assert_eq!( + keeper.backoff.get(&relay).expect("state kept").delay, + RESUBSCRIBE_MAX_BACKOFF, + "a hostile relay must still be retried every {RESUBSCRIBE_MAX_BACKOFF:?}" + ); + } + + #[test] + fn backoff_is_per_relay() { + let mut keeper = keeper(); + let hostile = relay_url("ws://hostile.example"); + let healthy = relay_url("ws://healthy.example"); + let now = Instant::now(); + + assert!(keeper.allow_attempt(&hostile, now)); + assert!(!keeper.allow_attempt(&hostile, now)); + // One misbehaving relay must not delay recovery on another. + assert!(keeper.allow_attempt(&healthy, now)); + } + + // ───────────────────────── control-plane handling ───────────────────────── + + #[tokio::test] + async fn eose_for_the_inbox_clears_the_backoff() { + let client = crate::util::mostro_nostr_client_options().build(); + let mut keeper = keeper(); + let relay = relay_url("ws://relay.example"); + let now = Instant::now(); + + assert!(keeper.allow_attempt(&relay, now)); + assert!(keeper.backoff.contains_key(&relay)); + + let eose = RelayMessage::EndOfStoredEvents(std::borrow::Cow::Owned( + keeper.subscription.id().clone(), + )); + keeper.on_relay_message(&client, &relay, &eose).await; + + assert!( + !keeper.backoff.contains_key(&relay), + "an accepted REQ must reset the pacing for the next failure" + ); + } + + #[tokio::test] + async fn frames_for_other_subscriptions_are_ignored() { + let client = crate::util::mostro_nostr_client_options().build(); + let mut keeper = keeper(); + let relay = relay_url("ws://relay.example"); + + // Mostro's price provider and NIP-33 queries share these relays; their + // CLOSED frames must not touch the inbox's state. + let other = RelayMessage::Closed { + subscription_id: std::borrow::Cow::Owned(SubscriptionId::new("someone-else")), + message: std::borrow::Cow::Borrowed("error: not yours"), + }; + keeper.on_relay_message(&client, &relay, &other).await; + + assert!( + keeper.backoff.is_empty(), + "a CLOSED for another subscription must not be treated as an inbox failure" + ); + } + + // ────────────────────────── end-to-end regression ───────────────────────── + + /// Rejects the first REQ it sees and admits every one after it: a relay + /// having a bad moment, which is exactly the case the daemon used to never + /// recover from. + #[derive(Debug, Default)] + struct RejectFirstQuery { + seen: std::sync::atomic::AtomicUsize, + } + + impl nostr_sdk::local_relay::QueryPolicy for RejectFirstQuery { + fn admit_query<'a>( + &'a self, + _query: &'a mut Filter, + _addr: &'a std::net::SocketAddr, + ) -> std::pin::Pin< + Box< + dyn std::future::Future + + Send + + 'a, + >, + > { + Box::pin(async move { + let first = self.seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0; + if first { + nostr_sdk::local_relay::QueryPolicyResult::reject( + MachineReadablePrefix::Error, + "subscription refused", + ) + } else { + nostr_sdk::local_relay::QueryPolicyResult::Accept + } + }) + } + } + + /// A gift wrap addressed to `recipient` — one p-tag, as the transport (and + /// the relay's own validation) requires. + fn wrap_for(recipient: PublicKey) -> Event { + EventBuilder::new(Kind::GiftWrap, "sealed") + .tag(Tag::public_key(recipient)) + .finalize(&Keys::generate()) + .expect("sign gift wrap") + } + + #[tokio::test] + async fn closed_inbox_is_resubscribed_and_hears_again() { + use futures::StreamExt; + use nostr_sdk::local_relay::LocalRelay; + + let relay = LocalRelay::builder() + .query_policy(RejectFirstQuery::default()) + .build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let mostro = Keys::generate(); + let subscription = InboxSubscription::new(mostro.public_key(), Kind::GiftWrap); + + let client = crate::util::mostro_nostr_client_options().build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + + let mut notifications = client.notifications(); + // The relay CLOSEs this one; without the keeper the ear is gone here. + subscription.subscribe(&client).await.expect("subscribe"); + + let publisher = ClientBuilder::default().build(); + publisher.add_relay(url.clone()).await.expect("add_relay"); + publisher.connect().await; + + let mut keeper = InboxKeeper::new(subscription.clone()); + let wanted = wrap_for(mostro.public_key()); + let wanted_id = wanted.id; + let mut published = false; + + let heard = tokio::time::timeout(Duration::from_secs(20), async { + while let Some(notification) = notifications.next().await { + match notification { + ClientNotification::Event { event, .. } => { + if event.id == wanted_id { + return true; + } + } + ClientNotification::Message { relay_url, message } => { + keeper.on_relay_message(&client, &relay_url, &message).await; + // Publish only once the inbox is confirmed live again, + // so the event cannot be mistaken for stored history. + if !published && matches!(&*message, RelayMessage::EndOfStoredEvents(_)) { + published = true; + publisher.send_event(&wanted).await.expect("publish"); + } + } + ClientNotification::Shutdown => return false, + } + } + false + }) + .await + .expect("timed out: the inbox never recovered from the relay's CLOSED"); + + assert!( + heard, + "after a relay CLOSED the inbox, Mostro must re-subscribe and receive again" + ); + + relay.shutdown(); + } + + #[tokio::test] + async fn without_the_keeper_a_closed_inbox_stays_dead() { + use nostr_sdk::local_relay::LocalRelay; + + // The defect this module exists for: the SDK drops a CLOSED + // subscription outright, and nothing re-issues the REQ. Pinned here so + // that a future SDK bump changing this behaviour is noticed. + let relay = LocalRelay::builder() + .query_policy(RejectFirstQuery::default()) + .build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options().build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + subscription.subscribe(&client).await.expect("subscribe"); + + // Give the relay time to answer with CLOSED. + tokio::time::sleep(Duration::from_secs(2)).await; + + assert!( + !client.subscriptions().await.contains_key(subscription.id()), + "SDK behaviour changed: a CLOSED subscription is no longer dropped, \ + so the keeper's premise needs revisiting" + ); + + relay.shutdown(); + } } From af4c8d5bf0bad45c457fe2ef8bfc9e83abb7c936 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 16:46:52 -0300 Subject: [PATCH 03/25] feat: add NIP-42 authentication to prevent relay read-gating --- src/inbox.rs | 8 +-- src/price/providers/nostr.rs | 2 +- src/util.rs | 136 +++++++++++++++++++++++++++++++---- 3 files changed, 129 insertions(+), 17 deletions(-) diff --git a/src/inbox.rs b/src/inbox.rs index 62cee585..de7fc2f9 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -398,7 +398,7 @@ mod tests { #[tokio::test] async fn eose_for_the_inbox_clears_the_backoff() { - let client = crate::util::mostro_nostr_client_options().build(); + let client = crate::util::mostro_nostr_client_options(None).build(); let mut keeper = keeper(); let relay = relay_url("ws://relay.example"); let now = Instant::now(); @@ -419,7 +419,7 @@ mod tests { #[tokio::test] async fn frames_for_other_subscriptions_are_ignored() { - let client = crate::util::mostro_nostr_client_options().build(); + let client = crate::util::mostro_nostr_client_options(None).build(); let mut keeper = keeper(); let relay = relay_url("ws://relay.example"); @@ -496,7 +496,7 @@ mod tests { let mostro = Keys::generate(); let subscription = InboxSubscription::new(mostro.public_key(), Kind::GiftWrap); - let client = crate::util::mostro_nostr_client_options().build(); + let client = crate::util::mostro_nostr_client_options(None).build(); client.add_relay(url.clone()).await.expect("add_relay"); client.connect().await; @@ -560,7 +560,7 @@ mod tests { let url = relay.url().await; let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); - let client = crate::util::mostro_nostr_client_options().build(); + let client = crate::util::mostro_nostr_client_options(None).build(); client.add_relay(url.clone()).await.expect("add_relay"); client.connect().await; subscription.subscribe(&client).await.expect("subscribe"); diff --git a/src/price/providers/nostr.rs b/src/price/providers/nostr.rs index 7c55e231..dac6c798 100644 --- a/src/price/providers/nostr.rs +++ b/src/price/providers/nostr.rs @@ -715,7 +715,7 @@ mod tests { #[tokio::test] #[ignore = "hits a real Nostr relay; run explicitly for manual verification"] async fn live_relay_fetch_returns_real_rates() { - let client = crate::util::price_nostr_client_options().build(); + let client = crate::util::price_nostr_client_options(None).build(); client .add_relay("wss://relay.mostro.network") .await diff --git a/src/util.rs b/src/util.rs index 558c1681..1150d65c 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1356,6 +1356,25 @@ async fn update_order_event_stamped( Ok(Some(order_updated)) } +/// The identity Mostro authenticates with on NIP-42 relays, when it has one. +/// +/// Keys are set by `settings_init()` long before any client is built, so in a +/// running daemon this is always `Some`; it is `None` only in tests that skip +/// the configuration bootstrap. Saying so out loud matters because the failure +/// it causes is silent — an auth-gated relay simply stops delivering. +fn nip42_identity() -> Option<&'static Keys> { + match get_keys() { + Ok(keys) => Some(keys), + Err(e) => { + tracing::warn!( + "Nostr keys unavailable ({e}); this client cannot answer NIP-42 challenges and \ + will be refused by relays that require authentication" + ); + None + } + } +} + pub async fn connect_nostr() -> Result { let nostr_settings = Settings::get_nostr(); @@ -1365,7 +1384,7 @@ pub async fn connect_nostr() -> Result { // would drop matching trade messages before dispatch (hermeme, PR #841). // Price queries use [`connect_price_nostr`] / [`PRICE_NOSTR_CLIENT`] with // verification enabled instead. - let client = mostro_nostr_client_options().build(); + let client = mostro_nostr_client_options(nip42_identity()).build(); // Add relays for relay in nostr_settings.relays.iter() { @@ -1385,7 +1404,7 @@ pub async fn connect_nostr() -> Result { /// daemon client, with [`price_nostr_client_options`]). pub async fn connect_price_nostr() -> Result { let nostr_settings = Settings::get_nostr(); - let client = price_nostr_client_options().build(); + let client = price_nostr_client_options(nip42_identity()).build(); for relay in nostr_settings.relays.iter() { client @@ -1432,23 +1451,45 @@ fn mostro_nostr_relay_limits() -> RelayLimits { limits } -fn client_options_from_policy(policy: MostroNostrClientPolicy) -> ClientBuilder { +fn client_options_from_policy( + policy: MostroNostrClientPolicy, + authenticate_as: Option<&Keys>, +) -> ClientBuilder { let mut builder = ClientBuilder::new().relay_limits(mostro_nostr_relay_limits()); if policy.verify_subscriptions { builder = builder.verify_subscriptions(true); } + if let Some(keys) = authenticate_as { + // NIP-42. Without an authenticator the SDK cannot answer a relay's + // AUTH challenge, and a relay that gates reads behind it answers the + // REQ with `CLOSED "auth-required: …"` — which the SDK then treats as + // permanent, dropping the subscription for good. With one, the closure + // is provisional: the client authenticates and the REQ is re-sent. + // + // Authenticating costs no privacy Mostro has not already spent. The + // AUTH event is bound to that relay's challenge and URL, so it cannot + // be replayed elsewhere, and the node publishes orders signed with this + // very key to these very relays. + builder = builder.authenticator(SignerAuthenticator::new(keys.clone())); + } builder } /// Process-wide daemon Nostr [`ClientBuilder`] (inbox / publishing). -pub(crate) fn mostro_nostr_client_options() -> ClientBuilder { - client_options_from_policy(daemon_nostr_client_policy()) +/// +/// `authenticate_as` is the identity used for NIP-42; passing `None` builds a +/// client that cannot read from auth-gated relays. +pub(crate) fn mostro_nostr_client_options(authenticate_as: Option<&Keys>) -> ClientBuilder { + client_options_from_policy(daemon_nostr_client_policy(), authenticate_as) } /// Price-provider Nostr [`ClientBuilder`]: size limits plus subscription /// filter verification, scoped away from the daemon inbox client. -pub(crate) fn price_nostr_client_options() -> ClientBuilder { - client_options_from_policy(price_nostr_client_policy()) +/// +/// Shares the daemon's relay list, so it needs the same NIP-42 identity: an +/// auth-gated relay blinds the price feed exactly like it blinds the inbox. +pub(crate) fn price_nostr_client_options(authenticate_as: Option<&Keys>) -> ClientBuilder { + client_options_from_policy(price_nostr_client_policy(), authenticate_as) } /// Which caller drove `show_hold_invoice`, and therefore which order @@ -3323,8 +3364,79 @@ mod tests { "price client must enable verify_subscriptions" ); // Helpers remain constructible (SDK copies the flag into RelayOptions). - let _daemon = mostro_nostr_client_options().build(); - let _price = price_nostr_client_options().build(); + let _daemon = mostro_nostr_client_options(None).build(); + let _price = price_nostr_client_options(None).build(); + // Both accept a NIP-42 identity: they share a relay list, so an + // auth-gated relay blinds the price feed exactly like the inbox. + let keys = Keys::generate(); + let _daemon_auth = mostro_nostr_client_options(Some(&keys)).build(); + let _price_auth = price_nostr_client_options(Some(&keys)).build(); + } + + /// A local relay that requires NIP-42 authentication before it will serve + /// reads — the shape that silently blinded the daemon. + async fn nip42_read_gated_relay() -> nostr_sdk::local_relay::LocalRelay { + let relay = nostr_sdk::local_relay::LocalRelay::builder() + .nip42(nostr_sdk::local_relay::LocalRelayBuilderNip42::read()) + .build(); + relay.run().await.expect("run nip42 relay"); + relay + } + + #[tokio::test] + async fn auth_gated_relay_keeps_the_subscription_only_when_authenticating() { + use std::time::Duration; + + let relay = nip42_read_gated_relay().await; + let url = relay.url().await; + + let keys = Keys::generate(); + let seeded = EventBuilder::new(nostr::event::Kind::TextNote, "gated") + .finalize(&keys) + .expect("sign"); + relay.add_event(seeded).await.expect("seed event"); + + let filter = Filter::new().kind(nostr::event::Kind::TextNote).limit(0); + let id = SubscriptionId::new("nip42-probe"); + + // Without an identity the relay answers `auth-required`, and the SDK + // drops the subscription for good: the daemon is deaf on this relay + // with no way back. + let anonymous = mostro_nostr_client_options(None).build(); + anonymous.add_relay(url.clone()).await.expect("add_relay"); + anonymous.connect().await; + anonymous + .subscribe(filter.clone()) + .with_id(id.clone()) + .await + .expect("subscribe"); + + // With one, the closure is provisional: the client answers the + // challenge and the REQ is re-sent, so the subscription survives. + let authenticated = mostro_nostr_client_options(Some(&keys)).build(); + authenticated + .add_relay(url.clone()) + .await + .expect("add_relay"); + authenticated.connect().await; + authenticated + .subscribe(filter) + .with_id(id.clone()) + .await + .expect("subscribe"); + + tokio::time::sleep(Duration::from_secs(2)).await; + + assert!( + !anonymous.subscriptions().await.contains_key(&id), + "SDK behaviour changed: an auth-required CLOSED no longer drops the subscription" + ); + assert!( + authenticated.subscriptions().await.contains_key(&id), + "a client with a NIP-42 identity must keep its subscription on an auth-gated relay" + ); + + relay.shutdown(); } #[tokio::test] @@ -3342,7 +3454,7 @@ mod tests { .expect("mock relay"); let url = mock.url().await; - let price_client = price_nostr_client_options().build(); + let price_client = price_nostr_client_options(None).build(); price_client .add_relay(url.clone()) .await @@ -3378,7 +3490,7 @@ mod tests { let mock = MockRelay::run().await.expect("mock relay"); let url = mock.url().await; - let daemon = mostro_nostr_client_options().build(); + let daemon = mostro_nostr_client_options(None).build(); daemon.add_relay(url.clone()).await.expect("add_relay"); daemon.connect().await; @@ -3445,7 +3557,7 @@ mod tests { seeder.send_event(&event).await.expect("seed"); } - let price_client = price_nostr_client_options().build(); + let price_client = price_nostr_client_options(None).build(); price_client.add_relay(url).await.expect("add_relay"); price_client.connect().await; From a6add01f95e2194d13f30a0babeba87255a1d7c1 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 16:54:38 -0300 Subject: [PATCH 04/25] feat: add inbox subscription watchdog to detect and recover silent failures --- src/inbox.rs | 401 +++++++++++++++++++++++++++++++++++++++++++++-- src/main.rs | 8 +- src/scheduler.rs | 31 ++++ 3 files changed, 424 insertions(+), 16 deletions(-) diff --git a/src/inbox.rs b/src/inbox.rs index de7fc2f9..f5876c73 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -36,8 +36,17 @@ //! re-issues the REQ to the relay that sent it — under per-relay backoff, so a //! relay that refuses the inbox on principle is retried at a decreasing rate //! instead of being hammered. +//! +//! Not every way of losing the ear announces itself with a frame, though: the +//! notification channel silently drops messages when the consumer falls +//! behind, a REQ can fail to go out, a relay can be added after startup. +//! [`check_inbox_health`] is the backstop — it asks each connected relay +//! whether it is still serving the subscription, re-subscribes the ones that +//! are not, and records the verdict in [`InboxHealth`] so the rest of the +//! daemon can tell whether Mostro is currently able to hear anything at all. use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; use nostr_sdk::prelude::*; @@ -220,21 +229,7 @@ impl InboxKeeper { } }; - // A `CLOSED` does not always remove the subscription: rate-limited and - // auth-required closures only *mark* it, and a marked subscription is - // re-REQ'd no earlier than the next reconnect — which may never come on - // a healthy connection. Dropping the registration first makes the REQ - // below unconditional, instead of being refused as a duplicate id. - let _ = relay.unsubscribe(self.subscription.id()).await; - - match relay - .subscribe(self.subscription.filter().clone()) - .with_id(self.subscription.id().clone()) - .await - { - Ok(_) => info!("Re-sent the inbox subscription to relay {relay_url}"), - Err(e) => warn!("Failed to re-subscribe the inbox on relay {relay_url}: {e}"), - } + resubscribe_relay(&relay, &self.subscription).await; } /// Whether a re-subscribe to `relay` may go out at `now`, arming the next @@ -263,6 +258,192 @@ impl InboxKeeper { } } +/// Re-send the inbox REQ to one relay. +/// +/// Shared by the event-loop keeper (reacting to a `CLOSED`) and the watchdog +/// (finding an ear that went missing without one), so both recover a relay the +/// same way. +async fn resubscribe_relay(relay: &Relay, subscription: &InboxSubscription) { + // A `CLOSED` does not always remove the subscription: rate-limited and + // auth-required closures only *mark* it, and a marked subscription is + // re-REQ'd no earlier than the next reconnect — which may never come on a + // healthy connection. Dropping the registration first makes the REQ below + // unconditional, instead of being refused as a duplicate id. + let _ = relay.unsubscribe(subscription.id()).await; + + match relay + .subscribe(subscription.filter().clone()) + .with_id(subscription.id().clone()) + .await + { + Ok(_) => info!("Re-sent the inbox subscription to relay {}", relay.url()), + Err(e) => warn!( + "Failed to re-subscribe the inbox on relay {}: {e}", + relay.url() + ), + } +} + +/// Whether the daemon can currently hear anything at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InboxStatus { + /// At least one connected relay is serving the inbox subscription. + Listening, + /// No connected relay is serving it: every message sent to Mostro right + /// now is being lost. + Blind, +} + +/// Process-wide inbox health. `None` until [`InboxHealth::install_global`] +/// runs at startup; consumers treat an absent health record as "listening", so +/// unit tests that never install it behave as before. +static INBOX_HEALTH: OnceLock = OnceLock::new(); + +/// Why [`InboxHealth::install_global`] refused. Mirrors `spam_gate::InstallError`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InstallError { + /// A health record is already installed. + AlreadyInstalled, +} + +#[derive(Debug, Default)] +struct HealthState { + /// When the current outage began, if the inbox is deaf right now. + blind_since: Option, + /// The most recent finished outage: when it ended, and how long it lasted. + last_outage: Option<(Instant, Duration)>, +} + +/// Tracks whether the daemon's ear is open, and for how long it was not. +/// +/// The scheduler's timeout machinery reads this: an order is only "late" if +/// Mostro was in a position to hear from the user, and the ten-second replay +/// window means a message sent into a dead inbox is gone rather than delayed +/// (see the module docs). Punishing a user for a silence the node itself +/// caused would be unfair, so the timeout clock stops while the inbox is down. +#[derive(Debug, Default)] +pub struct InboxHealth { + state: Mutex, +} + +impl InboxHealth { + pub fn new() -> Self { + Self::default() + } + + /// Install as the process-wide health record. + pub fn install_global(self) -> Result<(), InstallError> { + INBOX_HEALTH + .set(self) + .map_err(|_| InstallError::AlreadyInstalled) + } + + /// The process-wide health record, if one was installed. + pub fn global() -> Option<&'static InboxHealth> { + INBOX_HEALTH.get() + } + + /// Record the current observation, returning the resulting status. + fn observe(&self, status: InboxStatus, now: Instant) -> InboxStatus { + let mut state = self.state.lock().expect("inbox health mutex poisoned"); + match (status, state.blind_since) { + (InboxStatus::Blind, None) => state.blind_since = Some(now), + (InboxStatus::Listening, Some(since)) => { + let outage = now.saturating_duration_since(since); + state.blind_since = None; + state.last_outage = Some((now, outage)); + } + _ => {} + } + status + } + + /// Whether the inbox is deaf right now. + pub fn is_blind(&self) -> bool { + self.state + .lock() + .expect("inbox health mutex poisoned") + .blind_since + .is_some() + } +} + +/// Check every read relay, re-subscribing any that lost the inbox, and record +/// the verdict in the process-wide [`InboxHealth`]. +/// +/// The health question is asked of the *subscription*, not of traffic: a node +/// with no trades in flight is legitimately silent, so treating quiet as +/// failure would raise false alarms on an idle instance and, worse, would stop +/// the timeout machinery for no reason. +/// +/// A relay re-subscribed during this very audit does **not** count as +/// listening. Sending a REQ is not the same as having it honoured — a relay +/// that closes the inbox on principle would accept the REQ and close it again +/// moments later, and counting the attempt would report a healthy inbox +/// forever while nothing was ever delivered. Only a subscription that was +/// already in place when the audit ran proves the relay kept it. Recovery is +/// therefore confirmed on the following round, which costs one extra interval +/// before the inbox is declared healthy again and keeps the error on the safe +/// side: the timeout clock stays frozen a little longer than strictly needed, +/// rather than resuming while the node is still deaf. +pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscription) -> InboxStatus { + let relays = client + .relays() + .with_capabilities(RelayCapabilities::READ) + .await; + + let mut listening = 0usize; + let mut retried = 0usize; + + for (url, relay) in relays.iter() { + if !relay.status().is_connected() { + continue; + } + if relay.subscription(subscription.id()).await.is_some() { + listening += 1; + } else { + // Connected but not subscribed: a CLOSED the event loop never saw + // (the notification channel drops frames when it lags), a REQ that + // failed to go out, or a relay re-added after startup. + warn!("Relay {url} is connected but not serving the Mostro inbox; re-subscribing"); + resubscribe_relay(relay, subscription).await; + retried += 1; + } + } + + let status = if listening > 0 { + InboxStatus::Listening + } else { + InboxStatus::Blind + }; + + if let Some(health) = InboxHealth::global() { + let was_blind = health.is_blind(); + health.observe(status, Instant::now()); + + match (was_blind, status) { + (false, InboxStatus::Blind) => error!( + "Mostro inbox is BLIND: no connected relay is serving subscription '{}'. \ + Trade messages sent now are lost, and order timeouts are on hold until it \ + recovers", + subscription.id() + ), + (true, InboxStatus::Listening) => info!( + "Mostro inbox recovered: subscription '{}' is live on {listening} relay(s)", + subscription.id() + ), + (true, InboxStatus::Blind) => { + warn!("Mostro inbox still blind ({retried} relay(s) retried this round)") + } + (false, InboxStatus::Listening) => { + debug!("Inbox healthy on {listening} relay(s)"); + } + } + } + + status +} + #[cfg(test)] mod tests { use super::*; @@ -576,4 +757,194 @@ mod tests { relay.shutdown(); } + + // ──────────────────────────────── watchdog ──────────────────────────────── + + #[test] + fn health_records_an_outage_from_first_blindness_to_recovery() { + let health = InboxHealth::new(); + let start = Instant::now(); + + assert!(!health.is_blind(), "a fresh record starts out listening"); + + health.observe(InboxStatus::Blind, start); + assert!(health.is_blind()); + + // Staying blind must not restart the clock — the outage began at the + // first observation, and that is what the timeout discount is owed on. + health.observe(InboxStatus::Blind, start + Duration::from_secs(30)); + assert!(health.is_blind()); + + health.observe(InboxStatus::Listening, start + Duration::from_secs(90)); + assert!(!health.is_blind()); + + let state = health.state.lock().expect("lock"); + let (_, outage) = state.last_outage.expect("outage recorded"); + assert_eq!( + outage, + Duration::from_secs(90), + "the recorded outage must span the whole blind window" + ); + } + + #[test] + fn health_ignores_repeated_healthy_observations() { + let health = InboxHealth::new(); + let now = Instant::now(); + + health.observe(InboxStatus::Listening, now); + health.observe(InboxStatus::Listening, now + Duration::from_secs(30)); + + assert!(!health.is_blind()); + assert!( + health.state.lock().expect("lock").last_outage.is_none(), + "a node that was never blind has no outage to compensate for" + ); + } + + #[tokio::test] + async fn watchdog_resubscribes_a_relay_that_lost_the_inbox() { + use nostr_sdk::local_relay::LocalRelay; + + // Accepts every REQ: the point here is the *missing* subscription, not + // a refusing relay. + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + subscription.subscribe(&client).await.expect("subscribe"); + tokio::time::sleep(Duration::from_millis(500)).await; + + // Simulate the ear vanishing without a CLOSED the loop could see — + // a frame dropped by a lagging notification channel looks like this. + client + .unsubscribe(subscription.id()) + .await + .expect("drop the subscription"); + assert!(!client.subscriptions().await.contains_key(subscription.id())); + + // The audit re-subscribes, but does not yet claim to be listening: a + // REQ that just went out proves nothing about whether the relay will + // honour it. + assert_eq!( + check_inbox_health(&client, &subscription).await, + InboxStatus::Blind, + "a relay re-subscribed during this audit must not count as listening yet" + ); + assert!( + client.subscriptions().await.contains_key(subscription.id()), + "the inbox subscription must be back after the audit" + ); + + // The relay kept it, so the next round confirms the recovery. + assert_eq!( + check_inbox_health(&client, &subscription).await, + InboxStatus::Listening, + "a subscription that survived to the next audit means the ear is open" + ); + + relay.shutdown(); + } + + #[tokio::test] + async fn watchdog_stays_blind_against_a_relay_that_keeps_closing() { + use nostr_sdk::local_relay::LocalRelay; + + /// Refuses every REQ, always. + #[derive(Debug)] + struct RejectAllQueries; + + impl nostr_sdk::local_relay::QueryPolicy for RejectAllQueries { + fn admit_query<'a>( + &'a self, + _query: &'a mut Filter, + _addr: &'a std::net::SocketAddr, + ) -> std::pin::Pin< + Box< + dyn std::future::Future + + Send + + 'a, + >, + > { + Box::pin(async { + nostr_sdk::local_relay::QueryPolicyResult::reject( + MachineReadablePrefix::Blocked, + "no subscriptions here", + ) + }) + } + } + + let relay = LocalRelay::builder().query_policy(RejectAllQueries).build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + subscription.subscribe(&client).await.expect("subscribe"); + + // However many rounds it runs, a relay that keeps closing the inbox + // never makes the node look healthy — this is what keeps the timeout + // machinery paused while trade messages are being lost. + for round in 0..3 { + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + check_inbox_health(&client, &subscription).await, + InboxStatus::Blind, + "round {round}: a relay that refuses every REQ must never read as listening" + ); + } + + relay.shutdown(); + } + + #[tokio::test] + async fn watchdog_reports_blind_when_no_relay_serves_the_inbox() { + // A client with no relays at all is the limit case of every relay + // being down: nothing can deliver a trade message. + let client = crate::util::mostro_nostr_client_options(None).build(); + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + + assert_eq!( + check_inbox_health(&client, &subscription).await, + InboxStatus::Blind + ); + } + + #[tokio::test] + async fn watchdog_ignores_a_disconnected_relay() { + use nostr_sdk::local_relay::LocalRelay; + + let live = LocalRelay::builder().build(); + live.run().await.expect("run local relay"); + let live_url = live.url().await; + + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(live_url.clone()).await.expect("add_relay"); + // Never connects: a relay that is down must not be re-subscribed on + // every tick, nor drag the verdict to blind while another one serves. + client + .add_relay("ws://127.0.0.1:1") + .await + .expect("add_relay"); + client.connect().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + subscription.subscribe(&client).await.expect("subscribe"); + tokio::time::sleep(Duration::from_millis(500)).await; + + assert_eq!( + check_inbox_health(&client, &subscription).await, + InboxStatus::Listening, + "one healthy relay is enough to keep hearing" + ); + + live.shutdown(); + } } diff --git a/src/main.rs b/src/main.rs index 6becd38e..c0f7b56b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,7 @@ use crate::config::{ get_db_pool, Settings, DB_POOL, LN_STATUS, MESSAGE_QUEUES, MOSTRO_CONFIG, NOSTR_CLIENT, }; use crate::db::find_held_invoices; -use crate::inbox::InboxSubscription; +use crate::inbox::{InboxHealth, InboxSubscription}; use crate::lightning::LnStatus; use crate::lightning::LndConnector; use crate::rpc::RpcServer; @@ -108,6 +108,12 @@ async fn main() -> Result<()> { } let inbox = InboxSubscription::new(mostro_keys.public_key(), transport.event_kind()); + // Install the inbox health record before the subscription goes out, so the + // watchdog and the scheduler read the same one from their own tasks. + if InboxHealth::new().install_global().is_err() { + tracing::warn!("Inbox health record already installed"); + } + let client = match get_nostr_client() { Ok(client) => client, Err(e) => { diff --git a/src/scheduler.rs b/src/scheduler.rs index 57651e9f..d2ab240b 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -54,10 +54,41 @@ pub async fn start_scheduler(ctx: AppContext) { job_update_bitcoin_prices().await; job_flush_messages_queue(ctx.clone()).await; job_refresh_active_pubkeys(ctx.clone()).await; + job_inbox_watchdog(ctx.clone()).await; info!("Scheduler Started"); } +/// How often the inbox watchdog audits the subscription across relays. +/// +/// Short enough that a lost ear is measured in seconds rather than the 60s +/// timeout tick, long enough that it is not a source of traffic on its own. +/// Hardcoded like the other maintenance intervals in this module. +const INBOX_WATCHDOG_INTERVAL: u64 = 30; + +/// Audit the daemon's Nostr inbox and re-subscribe any relay that stopped +/// serving it (see `crate::inbox`). +/// +/// The event loop already reacts to a `CLOSED` frame, but only to frames it +/// actually receives — the SDK's notification channel drops them when the +/// consumer lags, and some ways of losing a subscription produce no frame at +/// all. This job is the backstop, and the only thing that notices when *every* +/// relay has gone quiet. +async fn job_inbox_watchdog(ctx: AppContext) { + #[allow(deprecated)] + let event_kind = ctx.settings().mostro.transport.event_kind(); + let subscription = crate::inbox::InboxSubscription::new(ctx.keys().public_key(), event_kind); + + tokio::spawn(async move { + loop { + // Sleep first: at startup `main` has just subscribed, and a REQ + // still in flight would look exactly like a missing one. + tokio::time::sleep(tokio::time::Duration::from_secs(INBOX_WATCHDOG_INTERVAL)).await; + crate::inbox::check_inbox_health(ctx.nostr_client(), &subscription).await; + } + }); +} + /// Periodically rebuild the protocol-v2 anti-spam gate's active-trade-pubkey /// cache from the DB (spec §6 Phase 2). Status mutations are scattered across /// many handlers with no single choke-point, so a periodic full reload is the From 50b27a853d010f0f7f4c6244d7f7c408155f4a6c Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 17:02:43 -0300 Subject: [PATCH 05/25] feat: compensate order timeout deadlines for inbox downtime --- src/db.rs | 68 ++++++++++++++++++++++-- src/inbox.rs | 136 +++++++++++++++++++++++++++++++++++++++++++---- src/scheduler.rs | 31 ++++++++++- 3 files changed, 220 insertions(+), 15 deletions(-) diff --git a/src/db.rs b/src/db.rs index ec24968d..f440b380 100644 --- a/src/db.rs +++ b/src/db.rs @@ -10,6 +10,7 @@ use std::collections::HashSet; use std::fs::{set_permissions, Permissions}; use std::path::Path; use std::sync::Arc; +use std::time::Duration; use uuid::Uuid; // Constants for status filtering used across restore session functions @@ -675,10 +676,21 @@ pub async fn find_order_by_date(pool: &SqlitePool) -> Result, MostroE Ok(order) } -pub async fn find_order_by_seconds(pool: &SqlitePool) -> Result, MostroError> { +/// Orders whose waiting deadline has passed and are therefore candidates for +/// the timeout job. +/// +/// `grace` widens the deadline by time the daemon owes users — currently the +/// span its Nostr inbox spent unable to receive anything (see +/// [`crate::inbox::InboxHealth::timeout_debt`]). A user who answered on time +/// into a deaf node must not read as late, so the eligibility window is pushed +/// back by exactly as long as the node could not listen. +pub async fn find_order_by_seconds( + pool: &SqlitePool, + grace: Duration, +) -> Result, MostroError> { let mostro_settings = Settings::get_mostro(); let exp_seconds = mostro_settings.expiration_seconds as u64; - let expire_time = Timestamp::now() - exp_seconds; + let expire_time = Timestamp::now() - exp_seconds - grace.as_secs(); let order = sqlx::query_as::<_, Order>( r#" SELECT * @@ -5123,7 +5135,7 @@ mod migration_and_query_tests { ) .await; - let stale = find_order_by_seconds(&pool).await.unwrap(); + let stale = find_order_by_seconds(&pool, Duration::ZERO).await.unwrap(); assert_eq!(stale.len(), 1); assert_eq!(stale[0].id, stale_id); } @@ -5167,13 +5179,61 @@ mod migration_and_query_tests { let after = Order::by_id(&pool, id).await.unwrap().unwrap(); assert!(after.taken_at > 0, "take must persist taken_at"); - let stale = find_order_by_seconds(&pool).await.unwrap(); + let stale = find_order_by_seconds(&pool, Duration::ZERO).await.unwrap(); assert!( stale.is_empty(), "a just-taken order must not be timeout-cancel eligible" ); } + #[tokio::test] + async fn find_order_by_seconds_grace_spares_orders_the_daemon_could_not_hear() { + init_test_settings(); + let pool = migrated_pool().await; + + let exp_seconds = Settings::get_mostro().expiration_seconds as i64; + // Taken one minute past the deadline: late by wall time, and the + // caller is about to say the node was deaf for longer than that. + let taken_at = Timestamp::now().as_secs() as i64 - exp_seconds - 60; + let order_id = Uuid::new_v4(); + insert_order( + &pool, + order_id, + "sell", + "waiting-buyer-invoice", + Some(HEX_KEY_A), + Some(HEX_KEY_B), + HEX_KEY_B, + taken_at, + ) + .await; + + // No grace: the order is treated as late, which is what would cancel + // the escrow and slash the bond. + let without_grace = find_order_by_seconds(&pool, Duration::ZERO).await.unwrap(); + assert_eq!(without_grace.len(), 1); + assert_eq!(without_grace[0].id, order_id); + + // Owed more downtime than the order is late by: not late at all. + let with_grace = find_order_by_seconds(&pool, Duration::from_secs(300)) + .await + .unwrap(); + assert!( + with_grace.is_empty(), + "an order cannot be late for a window the daemon spent unable to listen" + ); + + // Once the debt wears below the overshoot, the order is late again. + let partly_repaid = find_order_by_seconds(&pool, Duration::from_secs(30)) + .await + .unwrap(); + assert_eq!( + partly_repaid.len(), + 1, + "grace only postpones the deadline, it does not remove it" + ); + } + #[tokio::test] async fn find_dispute_by_order_id_finds_and_misses() { let pool = migrated_pool().await; diff --git a/src/inbox.rs b/src/inbox.rs index f5876c73..b6767300 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -310,8 +310,12 @@ pub enum InstallError { struct HealthState { /// When the current outage began, if the inbox is deaf right now. blind_since: Option, - /// The most recent finished outage: when it ended, and how long it lasted. - last_outage: Option<(Instant, Duration)>, + /// Debt owed by finished outages, exact as of `owed_as_of`. It wears off + /// as time passes with the inbox listening, and is frozen while it is + /// blind — repaying it requires time the user could actually talk in. + owed: Duration, + /// When `owed` was last exact. `None` before the first outage. + owed_as_of: Option, } /// Tracks whether the daemon's ear is open, and for how long it was not. @@ -347,11 +351,20 @@ impl InboxHealth { fn observe(&self, status: InboxStatus, now: Instant) -> InboxStatus { let mut state = self.state.lock().expect("inbox health mutex poisoned"); match (status, state.blind_since) { - (InboxStatus::Blind, None) => state.blind_since = Some(now), + (InboxStatus::Blind, None) => { + // Freeze whatever an earlier outage still owes: from here on, + // no time passes that a user could have used to answer, so + // none of that debt gets repaid. + state.owed = settled_debt(&state, now); + state.owed_as_of = Some(now); + state.blind_since = Some(now); + } (InboxStatus::Listening, Some(since)) => { - let outage = now.saturating_duration_since(since); + // The frozen debt plus this outage; two outages in quick + // succession add up rather than cancelling each other out. + state.owed += now.saturating_duration_since(since); + state.owed_as_of = Some(now); state.blind_since = None; - state.last_outage = Some((now, outage)); } _ => {} } @@ -366,6 +379,46 @@ impl InboxHealth { .blind_since .is_some() } + + /// How much time the order-timeout clock currently owes users. + /// + /// Deadlines are measured against wall time, but a user cannot answer a + /// node that cannot hear — and because a message sent into a dead inbox is + /// lost rather than queued (the ten-second replay window, see the module + /// docs), they have to send it again once the ear is back. So the time the + /// inbox spent down is given back: the debt equals the outage when it ends + /// and decays to zero over an equal span, which is the same as saying the + /// timeout clock stood still while Mostro was deaf. + /// + /// Compensation is granted to every waiting order rather than only to + /// those in flight during the outage — an order taken *during* the blind + /// window gets slightly more grace than it strictly lost. That error is + /// deliberate: it delays a cancellation, where the opposite would slash an + /// honest user's bond for the node's own failure. + pub fn timeout_debt(&self) -> Duration { + self.timeout_debt_at(Instant::now()) + } + + fn timeout_debt_at(&self, now: Instant) -> Duration { + let state = self.state.lock().expect("inbox health mutex poisoned"); + match state.blind_since { + // Still deaf: the debt frozen at the start of this outage, plus + // every second it has run for. + Some(since) => state.owed + now.saturating_duration_since(since), + None => settled_debt(&state, now), + } + } +} + +/// The debt still outstanding at `now` while the inbox is listening: it starts +/// at the recorded amount and wears off second for second. +fn settled_debt(state: &HealthState, now: Instant) -> Duration { + match state.owed_as_of { + Some(as_of) => state + .owed + .saturating_sub(now.saturating_duration_since(as_of)), + None => Duration::ZERO, + } } /// Check every read relay, re-subscribing any that lost the inbox, and record @@ -778,15 +831,77 @@ mod tests { health.observe(InboxStatus::Listening, start + Duration::from_secs(90)); assert!(!health.is_blind()); - let state = health.state.lock().expect("lock"); - let (_, outage) = state.last_outage.expect("outage recorded"); assert_eq!( - outage, + health.timeout_debt_at(start + Duration::from_secs(90)), Duration::from_secs(90), "the recorded outage must span the whole blind window" ); } + #[test] + fn a_node_that_was_never_blind_owes_nothing() { + let health = InboxHealth::new(); + assert_eq!(health.timeout_debt_at(Instant::now()), Duration::ZERO); + } + + #[test] + fn debt_grows_while_blind_and_wears_off_after_recovery() { + let health = InboxHealth::new(); + let start = Instant::now(); + + health.observe(InboxStatus::Blind, start); + + // While deaf, the clock is stopped: the debt is the whole outage so far. + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(120)), + Duration::from_secs(120) + ); + + health.observe(InboxStatus::Listening, start + Duration::from_secs(300)); + + // On recovery, users are owed the full outage... + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(300)), + Duration::from_secs(300) + ); + // ...which then decays second for second, so a five-minute outage + // gives back five minutes and no more. + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(400)), + Duration::from_secs(200) + ); + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(600)), + Duration::ZERO + ); + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(10_000)), + Duration::ZERO, + "the debt must not linger once repaid" + ); + } + + #[test] + fn consecutive_outages_accumulate_their_debt() { + let health = InboxHealth::new(); + let start = Instant::now(); + + // A 100s outage, recovered at t=100 — debt 100s. + health.observe(InboxStatus::Blind, start); + health.observe(InboxStatus::Listening, start + Duration::from_secs(100)); + + // A second outage begins at t=140, when 60s of the first is still owed, + // and lasts 50s. + health.observe(InboxStatus::Blind, start + Duration::from_secs(140)); + health.observe(InboxStatus::Listening, start + Duration::from_secs(190)); + + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(190)), + Duration::from_secs(110), + "the second outage must not wipe out what the first still owed" + ); + } + #[test] fn health_ignores_repeated_healthy_observations() { let health = InboxHealth::new(); @@ -796,8 +911,9 @@ mod tests { health.observe(InboxStatus::Listening, now + Duration::from_secs(30)); assert!(!health.is_blind()); - assert!( - health.state.lock().expect("lock").last_outage.is_none(), + assert_eq!( + health.timeout_debt_at(now + Duration::from_secs(30)), + Duration::ZERO, "a node that was never blind has no outage to compensate for" ); } diff --git a/src/scheduler.rs b/src/scheduler.rs index d2ab240b..a0d36d3c 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -512,7 +512,36 @@ async fn job_cancel_orders(ctx: AppContext) { loop { info!("Check for order to republish for late actions of users"); - if let Ok(older_orders_list) = crate::db::find_order_by_seconds(pool).await { + // A timeout means "the user did not answer in time", and that + // conclusion is only sound while Mostro can hear. With the inbox + // down, an answer that was sent is simply never delivered — and + // the ten-second replay window means it is lost, not queued — so + // acting on the deadline would cancel escrows and slash bonds over + // the node's own deafness. Skip the whole tick: cancel, refund, + // republish and slash all rest on the same unsound premise. + if let Some(health) = crate::inbox::InboxHealth::global() { + if health.is_blind() { + warn!( + "scheduler_timeout: inbox is blind, holding order timeouts until it recovers" + ); + tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + continue; + } + } + + // Time the inbox spent deaf is given back to users before an order + // counts as late (see `InboxHealth::timeout_debt`). + let grace = crate::inbox::InboxHealth::global() + .map(|health| health.timeout_debt()) + .unwrap_or_default(); + if !grace.is_zero() { + info!( + "scheduler_timeout: extending order deadlines by {}s to make up for inbox downtime", + grace.as_secs() + ); + } + + if let Ok(older_orders_list) = crate::db::find_order_by_seconds(pool, grace).await { for order in older_orders_list.into_iter() { // The tick-start snapshot may be stale by the time this // iteration is reached — re-read and re-confirm before From 67ff82ab684d724829ef05227d3252f5e496a51d Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 17:09:39 -0300 Subject: [PATCH 06/25] docs: document inbox subscription lifecycle and recovery mechanisms --- docs/EVENT_ROUTING.md | 73 ++++++++++++++++++++++++++++++++++---- docs/STARTUP_AND_CONFIG.md | 6 ++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index e105a180..0f552618 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -6,6 +6,31 @@ How Nostr events become actions and side effects. - Source: `src/app.rs:run` - Steps: POW check → signature verify → recency guard → NIP-59 unwrap → parse `mostro_core::Message` → inner verify → `check_trade_index` → dispatch. +## The Inbox Subscription +- Source: `src/inbox.rs` +- Every trade message reaches Mostro over a single long-lived subscription, created at startup by `main` with a stable id (`InboxSubscription`) so that later frames can be attributed to it. Its filter is p-tagged to the node, restricted to the configured transport's event kind, and carries `limit(0)`: only live traffic is wanted. +- `run` consumes the whole notification stream, not just events. `ClientNotification::Message` carries the relay control plane and goes to `InboxKeeper`; `ClientNotification::Shutdown` ends the loop. + +### Recovering a lost ear +A relay can end a subscription at any time by sending `CLOSED`. The nostr-sdk removes the subscription for almost every reason prefix, and a removed subscription is never re-REQ'd, not even after a reconnect — so without handling, one frame from one relay leaves the daemon running, connected, and unable to receive anything. + +Two mechanisms keep the subscription alive: + +- `InboxKeeper::on_relay_message` reacts to a `CLOSED` naming the inbox by re-sending the REQ to that relay, paced by a per-relay backoff (immediate first retry, doubling to a five-minute ceiling, cleared when the relay answers with `EOSE`). +- `check_inbox_health`, run every 30 seconds by `job_inbox_watchdog`, audits each connected read relay and re-subscribes any that is no longer serving the inbox. This covers the losses that produce no frame the loop can see: a notification channel that dropped messages under lag, a REQ that failed to go out, a relay added after startup. + +Health is judged by the presence of the subscription, never by traffic volume: an instance with no trades in flight is legitimately silent. + +A relay re-subscribed during an audit is not counted as listening until the following round. Sending a REQ says nothing about whether the relay will honour it, and counting the attempt would report a healthy inbox indefinitely against a relay that closes it on principle. + +### NIP-42 +The daemon and price clients are built with a `SignerAuthenticator` over the node's keys (`src/util.rs:connect_nostr`). Without it a relay that gates reads behind authentication answers the REQ with `CLOSED "auth-required: …"`, which the SDK treats as permanent. The AUTH event is bound to the relay's challenge and URL, so it cannot be replayed elsewhere. + +### Messages lost while blind are not recovered +`accept_event` rejects anything whose `created_at` is older than ten seconds. A message sent while the inbox was down is therefore already too old to be accepted by the time the subscription returns, and re-subscribing with `since` instead of `limit(0)` would not change that. Whoever sent it has to send it again. + +Because those messages are lost rather than delayed, order timeouts cannot be trusted while the inbox is down — a user who answered on time would look silent. `job_cancel_orders` therefore skips its tick entirely while `InboxHealth` reports blind (no slash, no refund, no republish), and once the inbox recovers, `find_order_by_seconds` widens the eligibility window by the time the node spent deaf (`InboxHealth::timeout_debt`), decaying second for second until repaid. + ## Dispatch - Router: `src/app.rs:handle_message_action` - Maps `Action` → module function under `src/app/*`. @@ -26,20 +51,56 @@ How Nostr events become actions and side effects. ```mermaid sequenceDiagram participant Relay as Nostr Relay - participant Loop as app.rs (run) + participant EventLoop as app.rs (run) + participant Keeper as InboxKeeper participant Router as handle_message_action participant Mod as app/* participant DB as DB participant LND as LND - Relay-->>Loop: GiftWrap Event - Loop->>Loop: POW + verify + freshness - Loop->>Loop: unwrap + parse Message - Loop->>DB: check_trade_index - Loop->>Router: dispatch(Action) + Relay-->>EventLoop: GiftWrap Event + EventLoop->>EventLoop: POW + verify + freshness + EventLoop->>EventLoop: unwrap + parse Message + EventLoop->>DB: check_trade_index + EventLoop->>Router: dispatch(Action) Router->>Mod: handler(...) par side-effects Mod->>DB: read/write Mod->>LND: hold/settle/cancel/pay end + + Relay-->>EventLoop: CLOSED (inbox subscription) + EventLoop->>Keeper: on_relay_message + Keeper->>Relay: REQ (same subscription id) +``` + +The watchdog runs on its own schedule, independently of the loop above: + +```mermaid +sequenceDiagram + participant Job as job_inbox_watchdog + participant Relay as connected read relays + participant Health as InboxHealth + participant Timeouts as job_cancel_orders + + loop every 30s + Job->>Relay: still serving the inbox subscription? + alt not serving it + Job->>Relay: REQ (same subscription id) + end + alt none were serving it + Job->>Health: Blind + else at least one was + Job->>Health: Listening + end + end + + loop every 60s + Timeouts->>Health: is_blind / timeout_debt + alt blind + Timeouts->>Timeouts: skip the tick + else listening + Timeouts->>Timeouts: run, deadlines widened by the debt + end + end ``` diff --git a/docs/STARTUP_AND_CONFIG.md b/docs/STARTUP_AND_CONFIG.md index 2efad905..2b195169 100644 --- a/docs/STARTUP_AND_CONFIG.md +++ b/docs/STARTUP_AND_CONFIG.md @@ -100,6 +100,12 @@ Configuration is loaded from `~/.mostro/settings.toml` (template: `settings.tpl. - `relays` (Vec): List of Nostr relay URLs for event broadcasting - Default: `['ws://localhost:7000']` - Note: At least one relay required + - Relays that require NIP-42 authentication are supported: Mostro answers the + challenge with its own key. No configuration is needed. + - A relay that ends Mostro's inbox subscription is re-subscribed + automatically, and a watchdog audits the subscription every 30 seconds. If + no relay is serving it, the log carries `Mostro inbox is BLIND` and order + timeouts are held until it recovers. See `docs/EVENT_ROUTING.md`. **Lightning** (`src/config/types.rs:27-46`): - `lnd_cert_file` (String): Path to LND TLS certificate From 86d69189cdf2e084837f1d06dedd8d103bb4dfb6 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 17:41:29 -0300 Subject: [PATCH 07/25] feat: date first-audit blindness from startup, gate timeouts on confirmed listening --- src/inbox.rs | 107 +++++++++++++++++++++++++++++++++++++++++++---- src/scheduler.rs | 9 +++- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/src/inbox.rs b/src/inbox.rs index b6767300..e5a595aa 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -306,8 +306,15 @@ pub enum InstallError { AlreadyInstalled, } -#[derive(Debug, Default)] +#[derive(Debug)] struct HealthState { + /// The last verdict an audit reached. `None` until the first one runs: + /// startup is not evidence that the inbox works, and must not be read as + /// such (see [`InboxHealth::is_confirmed_listening`]). + verdict: Option, + /// When the record was installed, which is the earliest moment an outage + /// discovered by the first audit could have begun. + installed_at: Instant, /// When the current outage began, if the inbox is deaf right now. blind_since: Option, /// Debt owed by finished outages, exact as of `owed_as_of`. It wears off @@ -325,14 +332,32 @@ struct HealthState { /// window means a message sent into a dead inbox is gone rather than delayed /// (see the module docs). Punishing a user for a silence the node itself /// caused would be unfair, so the timeout clock stops while the inbox is down. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct InboxHealth { state: Mutex, } +impl Default for InboxHealth { + fn default() -> Self { + Self::new() + } +} + impl InboxHealth { pub fn new() -> Self { - Self::default() + Self::at(Instant::now()) + } + + fn at(installed_at: Instant) -> Self { + Self { + state: Mutex::new(HealthState { + verdict: None, + installed_at, + blind_since: None, + owed: Duration::ZERO, + owed_as_of: None, + }), + } } /// Install as the process-wide health record. @@ -350,6 +375,9 @@ impl InboxHealth { /// Record the current observation, returning the resulting status. fn observe(&self, status: InboxStatus, now: Instant) -> InboxStatus { let mut state = self.state.lock().expect("inbox health mutex poisoned"); + let first_verdict = state.verdict.is_none(); + state.verdict = Some(status); + match (status, state.blind_since) { (InboxStatus::Blind, None) => { // Freeze whatever an earlier outage still owes: from here on, @@ -357,7 +385,14 @@ impl InboxHealth { // none of that debt gets repaid. state.owed = settled_debt(&state, now); state.owed_as_of = Some(now); - state.blind_since = Some(now); + // A first audit that finds the inbox deaf has found an outage + // that was already running: the node has not heard anything + // since it came up, so that is when the outage began. + state.blind_since = Some(if first_verdict { + state.installed_at + } else { + now + }); } (InboxStatus::Listening, Some(since)) => { // The frozen debt plus this outage; two outages in quick @@ -372,6 +407,10 @@ impl InboxHealth { } /// Whether the inbox is deaf right now. + /// + /// A record that has never been audited is not blind — but neither is it + /// known to be listening, which is the question a caller about to act on a + /// user's silence should be asking. See [`Self::is_confirmed_listening`]. pub fn is_blind(&self) -> bool { self.state .lock() @@ -380,6 +419,22 @@ impl InboxHealth { .is_some() } + /// Whether an audit has actually confirmed that Mostro can hear. + /// + /// This is the predicate for anything that punishes a user for not + /// answering. It is deliberately false before the first audit: the daemon + /// subscribes at startup and the watchdog's first pass comes later, so + /// between the two there is a window in which a node that never obtained a + /// working inbox would otherwise look healthy and start cancelling orders + /// and slashing bonds over messages it was never in a position to receive. + pub fn is_confirmed_listening(&self) -> bool { + self.state + .lock() + .expect("inbox health mutex poisoned") + .verdict + == Some(InboxStatus::Listening) + } + /// How much time the order-timeout clock currently owes users. /// /// Deadlines are measured against wall time, but a user cannot answer a @@ -815,8 +870,8 @@ mod tests { #[test] fn health_records_an_outage_from_first_blindness_to_recovery() { - let health = InboxHealth::new(); let start = Instant::now(); + let health = InboxHealth::at(start); assert!(!health.is_blind(), "a fresh record starts out listening"); @@ -838,6 +893,44 @@ mod tests { ); } + #[test] + fn health_is_not_listening_until_an_audit_says_so() { + let health = InboxHealth::new(); + + // Startup is not evidence. Between `main` subscribing and the + // watchdog's first pass, a node whose inbox never worked would + // otherwise process timeouts as if it had been listening all along. + assert!( + !health.is_confirmed_listening(), + "an unaudited record must not authorise acting on a user's silence" + ); + assert!( + !health.is_blind(), + "nor should it claim an outage it has not observed" + ); + + health.observe(InboxStatus::Listening, Instant::now()); + assert!(health.is_confirmed_listening()); + } + + #[test] + fn a_blind_first_audit_dates_the_outage_from_startup() { + let start = Instant::now(); + let health = InboxHealth::at(start); + + // The watchdog's first pass comes some time after boot. Finding the + // inbox deaf then means it was deaf for that whole stretch, not just + // from the moment somebody looked. + health.observe(InboxStatus::Blind, start + Duration::from_secs(30)); + health.observe(InboxStatus::Listening, start + Duration::from_secs(90)); + + assert_eq!( + health.timeout_debt_at(start + Duration::from_secs(90)), + Duration::from_secs(90), + "the outage must be dated from startup, not from the first audit" + ); + } + #[test] fn a_node_that_was_never_blind_owes_nothing() { let health = InboxHealth::new(); @@ -846,8 +939,8 @@ mod tests { #[test] fn debt_grows_while_blind_and_wears_off_after_recovery() { - let health = InboxHealth::new(); let start = Instant::now(); + let health = InboxHealth::at(start); health.observe(InboxStatus::Blind, start); @@ -883,8 +976,8 @@ mod tests { #[test] fn consecutive_outages_accumulate_their_debt() { - let health = InboxHealth::new(); let start = Instant::now(); + let health = InboxHealth::at(start); // A 100s outage, recovered at t=100 — debt 100s. health.observe(InboxStatus::Blind, start); diff --git a/src/scheduler.rs b/src/scheduler.rs index a0d36d3c..75e7cdec 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -519,10 +519,15 @@ async fn job_cancel_orders(ctx: AppContext) { // acting on the deadline would cancel escrows and slash bonds over // the node's own deafness. Skip the whole tick: cancel, refund, // republish and slash all rest on the same unsound premise. + // + // The test is "an audit confirmed we can hear", not "no audit has + // reported deafness". This job's first tick runs immediately while + // the watchdog's comes later, so an unaudited record would let a + // node that never obtained a working inbox act on that window. if let Some(health) = crate::inbox::InboxHealth::global() { - if health.is_blind() { + if !health.is_confirmed_listening() { warn!( - "scheduler_timeout: inbox is blind, holding order timeouts until it recovers" + "scheduler_timeout: inbox not confirmed listening, holding order timeouts" ); tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; continue; From 1525b5528dfaf4edba8ddf8816f466e93f106c92 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 17:52:34 -0300 Subject: [PATCH 08/25] docs: refactor timeout compensation from decaying allowance to per-order downtime tracking --- docs/EVENT_ROUTING.md | 15 +- src/db.rs | 20 ++- src/inbox.rs | 361 +++++++++++++++++++++++++++--------------- src/scheduler.rs | 44 +++-- 4 files changed, 286 insertions(+), 154 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 0f552618..a2a5aaed 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -29,7 +29,11 @@ The daemon and price clients are built with a `SignerAuthenticator` over the nod ### Messages lost while blind are not recovered `accept_event` rejects anything whose `created_at` is older than ten seconds. A message sent while the inbox was down is therefore already too old to be accepted by the time the subscription returns, and re-subscribing with `since` instead of `limit(0)` would not change that. Whoever sent it has to send it again. -Because those messages are lost rather than delayed, order timeouts cannot be trusted while the inbox is down — a user who answered on time would look silent. `job_cancel_orders` therefore skips its tick entirely while `InboxHealth` reports blind (no slash, no refund, no republish), and once the inbox recovers, `find_order_by_seconds` widens the eligibility window by the time the node spent deaf (`InboxHealth::timeout_debt`), decaying second for second until repaid. +Because those messages are lost rather than delayed, order timeouts cannot be trusted while the inbox is down — a user who answered on time would look silent. `job_cancel_orders` therefore skips its tick entirely unless an audit has confirmed the daemon is listening (`InboxHealth::is_confirmed_listening`): no slash, no refund, no republish. Startup counts as unconfirmed, since the daemon subscribes before the watchdog's first pass. + +Once the inbox recovers, each order is credited the downtime **it** waited through. `InboxHealth` keeps the wall-clock windows during which the node was deaf; `blind_seconds_since(taken_at)` intersects them with the order's own wait. An order already waiting when a relay went quiet is owed all of that outage; one taken after it ended is owed nothing. The query widens its window by `max_blind_seconds` so no eligible order is missed, and the exact per-order figure decides. + +The credit has to be per order rather than one global allowance: a single figure either under-credits an order that waited through the whole outage or hands the same credit to one taken long afterwards. ## Dispatch - Router: `src/app.rs:handle_message_action` @@ -96,11 +100,12 @@ sequenceDiagram end loop every 60s - Timeouts->>Health: is_blind / timeout_debt - alt blind + Timeouts->>Health: is_confirmed_listening? + alt not confirmed Timeouts->>Timeouts: skip the tick - else listening - Timeouts->>Timeouts: run, deadlines widened by the debt + else confirmed + Timeouts->>Health: blind_seconds_since(taken_at) per order + Timeouts->>Timeouts: run, each order credited its own downtime end end ``` diff --git a/src/db.rs b/src/db.rs index f440b380..27173c9c 100644 --- a/src/db.rs +++ b/src/db.rs @@ -679,11 +679,13 @@ pub async fn find_order_by_date(pool: &SqlitePool) -> Result, MostroE /// Orders whose waiting deadline has passed and are therefore candidates for /// the timeout job. /// -/// `grace` widens the deadline by time the daemon owes users — currently the -/// span its Nostr inbox spent unable to receive anything (see -/// [`crate::inbox::InboxHealth::timeout_debt`]). A user who answered on time -/// into a deaf node must not read as late, so the eligibility window is pushed -/// back by exactly as long as the node could not listen. +/// `grace` widens the window by the **most** any order could be owed for time +/// the daemon spent unable to receive anything (see +/// [`crate::inbox::InboxHealth::max_blind_seconds`]), so that no order the +/// caller may still have to spare is filtered out here. It deliberately +/// over-selects: what a given order is actually owed depends on when it began +/// waiting, which this query cannot express, so the caller applies the exact +/// per-order figure to the rows returned. pub async fn find_order_by_seconds( pool: &SqlitePool, grace: Duration, @@ -5223,12 +5225,14 @@ mod migration_and_query_tests { "an order cannot be late for a window the daemon spent unable to listen" ); - // Once the debt wears below the overshoot, the order is late again. - let partly_repaid = find_order_by_seconds(&pool, Duration::from_secs(30)) + // A grace smaller than the overshoot still selects it: the query only + // has to avoid filtering out rows the caller may spare, and the caller + // decides from each order's own downtime. + let smaller_grace = find_order_by_seconds(&pool, Duration::from_secs(30)) .await .unwrap(); assert_eq!( - partly_repaid.len(), + smaller_grace.len(), 1, "grace only postpones the deadline, it does not remove it" ); diff --git a/src/inbox.rs b/src/inbox.rs index e5a595aa..b490ce31 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -306,6 +306,35 @@ pub enum InstallError { AlreadyInstalled, } +/// One stretch during which the daemon could not hear. +/// +/// Timestamps are wall-clock seconds, the same base as an order's `taken_at`, +/// because that is what these windows are ultimately intersected against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BlindWindow { + start: i64, + /// `None` while the outage is still running. + end: Option, +} + +impl BlindWindow { + /// Seconds of this window that fall inside `[from, to]`. + fn overlap(&self, from: i64, to: i64, now: i64) -> i64 { + let end = self.end.unwrap_or(now); + (end.min(to) - self.start.max(from)).max(0) + } +} + +/// Windows that ended longer ago than this are dropped: no order the timeout +/// job can still be looking at was waiting back then, so they can no longer +/// change any verdict. Generous next to `expiration_seconds` (900s by default) +/// so the bound is never the reason a user loses compensation. +const BLIND_WINDOW_RETENTION_SECS: i64 = 7 * 24 * 3600; + +/// Hard cap on retained windows, so a relay flapping in a tight loop cannot +/// grow this unboundedly between prunes. +const MAX_BLIND_WINDOWS: usize = 512; + #[derive(Debug)] struct HealthState { /// The last verdict an audit reached. `None` until the first one runs: @@ -314,18 +343,18 @@ struct HealthState { verdict: Option, /// When the record was installed, which is the earliest moment an outage /// discovered by the first audit could have begun. - installed_at: Instant, - /// When the current outage began, if the inbox is deaf right now. - blind_since: Option, - /// Debt owed by finished outages, exact as of `owed_as_of`. It wears off - /// as time passes with the inbox listening, and is frozen while it is - /// blind — repaying it requires time the user could actually talk in. - owed: Duration, - /// When `owed` was last exact. `None` before the first outage. - owed_as_of: Option, + installed_at: i64, + /// Every outage this process has seen, oldest first, pruned by age. + windows: Vec, +} + +impl HealthState { + fn blind_now(&self) -> Option<&BlindWindow> { + self.windows.last().filter(|w| w.end.is_none()) + } } -/// Tracks whether the daemon's ear is open, and for how long it was not. +/// Tracks whether the daemon's ear is open, and when it was not. /// /// The scheduler's timeout machinery reads this: an order is only "late" if /// Mostro was in a position to hear from the user, and the ten-second replay @@ -345,17 +374,15 @@ impl Default for InboxHealth { impl InboxHealth { pub fn new() -> Self { - Self::at(Instant::now()) + Self::at(now_secs()) } - fn at(installed_at: Instant) -> Self { + fn at(installed_at: i64) -> Self { Self { state: Mutex::new(HealthState { verdict: None, installed_at, - blind_since: None, - owed: Duration::ZERO, - owed_as_of: None, + windows: Vec::new(), }), } } @@ -373,36 +400,40 @@ impl InboxHealth { } /// Record the current observation, returning the resulting status. - fn observe(&self, status: InboxStatus, now: Instant) -> InboxStatus { + fn observe(&self, status: InboxStatus, now: i64) -> InboxStatus { let mut state = self.state.lock().expect("inbox health mutex poisoned"); let first_verdict = state.verdict.is_none(); state.verdict = Some(status); - match (status, state.blind_since) { - (InboxStatus::Blind, None) => { - // Freeze whatever an earlier outage still owes: from here on, - // no time passes that a user could have used to answer, so - // none of that debt gets repaid. - state.owed = settled_debt(&state, now); - state.owed_as_of = Some(now); + match (status, state.blind_now().is_some()) { + (InboxStatus::Blind, false) => { // A first audit that finds the inbox deaf has found an outage // that was already running: the node has not heard anything // since it came up, so that is when the outage began. - state.blind_since = Some(if first_verdict { + let start = if first_verdict { state.installed_at } else { now - }); + }; + state.windows.push(BlindWindow { start, end: None }); } - (InboxStatus::Listening, Some(since)) => { - // The frozen debt plus this outage; two outages in quick - // succession add up rather than cancelling each other out. - state.owed += now.saturating_duration_since(since); - state.owed_as_of = Some(now); - state.blind_since = None; + (InboxStatus::Listening, true) => { + if let Some(open) = state.windows.last_mut() { + open.end = Some(now); + } } _ => {} } + + state.windows.retain(|w| match w.end { + Some(end) => end > now - BLIND_WINDOW_RETENTION_SECS, + None => true, + }); + if state.windows.len() > MAX_BLIND_WINDOWS { + let excess = state.windows.len() - MAX_BLIND_WINDOWS; + state.windows.drain(..excess); + } + status } @@ -415,7 +446,7 @@ impl InboxHealth { self.state .lock() .expect("inbox health mutex poisoned") - .blind_since + .blind_now() .is_some() } @@ -435,45 +466,54 @@ impl InboxHealth { == Some(InboxStatus::Listening) } - /// How much time the order-timeout clock currently owes users. + /// Seconds the inbox was deaf between `from` and now. /// - /// Deadlines are measured against wall time, but a user cannot answer a - /// node that cannot hear — and because a message sent into a dead inbox is - /// lost rather than queued (the ten-second replay window, see the module - /// docs), they have to send it again once the ear is back. So the time the - /// inbox spent down is given back: the debt equals the outage when it ends - /// and decays to zero over an equal span, which is the same as saying the - /// timeout clock stood still while Mostro was deaf. - /// - /// Compensation is granted to every waiting order rather than only to - /// those in flight during the outage — an order taken *during* the blind - /// window gets slightly more grace than it strictly lost. That error is - /// deliberate: it delays a cancellation, where the opposite would slash an - /// honest user's bond for the node's own failure. - pub fn timeout_debt(&self) -> Duration { - self.timeout_debt_at(Instant::now()) + /// This is the compensation a single order is owed, and it is computed per + /// order on purpose. A deadline is wall-clock, but the user is answering a + /// node that has to be listening for the answer to land — and because a + /// message sent into a dead inbox is lost rather than queued (the + /// ten-second replay window, see the module docs), they must send it again + /// once the ear is back. So an order's clock effectively stops for exactly + /// the outages that overlap its own waiting period: an order that was + /// already waiting through an outage is owed all of it, one taken + /// afterwards is owed nothing. + pub fn blind_seconds_since(&self, from: i64) -> i64 { + self.blind_seconds_between(from, now_secs()) } - fn timeout_debt_at(&self, now: Instant) -> Duration { + fn blind_seconds_between(&self, from: i64, to: i64) -> i64 { let state = self.state.lock().expect("inbox health mutex poisoned"); - match state.blind_since { - // Still deaf: the debt frozen at the start of this outage, plus - // every second it has run for. - Some(since) => state.owed + now.saturating_duration_since(since), - None => settled_debt(&state, now), - } + state.windows.iter().map(|w| w.overlap(from, to, to)).sum() } -} -/// The debt still outstanding at `now` while the inbox is listening: it starts -/// at the recorded amount and wears off second for second. -fn settled_debt(state: &HealthState, now: Instant) -> Duration { - match state.owed_as_of { - Some(as_of) => state - .owed - .saturating_sub(now.saturating_duration_since(as_of)), - None => Duration::ZERO, + /// Upper bound on what any order could be owed, for callers that need to + /// widen a query before applying the exact per-order figure. + pub fn max_blind_seconds(&self) -> i64 { + let now = now_secs(); + let state = self.state.lock().expect("inbox health mutex poisoned"); + state + .windows + .iter() + .map(|w| w.end.unwrap_or(now) - w.start) + .sum::() + .max(0) } + + /// How long the current outage has been running, or zero if listening. + pub fn blind_for_secs(&self) -> i64 { + let now = now_secs(); + self.state + .lock() + .expect("inbox health mutex poisoned") + .blind_now() + .map(|w| (now - w.start).max(0)) + .unwrap_or(0) + } +} + +/// Wall-clock seconds, the base an order's `taken_at` is recorded in. +fn now_secs() -> i64 { + Timestamp::now().as_secs() as i64 } /// Check every read relay, re-subscribing any that lost the inbox, and record @@ -527,7 +567,7 @@ pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscriptio if let Some(health) = InboxHealth::global() { let was_blind = health.is_blind(); - health.observe(status, Instant::now()); + health.observe(status, now_secs()); match (was_blind, status) { (false, InboxStatus::Blind) => error!( @@ -866,36 +906,39 @@ mod tests { relay.shutdown(); } - // ──────────────────────────────── watchdog ──────────────────────────────── + // ───────────────────────────── health record ───────────────────────────── + + /// Health observations are wall-clock based, so tests drive a fixed origin + /// rather than the real clock. + const T0: i64 = 1_700_000_000; #[test] fn health_records_an_outage_from_first_blindness_to_recovery() { - let start = Instant::now(); - let health = InboxHealth::at(start); + let health = InboxHealth::at(T0); assert!(!health.is_blind(), "a fresh record starts out listening"); - health.observe(InboxStatus::Blind, start); + health.observe(InboxStatus::Blind, T0); assert!(health.is_blind()); // Staying blind must not restart the clock — the outage began at the - // first observation, and that is what the timeout discount is owed on. - health.observe(InboxStatus::Blind, start + Duration::from_secs(30)); + // first observation, and that is what an order is owed. + health.observe(InboxStatus::Blind, T0 + 30); assert!(health.is_blind()); - health.observe(InboxStatus::Listening, start + Duration::from_secs(90)); + health.observe(InboxStatus::Listening, T0 + 90); assert!(!health.is_blind()); assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(90)), - Duration::from_secs(90), + health.blind_seconds_between(T0, T0 + 90), + 90, "the recorded outage must span the whole blind window" ); } #[test] fn health_is_not_listening_until_an_audit_says_so() { - let health = InboxHealth::new(); + let health = InboxHealth::at(T0); // Startup is not evidence. Between `main` subscribing and the // watchdog's first pass, a node whose inbox never worked would @@ -909,108 +952,164 @@ mod tests { "nor should it claim an outage it has not observed" ); - health.observe(InboxStatus::Listening, Instant::now()); + health.observe(InboxStatus::Listening, T0); assert!(health.is_confirmed_listening()); } #[test] fn a_blind_first_audit_dates_the_outage_from_startup() { - let start = Instant::now(); - let health = InboxHealth::at(start); + let health = InboxHealth::at(T0); // The watchdog's first pass comes some time after boot. Finding the // inbox deaf then means it was deaf for that whole stretch, not just // from the moment somebody looked. - health.observe(InboxStatus::Blind, start + Duration::from_secs(30)); - health.observe(InboxStatus::Listening, start + Duration::from_secs(90)); + health.observe(InboxStatus::Blind, T0 + 30); + health.observe(InboxStatus::Listening, T0 + 90); assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(90)), - Duration::from_secs(90), + health.blind_seconds_between(T0, T0 + 90), + 90, "the outage must be dated from startup, not from the first audit" ); } #[test] fn a_node_that_was_never_blind_owes_nothing() { - let health = InboxHealth::new(); - assert_eq!(health.timeout_debt_at(Instant::now()), Duration::ZERO); + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + + assert_eq!(health.blind_seconds_between(T0, T0 + 10_000), 0); + assert_eq!(health.max_blind_seconds(), 0); } + // ──────────────────── what a single order is owed ──────────────────── + #[test] - fn debt_grows_while_blind_and_wears_off_after_recovery() { - let start = Instant::now(); - let health = InboxHealth::at(start); + fn an_order_is_owed_only_the_downtime_it_waited_through() { + let health = InboxHealth::at(T0); + // One outage: [T0+100, T0+400], five minutes. + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 400); + + let now = T0 + 1_000; + + // Waiting since before it started: owed the whole outage. + assert_eq!(health.blind_seconds_between(T0, now), 300); + // Taken midway through: owed only the remainder. + assert_eq!(health.blind_seconds_between(T0 + 250, now), 150); + // Taken after it ended: owed nothing. This is what a single global + // allowance got wrong — it credited orders that never lost a second. + assert_eq!(health.blind_seconds_between(T0 + 500, now), 0); + } + + #[test] + fn compensation_does_not_evaporate_as_time_passes() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 400); + + // The debt an order carries is a property of when it waited, not of + // how long ago the outage was. A decaying allowance wore off at the + // same rate the deadline advanced, so it compensated almost nothing. + for probe in [400, 700, 5_000, 50_000] { + assert_eq!( + health.blind_seconds_between(T0, T0 + probe), + 300, + "an order waiting since T0 is owed the outage regardless of when we ask" + ); + } + } - health.observe(InboxStatus::Blind, start); + #[test] + fn an_order_waiting_through_an_outage_survives_its_nominal_deadline() { + // The regression in full: 900s timeout, an order taken at T0, and a + // 300s outage right at the start. Under the old decaying allowance + // this order was cancelled at ~T0+900, having had only 600s of + // listening time. + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Blind, T0); + health.observe(InboxStatus::Listening, T0 + 300); + + let exp_seconds = 900i64; + let late_at = |now: i64| { + let owed = health.blind_seconds_between(T0, now); + (now - T0) >= exp_seconds + owed + }; - // While deaf, the clock is stopped: the debt is the whole outage so far. - assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(120)), - Duration::from_secs(120) + assert!(!late_at(T0 + 900), "cancelled after only 600s of listening"); + assert!(!late_at(T0 + 1_199)); + assert!( + late_at(T0 + 1_200), + "and it must still expire once it has had its full 900s" ); + } - health.observe(InboxStatus::Listening, start + Duration::from_secs(300)); + #[test] + fn consecutive_outages_accumulate_their_debt() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 200); + health.observe(InboxStatus::Blind, T0 + 240); + health.observe(InboxStatus::Listening, T0 + 290); - // On recovery, users are owed the full outage... - assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(300)), - Duration::from_secs(300) - ); - // ...which then decays second for second, so a five-minute outage - // gives back five minutes and no more. - assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(400)), - Duration::from_secs(200) - ); assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(600)), - Duration::ZERO + health.blind_seconds_between(T0, T0 + 1_000), + 150, + "an order waiting through both outages is owed both" ); assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(10_000)), - Duration::ZERO, - "the debt must not linger once repaid" + health.blind_seconds_between(T0 + 210, T0 + 1_000), + 50, + "one taken between them is owed only the second" ); } #[test] - fn consecutive_outages_accumulate_their_debt() { - let start = Instant::now(); - let health = InboxHealth::at(start); - - // A 100s outage, recovered at t=100 — debt 100s. - health.observe(InboxStatus::Blind, start); - health.observe(InboxStatus::Listening, start + Duration::from_secs(100)); + fn an_ongoing_outage_counts_up_to_now() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); - // A second outage begins at t=140, when 60s of the first is still owed, - // and lasts 50s. - health.observe(InboxStatus::Blind, start + Duration::from_secs(140)); - health.observe(InboxStatus::Listening, start + Duration::from_secs(190)); + assert_eq!(health.blind_seconds_between(T0, T0 + 400), 300); + assert_eq!(health.blind_seconds_between(T0, T0 + 900), 800); + } - assert_eq!( - health.timeout_debt_at(start + Duration::from_secs(190)), - Duration::from_secs(110), - "the second outage must not wipe out what the first still owed" - ); + #[test] + fn stale_windows_are_pruned() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 200); + + // Far past the retention horizon, the old window is dropped rather + // than accumulating for the life of the process. + let much_later = T0 + BLIND_WINDOW_RETENTION_SECS + 1_000; + health.observe(InboxStatus::Listening, much_later); + + assert_eq!(health.blind_seconds_between(T0, much_later), 0); + assert!(health.state.lock().expect("lock").windows.is_empty()); } #[test] fn health_ignores_repeated_healthy_observations() { - let health = InboxHealth::new(); - let now = Instant::now(); + let health = InboxHealth::at(T0); - health.observe(InboxStatus::Listening, now); - health.observe(InboxStatus::Listening, now + Duration::from_secs(30)); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Listening, T0 + 30); assert!(!health.is_blind()); assert_eq!( - health.timeout_debt_at(now + Duration::from_secs(30)), - Duration::ZERO, + health.blind_seconds_between(T0, T0 + 30), + 0, "a node that was never blind has no outage to compensate for" ); } + // ──────────────────────────────── watchdog ──────────────────────────────── + #[tokio::test] async fn watchdog_resubscribes_a_relay_that_lost_the_inbox() { use nostr_sdk::local_relay::LocalRelay; diff --git a/src/scheduler.rs b/src/scheduler.rs index 75e7cdec..2805ad2d 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -19,8 +19,9 @@ use nostr_sdk::prelude::EventBuilder; use nostr_sdk::prelude::{FinalizeEvent, Kind as NostrKind, Nip65Tag, Tag}; use std::collections::HashSet; use std::sync::Arc; +use std::time::Duration; use tokio::sync::RwLock; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use util::{enqueue_order_msg, get_nostr_relays, send_dm, update_order_event}; pub async fn start_scheduler(ctx: AppContext) { @@ -534,19 +535,25 @@ async fn job_cancel_orders(ctx: AppContext) { } } - // Time the inbox spent deaf is given back to users before an order - // counts as late (see `InboxHealth::timeout_debt`). - let grace = crate::inbox::InboxHealth::global() - .map(|health| health.timeout_debt()) - .unwrap_or_default(); - if !grace.is_zero() { + // Compensation for inbox downtime is per order, and applied in two + // steps. The query widens its window by the most any order could + // be owed, so nothing eligible is missed; the exact figure — the + // downtime that overlaps *this* order's own wait — then decides. + // A single global allowance cannot do this: it would either + // under-credit an order that waited through the whole outage or + // hand the same credit to one taken long after it ended. + let health = crate::inbox::InboxHealth::global(); + let max_grace = health.map(|h| h.max_blind_seconds()).unwrap_or(0); + if max_grace > 0 { info!( - "scheduler_timeout: extending order deadlines by {}s to make up for inbox downtime", - grace.as_secs() + "scheduler_timeout: up to {max_grace}s of inbox downtime is credited against order deadlines" ); } - if let Ok(older_orders_list) = crate::db::find_order_by_seconds(pool, grace).await { + if let Ok(older_orders_list) = + crate::db::find_order_by_seconds(pool, Duration::from_secs(max_grace.max(0) as u64)) + .await + { for order in older_orders_list.into_iter() { // The tick-start snapshot may be stale by the time this // iteration is reached — re-read and re-confirm before @@ -558,6 +565,23 @@ async fn job_cancel_orders(ctx: AppContext) { else { continue; }; + + // Give this order back the downtime it actually waited + // through. Orders taken after the outage are owed nothing + // and fall through unchanged. + if let Some(health) = health { + let owed = health.blind_seconds_since(order.taken_at); + let waited = + nostr_sdk::prelude::Timestamp::now().as_secs() as i64 - order.taken_at; + if waited < exp_seconds as i64 + owed { + debug!( + "scheduler_timeout: order {} not late yet; {owed}s of its wait was inbox downtime", + order.id + ); + continue; + } + } + // Check if order is a sell order and Buyer is not sending the invoice for too much time. // Same if seller is not paying hold invoice if order.status == Status::WaitingBuyerInvoice.to_string() From b6dd1a57cbd03d7124245c07030a35d8e852f276 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 17:59:25 -0300 Subject: [PATCH 09/25] feat: defer inbox subscription until notification stream exists to prevent EOSE loss --- src/app.rs | 27 +++++++++++++++++++++++---- src/inbox.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 15 ++++++++------- 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/app.rs b/src/app.rs index 8767e322..f8247c9c 100644 --- a/src/app.rs +++ b/src/app.rs @@ -458,13 +458,24 @@ pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { // gate is meaningless for v1 (gift wraps are signed by throwaway keys). let pow_first_contact = ctx.settings().mostro.effective_pow_first_contact(); let is_v2 = accepted_kind.as_u16() == crate::config::constants::DM_EVENT_KIND; - // Same id and filter `main.rs` subscribed with — the inbox identity is - // derived, not passed around (see `crate::inbox`). - let mut keeper = InboxKeeper::new(InboxSubscription::new(my_keys.public_key(), accepted_kind)); + // The inbox identity is derived here rather than passed around (see + // `crate::inbox`). + let subscription = InboxSubscription::new(my_keys.public_key(), accepted_kind); + let mut keeper = InboxKeeper::new(subscription.clone()); + let mut subscribed = false; loop { let mut notifications = client.notifications(); + // The REQ goes out only once this stream exists. A notification + // receiver never sees what was delivered before it was created, so + // subscribing any earlier throws away the relay's EOSE — and every + // event that lands while the rest of the daemon is still booting. + if !subscribed { + subscription.subscribe(client).await?; + subscribed = true; + } + while let Some(notification) = notifications.next().await { match notification { ClientNotification::Event { event, .. } => { @@ -518,11 +529,19 @@ pub async fn run_cashu(ctx: AppContext) -> Result<()> { let accepted_kind = ctx.settings().mostro.transport.event_kind(); let pow_first_contact = ctx.settings().mostro.effective_pow_first_contact(); let is_v2 = accepted_kind.as_u16() == crate::config::constants::DM_EVENT_KIND; - let mut keeper = InboxKeeper::new(InboxSubscription::new(my_keys.public_key(), accepted_kind)); + let subscription = InboxSubscription::new(my_keys.public_key(), accepted_kind); + let mut keeper = InboxKeeper::new(subscription.clone()); + let mut subscribed = false; loop { let mut notifications = client.notifications(); + // Subscribe only once the stream exists — see `run`. + if !subscribed { + subscription.subscribe(client).await?; + subscribed = true; + } + while let Some(notification) = notifications.next().await { match notification { ClientNotification::Event { event, .. } => { diff --git a/src/inbox.rs b/src/inbox.rs index b490ce31..4a540f9d 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -875,6 +875,52 @@ mod tests { relay.shutdown(); } + #[tokio::test] + async fn a_receiver_created_after_the_req_misses_its_eose() { + use futures::StreamExt; + use nostr_sdk::local_relay::LocalRelay; + + // Why the event loop must subscribe *after* taking its notification + // stream: the SDK delivers nothing that predates the receiver, so a + // REQ sent earlier loses its EOSE — and any event arriving meanwhile. + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + + // Subscribe first, listen second — the order this module avoids. + subscription.subscribe(&client).await.expect("subscribe"); + tokio::time::sleep(Duration::from_millis(500)).await; + let mut late = client.notifications(); + + let saw_eose = tokio::time::timeout(Duration::from_secs(2), async { + while let Some(notification) = late.next().await { + if let ClientNotification::Message { message, .. } = notification { + if let RelayMessage::EndOfStoredEvents(id) = &*message { + if id.as_ref() == subscription.id() { + return true; + } + } + } + } + false + }) + .await + .unwrap_or(false); + + assert!( + !saw_eose, + "SDK behaviour changed: a late receiver now sees earlier frames, so the \ + subscribe-after-stream ordering in `app::run` could be relaxed" + ); + + relay.shutdown(); + } + #[tokio::test] async fn without_the_keeper_a_closed_inbox_stays_dead() { use nostr_sdk::local_relay::LocalRelay; diff --git a/src/main.rs b/src/main.rs index c0f7b56b..9eac8db5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,7 @@ use crate::config::{ get_db_pool, Settings, DB_POOL, LN_STATUS, MESSAGE_QUEUES, MOSTRO_CONFIG, NOSTR_CLIENT, }; use crate::db::find_held_invoices; -use crate::inbox::{InboxHealth, InboxSubscription}; +use crate::inbox::InboxHealth; use crate::lightning::LnStatus; use crate::lightning::LndConnector; use crate::rpc::RpcServer; @@ -106,10 +106,8 @@ async fn main() -> Result<()> { support protocol v2. See https://github.com/MostroP2P/mostro/issues/786" ); } - let inbox = InboxSubscription::new(mostro_keys.public_key(), transport.event_kind()); - - // Install the inbox health record before the subscription goes out, so the - // watchdog and the scheduler read the same one from their own tasks. + // Install the inbox health record before anything can observe the inbox, + // so the watchdog and the scheduler read the same one from their own tasks. if InboxHealth::new().install_global().is_err() { tracing::warn!("Inbox health record already installed"); } @@ -123,8 +121,11 @@ async fn main() -> Result<()> { } }; - // Client subscription - inbox.subscribe(client).await?; + // The inbox REQ is sent by the event loop (`app::run` / `app::run_cashu`), + // which subscribes only after its notification stream exists. Sending it + // from here would put it ahead of any receiver, and the SDK delivers + // nothing that predates one — the relay's EOSE and any trade message + // arriving during the rest of this boot would be dropped on the floor. // Publish NIP-01 kind 0 metadata event let mostro_settings = Settings::get_mostro(); From 2e56872452a60be872fc78750d88f4450a22ab4d Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 18:12:00 -0300 Subject: [PATCH 10/25] feat: require relay EOSE acknowledgement before counting inbox as healthy --- docs/EVENT_ROUTING.md | 4 +- src/inbox.rs | 206 ++++++++++++++++++++++++++++++++++++------ src/scheduler.rs | 4 +- 3 files changed, 181 insertions(+), 33 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index a2a5aaed..c3e59818 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -19,9 +19,9 @@ Two mechanisms keep the subscription alive: - `InboxKeeper::on_relay_message` reacts to a `CLOSED` naming the inbox by re-sending the REQ to that relay, paced by a per-relay backoff (immediate first retry, doubling to a five-minute ceiling, cleared when the relay answers with `EOSE`). - `check_inbox_health`, run every 30 seconds by `job_inbox_watchdog`, audits each connected read relay and re-subscribes any that is no longer serving the inbox. This covers the losses that produce no frame the loop can see: a notification channel that dropped messages under lag, a REQ that failed to go out, a relay added after startup. -Health is judged by the presence of the subscription, never by traffic volume: an instance with no trades in flight is legitimately silent. +Health is judged by the subscription, never by traffic volume: an instance with no trades in flight is legitimately silent. -A relay re-subscribed during an audit is not counted as listening until the following round. Sending a REQ says nothing about whether the relay will honour it, and counting the attempt would report a healthy inbox indefinitely against a relay that closes it on principle. +A relay counts as serving the inbox only once **it** has said so, by answering the REQ with an `EOSE` that the event loop recorded in `InboxHealth`. The SDK's own subscription map is not evidence — it records what Mostro sent, so a relay that holds the connection open and quietly drops the REQ still appears subscribed there, and the daemon would resume order timeouts while deaf. The same rule means a relay re-subscribed during an audit does not count until it answers, which costs one interval before recovery is declared and errs toward keeping the timeout clock frozen slightly longer than strictly needed. ### NIP-42 The daemon and price clients are built with a `SignerAuthenticator` over the node's keys (`src/util.rs:connect_nostr`). Without it a relay that gates reads behind authentication answers the REQ with `CLOSED "auth-required: …"`, which the SDK treats as permanent. The AUTH event is bound to the relay's challenge and URL, so it cannot be replayed elsewhere. diff --git a/src/inbox.rs b/src/inbox.rs index 4a540f9d..d646b084 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -45,8 +45,8 @@ //! are not, and records the verdict in [`InboxHealth`] so the rest of the //! daemon can tell whether Mostro is currently able to hear anything at all. -use std::collections::HashMap; -use std::sync::{Mutex, OnceLock}; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use nostr_sdk::prelude::*; @@ -166,13 +166,22 @@ pub struct InboxKeeper { /// Only holds relays that are currently failing; a relay that accepts the /// REQ is dropped from the map, so the steady state is empty. backoff: HashMap, + /// Where relay acknowledgements are recorded. The event loop is the only + /// place an `EOSE` can be observed, but the watchdog is what acts on it, + /// so the fact has to be shared rather than kept here. + health: Option>, } impl InboxKeeper { pub fn new(subscription: InboxSubscription) -> Self { + Self::with_health(subscription, InboxHealth::global()) + } + + pub fn with_health(subscription: InboxSubscription, health: Option>) -> Self { Self { subscription, backoff: HashMap::new(), + health, } } @@ -200,8 +209,13 @@ impl InboxKeeper { RelayMessage::EndOfStoredEvents(subscription_id) if subscription_id.as_ref() == self.subscription.id() => { - // The relay answered the REQ, so whatever made it fail before - // is over and the next failure deserves a prompt retry again. + // The relay answered the REQ: it is really serving the inbox, + // which is what the watchdog needs to know, and whatever made + // it fail before is over, so the next failure deserves a prompt + // retry again. + if let Some(health) = &self.health { + health.note_relay_acknowledged(relay_url); + } if self.backoff.remove(relay_url).is_some() { info!("Inbox subscription re-established on relay {relay_url}"); } @@ -297,7 +311,7 @@ pub enum InboxStatus { /// Process-wide inbox health. `None` until [`InboxHealth::install_global`] /// runs at startup; consumers treat an absent health record as "listening", so /// unit tests that never install it behave as before. -static INBOX_HEALTH: OnceLock = OnceLock::new(); +static INBOX_HEALTH: OnceLock> = OnceLock::new(); /// Why [`InboxHealth::install_global`] refused. Mirrors `spam_gate::InstallError`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -346,6 +360,13 @@ struct HealthState { installed_at: i64, /// Every outage this process has seen, oldest first, pruned by age. windows: Vec, + /// Relays that have answered the inbox REQ since it was last sent to them. + /// + /// A relay is only credited with serving the inbox once it says so. The + /// SDK's own subscription map cannot stand in for this: it records what + /// *we* sent, so a relay that holds the connection open and quietly + /// ignores the REQ still looks subscribed there. + acknowledged: HashSet, } impl HealthState { @@ -383,6 +404,7 @@ impl InboxHealth { verdict: None, installed_at, windows: Vec::new(), + acknowledged: HashSet::new(), }), } } @@ -390,13 +412,13 @@ impl InboxHealth { /// Install as the process-wide health record. pub fn install_global(self) -> Result<(), InstallError> { INBOX_HEALTH - .set(self) + .set(Arc::new(self)) .map_err(|_| InstallError::AlreadyInstalled) } /// The process-wide health record, if one was installed. - pub fn global() -> Option<&'static InboxHealth> { - INBOX_HEALTH.get() + pub fn global() -> Option> { + INBOX_HEALTH.get().cloned() } /// Record the current observation, returning the resulting status. @@ -499,6 +521,35 @@ impl InboxHealth { .max(0) } + /// Record that `relay` answered the inbox REQ (an `EOSE` for our + /// subscription), which is the only evidence that it is really serving it. + pub fn note_relay_acknowledged(&self, relay: &RelayUrl) { + self.state + .lock() + .expect("inbox health mutex poisoned") + .acknowledged + .insert(relay.clone()); + } + + /// Forget `relay`'s acknowledgement, because the REQ has just been sent + /// again and has yet to be answered. + pub fn note_relay_resubscribed(&self, relay: &RelayUrl) { + self.state + .lock() + .expect("inbox health mutex poisoned") + .acknowledged + .remove(relay); + } + + /// Whether `relay` has answered the inbox REQ since it was last sent. + pub fn has_acknowledged(&self, relay: &RelayUrl) -> bool { + self.state + .lock() + .expect("inbox health mutex poisoned") + .acknowledged + .contains(relay) + } + /// How long the current outage has been running, or zero if listening. pub fn blind_for_secs(&self) -> i64 { let now = now_secs(); @@ -516,25 +567,34 @@ fn now_secs() -> i64 { Timestamp::now().as_secs() as i64 } -/// Check every read relay, re-subscribing any that lost the inbox, and record -/// the verdict in the process-wide [`InboxHealth`]. +/// Check every read relay, re-subscribing any that is not serving the inbox, +/// and record the verdict in the process-wide [`InboxHealth`]. /// /// The health question is asked of the *subscription*, not of traffic: a node /// with no trades in flight is legitimately silent, so treating quiet as /// failure would raise false alarms on an idle instance and, worse, would stop /// the timeout machinery for no reason. /// -/// A relay re-subscribed during this very audit does **not** count as -/// listening. Sending a REQ is not the same as having it honoured — a relay -/// that closes the inbox on principle would accept the REQ and close it again -/// moments later, and counting the attempt would report a healthy inbox -/// forever while nothing was ever delivered. Only a subscription that was -/// already in place when the audit ran proves the relay kept it. Recovery is -/// therefore confirmed on the following round, which costs one extra interval -/// before the inbox is declared healthy again and keeps the error on the safe -/// side: the timeout clock stays frozen a little longer than strictly needed, -/// rather than resuming while the node is still deaf. +/// A relay counts as serving the inbox only when **it** has said so, by +/// answering the REQ with an `EOSE` the event loop recorded. The SDK's own +/// subscription map is not evidence: it records what Mostro sent, so a relay +/// that keeps the connection open and quietly drops the REQ still appears +/// subscribed there — and the daemon would resume timeouts while deaf. It also +/// means a relay re-subscribed during this audit does not count until it +/// answers, which costs one interval before recovery is declared and keeps the +/// error on the safe side: the timeout clock stays frozen slightly longer than +/// strictly needed rather than restarting too early. pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscription) -> InboxStatus { + check_inbox_health_with(client, subscription, InboxHealth::global()).await +} + +/// [`check_inbox_health`] against an explicit health record, so tests do not +/// have to install the process-wide one. +async fn check_inbox_health_with( + client: &Client, + subscription: &InboxSubscription, + health: Option>, +) -> InboxStatus { let relays = client .relays() .with_capabilities(RelayCapabilities::READ) @@ -547,14 +607,26 @@ pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscriptio if !relay.status().is_connected() { continue; } - if relay.subscription(subscription.id()).await.is_some() { + let registered = relay.subscription(subscription.id()).await.is_some(); + // Without a health record there is nowhere to have stored an + // acknowledgement, so fall back to registration alone. + let acknowledged = health + .as_ref() + .map(|h| h.has_acknowledged(url)) + .unwrap_or(true); + + if registered && acknowledged { listening += 1; } else { - // Connected but not subscribed: a CLOSED the event loop never saw - // (the notification channel drops frames when it lags), a REQ that - // failed to go out, or a relay re-added after startup. + // Not serving it: a CLOSED the event loop never saw (the + // notification channel drops frames when it lags), a REQ that + // failed to go out, a relay re-added after startup — or one that + // took the REQ and never answered it. warn!("Relay {url} is connected but not serving the Mostro inbox; re-subscribing"); resubscribe_relay(relay, subscription).await; + if let Some(health) = &health { + health.note_relay_resubscribed(url); + } retried += 1; } } @@ -565,7 +637,7 @@ pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscriptio InboxStatus::Blind }; - if let Some(health) = InboxHealth::global() { + if let Some(health) = &health { let was_blind = health.is_blind(); health.observe(status, now_secs()); @@ -1181,11 +1253,13 @@ mod tests { .expect("drop the subscription"); assert!(!client.subscriptions().await.contains_key(subscription.id())); + let health = Arc::new(InboxHealth::at(T0)); + // The audit re-subscribes, but does not yet claim to be listening: a // REQ that just went out proves nothing about whether the relay will // honour it. assert_eq!( - check_inbox_health(&client, &subscription).await, + check_inbox_health_with(&client, &subscription, Some(health.clone())).await, InboxStatus::Blind, "a relay re-subscribed during this audit must not count as listening yet" ); @@ -1194,16 +1268,90 @@ mod tests { "the inbox subscription must be back after the audit" ); - // The relay kept it, so the next round confirms the recovery. + // The relay answers the new REQ; in the daemon this is the event loop + // seeing the EOSE and recording it. + health.note_relay_acknowledged(&url); + assert_eq!( - check_inbox_health(&client, &subscription).await, + check_inbox_health_with(&client, &subscription, Some(health)).await, InboxStatus::Listening, - "a subscription that survived to the next audit means the ear is open" + "a relay that answered the REQ is serving the inbox" ); relay.shutdown(); } + #[tokio::test] + async fn watchdog_does_not_trust_a_relay_that_never_answered() { + use nostr_sdk::local_relay::LocalRelay; + + // A relay can hold the connection open and quietly drop the REQ. The + // SDK still lists the subscription, because that map records what + // Mostro sent, not what the relay agreed to serve. Treating it as + // proof would resume order timeouts against a deaf node. + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + subscription.subscribe(&client).await.expect("subscribe"); + tokio::time::sleep(Duration::from_millis(500)).await; + + let health = Arc::new(InboxHealth::at(T0)); + + assert!( + client.subscriptions().await.contains_key(subscription.id()), + "precondition: the SDK has the subscription registered" + ); + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health.clone())).await, + InboxStatus::Blind, + "local registration alone must not count as the relay serving the inbox" + ); + + // Once it does answer, the same registration is finally evidence. + health.note_relay_acknowledged(&url); + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health)).await, + InboxStatus::Listening + ); + + relay.shutdown(); + } + + #[tokio::test] + async fn keeper_records_the_acknowledgement_the_watchdog_reads() { + // The EOSE is only observable from the event loop, while the watchdog + // is what acts on it — this is the handoff between the two. + let client = crate::util::mostro_nostr_client_options(None).build(); + let health = Arc::new(InboxHealth::at(T0)); + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + let url = relay_url("ws://relay.example"); + + assert!(!health.has_acknowledged(&url)); + + let eose = + RelayMessage::EndOfStoredEvents(std::borrow::Cow::Owned(subscription.id().clone())); + keeper.on_relay_message(&client, &url, &eose).await; + + assert!( + health.has_acknowledged(&url), + "an EOSE for the inbox must be recorded as the relay serving it" + ); + + // A frame for someone else's subscription proves nothing about ours. + let other_url = relay_url("ws://other.example"); + let other = RelayMessage::EndOfStoredEvents(std::borrow::Cow::Owned(SubscriptionId::new( + "not-the-inbox", + ))); + keeper.on_relay_message(&client, &other_url, &other).await; + assert!(!health.has_acknowledged(&other_url)); + } + #[tokio::test] async fn watchdog_stays_blind_against_a_relay_that_keeps_closing() { use nostr_sdk::local_relay::LocalRelay; diff --git a/src/scheduler.rs b/src/scheduler.rs index 2805ad2d..20e12fb8 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -543,7 +543,7 @@ async fn job_cancel_orders(ctx: AppContext) { // under-credit an order that waited through the whole outage or // hand the same credit to one taken long after it ended. let health = crate::inbox::InboxHealth::global(); - let max_grace = health.map(|h| h.max_blind_seconds()).unwrap_or(0); + let max_grace = health.as_ref().map(|h| h.max_blind_seconds()).unwrap_or(0); if max_grace > 0 { info!( "scheduler_timeout: up to {max_grace}s of inbox downtime is credited against order deadlines" @@ -569,7 +569,7 @@ async fn job_cancel_orders(ctx: AppContext) { // Give this order back the downtime it actually waited // through. Orders taken after the outage are owed nothing // and fall through unchanged. - if let Some(health) = health { + if let Some(health) = health.as_ref() { let owed = health.blind_seconds_since(order.taken_at); let waited = nostr_sdk::prelude::Timestamp::now().as_secs() as i64 - order.taken_at; From 9dccb34c2da8ea18458ff558704e43b9cc241488 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 18:19:00 -0300 Subject: [PATCH 11/25] feat: release bonds without slashing after prolonged inbox outage to prevent indefinite escrow lock --- docs/EVENT_ROUTING.md | 2 + src/app/bond/mod.rs | 6 +- src/app/bond/slash.rs | 13 +++++ src/inbox.rs | 48 ++++++++++++++++ src/scheduler.rs | 131 +++++++++++++++++++++++++++++------------- 5 files changed, 157 insertions(+), 43 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index c3e59818..526dc753 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -35,6 +35,8 @@ Once the inbox recovers, each order is credited the downtime **it** waited throu The credit has to be per order rather than one global allowance: a single figure either under-credits an order that waited through the whole outage or hands the same credit to one taken long afterwards. +Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped, since by then every waiting order would be owed more than its deadline and nothing would ever be unwound. + ## Dispatch - Router: `src/app.rs:handle_message_action` - Maps `Action` → module function under `src/app/*`. diff --git a/src/app/bond/mod.rs b/src/app/bond/mod.rs index c8fe9423..5ae4bd47 100644 --- a/src/app/bond/mod.rs +++ b/src/app/bond/mod.rs @@ -29,8 +29,8 @@ pub use model::Bond; pub use payout::{add_bond_invoice_action, run_bond_payout_cycle}; pub use slash::{ apply_bond_resolution, extract_bond_resolution, notify_bond_slashed, - reconcile_stranded_range_maker_bonds, resolve_range_maker_bond_at_close, - resolve_range_maker_bond_at_close_or_warn, slash_or_release_on_timeout, - validate_bond_resolution, + reconcile_stranded_range_maker_bonds, release_on_timeout_without_slashing, + resolve_range_maker_bond_at_close, resolve_range_maker_bond_at_close_or_warn, + slash_or_release_on_timeout, validate_bond_resolution, }; pub use types::{BondRole, BondSlashReason, BondState}; diff --git a/src/app/bond/slash.rs b/src/app/bond/slash.rs index 42908f42..2f074073 100644 --- a/src/app/bond/slash.rs +++ b/src/app/bond/slash.rs @@ -482,6 +482,19 @@ async fn release_on_timeout(pool: &Pool, order_id: Uuid, republishes: bo } } +/// Resolve a timed-out order's bonds without holding anyone responsible. +/// +/// Used when the timeout cannot be attributed to the user: the daemon's Nostr +/// inbox has been unreachable long enough that waiting any longer would keep +/// hold invoices encumbered until CLTV expiry (see `job_cancel_orders`). The +/// order still has to be unwound, but a silence the node could not hear is not +/// evidence of abandonment, so every bond involved is released rather than +/// settled — the republish-vs-cancel distinction is honoured exactly as in +/// [`slash_or_release_on_timeout`]. +pub async fn release_on_timeout_without_slashing(pool: &Pool, order: &Order) { + release_on_timeout(pool, order.id, order_republishes_on_timeout(order)).await; +} + pub async fn slash_or_release_on_timeout( pool: &Pool, ln_client: &mut L, diff --git a/src/inbox.rs b/src/inbox.rs index d646b084..f7b413d2 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -560,6 +560,25 @@ impl InboxHealth { .map(|w| (now - w.start).max(0)) .unwrap_or(0) } + + /// How long it has been since an audit confirmed Mostro can hear. + /// + /// Zero while listening. Otherwise it counts from the start of the current + /// outage, or — if no audit has ever run — from startup, so a watchdog + /// that never reported cannot leave a caller waiting forever on a verdict + /// that is not coming. + pub fn unconfirmed_for_secs(&self) -> i64 { + let now = now_secs(); + let state = self.state.lock().expect("inbox health mutex poisoned"); + if state.verdict == Some(InboxStatus::Listening) && state.blind_now().is_none() { + return 0; + } + let since = state + .blind_now() + .map(|w| w.start) + .unwrap_or(state.installed_at); + (now - since).max(0) + } } /// Wall-clock seconds, the base an order's `taken_at` is recorded in. @@ -1211,6 +1230,35 @@ mod tests { assert!(health.state.lock().expect("lock").windows.is_empty()); } + #[test] + fn unconfirmed_time_counts_from_the_outage_or_from_startup() { + // What bounds how long the timeout job may defer. It has to answer + // even when no audit ever ran, or a watchdog that died would park the + // job on a verdict that is never coming. + let never_audited = InboxHealth::at(now_secs() - 120); + assert!( + never_audited.unconfirmed_for_secs() >= 120, + "with no verdict at all, the clock runs from startup" + ); + + let healthy = InboxHealth::at(now_secs()); + healthy.observe(InboxStatus::Listening, now_secs()); + assert_eq!( + healthy.unconfirmed_for_secs(), + 0, + "a confirmed inbox owes no waiting" + ); + + let blind = InboxHealth::at(now_secs() - 600); + blind.observe(InboxStatus::Listening, now_secs() - 600); + blind.observe(InboxStatus::Blind, now_secs() - 300); + assert!( + (300..=310).contains(&blind.unconfirmed_for_secs()), + "while blind it runs from the start of the outage, got {}", + blind.unconfirmed_for_secs() + ); + } + #[test] fn health_ignores_repeated_healthy_observations() { let health = InboxHealth::at(T0); diff --git a/src/scheduler.rs b/src/scheduler.rs index 20e12fb8..0f0f01cc 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -60,6 +60,18 @@ pub async fn start_scheduler(ctx: AppContext) { info!("Scheduler Started"); } +/// Longest the timeout job defers to an inbox that has not been confirmed +/// listening. +/// +/// Holding timeouts is the right call for an outage, but it cannot be +/// unconditional: the same pass that slashes a bond is the one that releases +/// it, and the one that cancels the seller's hold invoice. Waiting forever on +/// a permanently broken inbox would leave escrows encumbered until CLTV expiry +/// and honest takers' bonds locked indefinitely. Three hours is far longer +/// than any transient relay problem and far shorter than the CLTV horizon, so +/// an operator has time to notice while the funds never become hostage to it. +const MAX_UNCONFIRMED_INBOX_PAUSE_SECS: i64 = 3 * 3600; + /// How often the inbox watchdog audits the subscription across relays. /// /// Short enough that a lost ear is measured in seconds rather than the 60s @@ -525,13 +537,32 @@ async fn job_cancel_orders(ctx: AppContext) { // reported deafness". This job's first tick runs immediately while // the watchdog's comes later, so an unaudited record would let a // node that never obtained a working inbox act on that window. - if let Some(health) = crate::inbox::InboxHealth::global() { + // + // Waiting cannot be unconditional, though. A permanently deaf + // inbox is an operator problem, and holding every tick forever + // turns it into a second one: hold invoices stay encumbered until + // CLTV expiry and honest takers never get their bonds back, since + // the same pass that slashes is the one that releases. Past + // `MAX_UNCONFIRMED_INBOX_PAUSE_SECS` the orders are unwound + // anyway — but blamelessly, which is the part that matters. + let health = crate::inbox::InboxHealth::global(); + let mut blameless = false; + if let Some(health) = health.as_ref() { if !health.is_confirmed_listening() { + let deaf_for = health.unconfirmed_for_secs(); + if deaf_for < MAX_UNCONFIRMED_INBOX_PAUSE_SECS { + warn!( + "scheduler_timeout: inbox not confirmed listening for {deaf_for}s, holding order timeouts" + ); + tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + continue; + } warn!( - "scheduler_timeout: inbox not confirmed listening, holding order timeouts" + "scheduler_timeout: inbox unconfirmed for {deaf_for}s, past the \ + {MAX_UNCONFIRMED_INBOX_PAUSE_SECS}s bound; unwinding timed-out orders \ + WITHOUT slashing so escrows and bonds are not held until CLTV expiry" ); - tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; - continue; + blameless = true; } } @@ -542,8 +573,17 @@ async fn job_cancel_orders(ctx: AppContext) { // A single global allowance cannot do this: it would either // under-credit an order that waited through the whole outage or // hand the same credit to one taken long after it ended. - let health = crate::inbox::InboxHealth::global(); - let max_grace = health.as_ref().map(|h| h.max_blind_seconds()).unwrap_or(0); + // + // The credit is skipped once the pause bound is passed. By then + // the outage is hours deep, so every waiting order would be owed + // more than its deadline and none would ever be unwound — which is + // the state this branch exists to escape. Nobody is punished for + // it: `blameless` releases the bonds instead of settling them. + let max_grace = if blameless { + 0 + } else { + health.as_ref().map(|h| h.max_blind_seconds()).unwrap_or(0) + }; if max_grace > 0 { info!( "scheduler_timeout: up to {max_grace}s of inbox downtime is credited against order deadlines" @@ -569,7 +609,7 @@ async fn job_cancel_orders(ctx: AppContext) { // Give this order back the downtime it actually waited // through. Orders taken after the outage are owed nothing // and fall through unchanged. - if let Some(health) = health.as_ref() { + if let Some(health) = health.as_ref().filter(|_| !blameless) { let owed = health.blind_seconds_since(order.taken_at); let waited = nostr_sdk::prelude::Timestamp::now().as_secs() as i64 - order.taken_at; @@ -668,39 +708,50 @@ async fn job_cancel_orders(ctx: AppContext) { // `order` is the pre-mutation snapshot — its // waiting status and trade pubkeys are intact, // which the §3.1 buyer/seller → bond mapping needs. - match bond::slash_or_release_on_timeout( - pool, - &mut ln_client, - &order, - Settings::get_bond(), - ) - .await - { - Ok(Some(slashed)) => { - bond::notify_bond_slashed(&order, &slashed).await; - } - Ok(None) => {} - Err(e) => { - // `Err` from `slash_or_release_on_timeout` is a DB-read - // failure (e.g. `find_active_bonds_for_order` / - // `timeout_slash_confirmed` couldn't read the bond - // rows), so we don't yet know whether the slash - // applies. Falling through to cancel/republish - // would persist the order out of - // `find_order_by_seconds`'s waiting-state - // eligibility window, and the next tick would - // never re-evaluate it — losing the slash whose - // applicability we couldn't even determine. - // `continue` keeps the order eligible so the - // next tick re-runs the full path (the slash - // primitive is idempotent on a settled HTLC and - // a `PendingPayout` bond, so a retry that - // finds the work already done is a no-op). - tracing::warn!( - "scheduler_timeout: bond slash/release errored for {} ({}); skipping cancel/republish so next tick retries", - order.id, e - ); - continue; + // + // Past the pause bound (`blameless`) none of that + // applies: the node has been unable to hear for hours, + // so the timeout says nothing about the user. The order + // is still unwound — otherwise the escrow sits until + // CLTV expiry — but every bond is released rather than + // settled. + if blameless { + bond::release_on_timeout_without_slashing(pool, &order).await; + } else { + match bond::slash_or_release_on_timeout( + pool, + &mut ln_client, + &order, + Settings::get_bond(), + ) + .await + { + Ok(Some(slashed)) => { + bond::notify_bond_slashed(&order, &slashed).await; + } + Ok(None) => {} + Err(e) => { + // `Err` from `slash_or_release_on_timeout` is a DB-read + // failure (e.g. `find_active_bonds_for_order` / + // `timeout_slash_confirmed` couldn't read the bond + // rows), so we don't yet know whether the slash + // applies. Falling through to cancel/republish + // would persist the order out of + // `find_order_by_seconds`'s waiting-state + // eligibility window, and the next tick would + // never re-evaluate it — losing the slash whose + // applicability we couldn't even determine. + // `continue` keeps the order eligible so the + // next tick re-runs the full path (the slash + // primitive is idempotent on a settled HTLC and + // a `PendingPayout` bond, so a retry that + // finds the work already done is a no-op). + tracing::warn!( + "scheduler_timeout: bond slash/release errored for {} ({}); skipping cancel/republish so next tick retries", + order.id, e + ); + continue; + } } } From 379e8192f4d3d0292275ec2b8b8f43d0ffd96f4a Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 13 Aug 2026 18:47:55 -0300 Subject: [PATCH 12/25] feat: invalidate relay acknowledgement when re-subscribing to prevent stale EOSE credit --- src/app.rs | 29 ++++++++++++++++++++++++ src/inbox.rs | 62 +++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/src/app.rs b/src/app.rs index f8247c9c..ec79c984 100644 --- a/src/app.rs +++ b/src/app.rs @@ -410,6 +410,11 @@ async fn accept_event( Some((action, message, unwrapped)) } +/// How long to wait before re-attaching to the notification stream after it +/// ended without a shutdown. Long enough that a persistent failure cannot burn +/// a core, short enough that a transient one costs no meaningful deaf time. +const NOTIFICATION_STREAM_RETRY: std::time::Duration = std::time::Duration::from_secs(1); + /// Shared post-dispatch error handling (identical in both loops). A handler /// `Err` is downcast to a `MostroError` and turned into the right reply /// (`manage_errors`) or logged (`warning_msg`); `Ok` is a no-op. Factored out @@ -509,6 +514,18 @@ pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { ClientNotification::Shutdown => return Ok(()), } } + + // The stream ended without a `Shutdown` frame. That frame can be + // missed — the SDK's notification channel silently drops messages when + // the consumer falls behind — and after a shutdown `notifications()` + // hands back an empty stream, so re-taking it unconditionally spins + // this loop at full tilt. Leave when the client is done, and pace the + // retry otherwise. + if client.is_shutdown() { + return Ok(()); + } + tracing::warn!("Nostr notification stream ended without a shutdown; re-attaching"); + tokio::time::sleep(NOTIFICATION_STREAM_RETRY).await; } } @@ -568,6 +585,18 @@ pub async fn run_cashu(ctx: AppContext) -> Result<()> { ClientNotification::Shutdown => return Ok(()), } } + + // The stream ended without a `Shutdown` frame. That frame can be + // missed — the SDK's notification channel silently drops messages when + // the consumer falls behind — and after a shutdown `notifications()` + // hands back an empty stream, so re-taking it unconditionally spins + // this loop at full tilt. Leave when the client is done, and pace the + // retry otherwise. + if client.is_shutdown() { + return Ok(()); + } + tracing::warn!("Nostr notification stream ended without a shutdown; re-attaching"); + tokio::time::sleep(NOTIFICATION_STREAM_RETRY).await; } } diff --git a/src/inbox.rs b/src/inbox.rs index f7b413d2..f43baf80 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -243,7 +243,7 @@ impl InboxKeeper { } }; - resubscribe_relay(&relay, &self.subscription).await; + resubscribe_relay(&relay, &self.subscription, self.health.as_deref()).await; } /// Whether a re-subscribe to `relay` may go out at `now`, arming the next @@ -277,7 +277,21 @@ impl InboxKeeper { /// Shared by the event-loop keeper (reacting to a `CLOSED`) and the watchdog /// (finding an ear that went missing without one), so both recover a relay the /// same way. -async fn resubscribe_relay(relay: &Relay, subscription: &InboxSubscription) { +/// +/// Invalidating the relay's acknowledgement is part of the operation, not +/// something callers remember to do: from the moment a fresh REQ goes out, an +/// earlier `EOSE` says nothing about whether the relay is serving *this* one. +/// A relay that answers, then closes the subscription, then quietly ignores +/// the replacement would otherwise keep its stale credit and read as healthy. +async fn resubscribe_relay( + relay: &Relay, + subscription: &InboxSubscription, + health: Option<&InboxHealth>, +) { + if let Some(health) = health { + health.note_relay_resubscribed(relay.url()); + } + // A `CLOSED` does not always remove the subscription: rate-limited and // auth-required closures only *mark* it, and a marked subscription is // re-REQ'd no earlier than the next reconnect — which may never come on a @@ -642,10 +656,7 @@ async fn check_inbox_health_with( // failed to go out, a relay re-added after startup — or one that // took the REQ and never answered it. warn!("Relay {url} is connected but not serving the Mostro inbox; re-subscribing"); - resubscribe_relay(relay, subscription).await; - if let Some(health) = &health { - health.note_relay_resubscribed(url); - } + resubscribe_relay(relay, subscription, health.as_deref()).await; retried += 1; } } @@ -1329,6 +1340,45 @@ mod tests { relay.shutdown(); } + #[tokio::test] + async fn a_closed_frame_invalidates_the_relays_earlier_acknowledgement() { + use nostr_sdk::local_relay::LocalRelay; + + // A relay can answer the REQ, later close the subscription, and then + // ignore the replacement. Its old EOSE must not carry over: the + // watchdog would see the re-registered subscription plus stale credit + // and resume order timeouts while the node is deaf. + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + + let health = Arc::new(InboxHealth::at(T0)); + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + + // The relay answered an earlier REQ. + health.note_relay_acknowledged(&url); + assert!(health.has_acknowledged(&url)); + + // Now it closes the subscription; the keeper re-sends the REQ. + let closed = RelayMessage::Closed { + subscription_id: std::borrow::Cow::Owned(subscription.id().clone()), + message: std::borrow::Cow::Borrowed("error: go away"), + }; + keeper.on_relay_message(&client, &url, &closed).await; + + assert!( + !health.has_acknowledged(&url), + "an EOSE from before the CLOSED must not vouch for the replacement REQ" + ); + + relay.shutdown(); + } + #[tokio::test] async fn watchdog_does_not_trust_a_relay_that_never_answered() { use nostr_sdk::local_relay::LocalRelay; From 9039505c6feeb479f8c76ada4fa41e7e97c447ec Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Fri, 21 Aug 2026 15:58:11 -0300 Subject: [PATCH 13/25] feat: bind relay acknowledgement to websocket session to prevent stale EOSE credit after reconnect --- src/inbox.rs | 144 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 22 deletions(-) diff --git a/src/inbox.rs b/src/inbox.rs index f43baf80..1d89df9c 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -45,7 +45,7 @@ //! are not, and records the verdict in [`InboxHealth`] so the rest of the //! daemon can tell whether Mostro is currently able to hear anything at all. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; @@ -374,13 +374,19 @@ struct HealthState { installed_at: i64, /// Every outage this process has seen, oldest first, pruned by age. windows: Vec, - /// Relays that have answered the inbox REQ since it was last sent to them. + /// Relay to the wall-clock second its last `EOSE` for the inbox arrived. /// /// A relay is only credited with serving the inbox once it says so. The /// SDK's own subscription map cannot stand in for this: it records what /// *we* sent, so a relay that holds the connection open and quietly /// ignores the REQ still looks subscribed there. - acknowledged: HashSet, + /// + /// The timestamp is what binds the credit to a single websocket session. + /// On reconnect the SDK re-sends the REQ by itself, with no frame this + /// module can observe, so an acknowledgement earned on the previous + /// connection says nothing about the current one — see + /// [`InboxHealth::has_acknowledged_since`]. + acknowledged: HashMap, } impl HealthState { @@ -418,7 +424,7 @@ impl InboxHealth { verdict: None, installed_at, windows: Vec::new(), - acknowledged: HashSet::new(), + acknowledged: HashMap::new(), }), } } @@ -538,11 +544,15 @@ impl InboxHealth { /// Record that `relay` answered the inbox REQ (an `EOSE` for our /// subscription), which is the only evidence that it is really serving it. pub fn note_relay_acknowledged(&self, relay: &RelayUrl) { + self.note_relay_acknowledged_at(relay, now_secs()); + } + + fn note_relay_acknowledged_at(&self, relay: &RelayUrl, at: i64) { self.state .lock() .expect("inbox health mutex poisoned") .acknowledged - .insert(relay.clone()); + .insert(relay.clone(), at); } /// Forget `relay`'s acknowledgement, because the REQ has just been sent @@ -555,13 +565,24 @@ impl InboxHealth { .remove(relay); } - /// Whether `relay` has answered the inbox REQ since it was last sent. - pub fn has_acknowledged(&self, relay: &RelayUrl) -> bool { + /// Whether `relay` answered the inbox REQ *on its current connection*. + /// + /// `connected_at` is when the websocket the audit is looking at was + /// established. An acknowledgement older than that was earned on a session + /// that no longer exists: the SDK re-sends the REQ on reconnect of its own + /// accord (`should_resubscribe`), emitting nothing this module can see, so + /// a relay that comes back and then quietly ignores the replacement would + /// otherwise keep reading as healthy on the strength of its old `EOSE`. + /// + /// The comparison is inclusive so that an `EOSE` landing in the same + /// second as the connect still counts. + pub fn has_acknowledged_since(&self, relay: &RelayUrl, connected_at: i64) -> bool { self.state .lock() .expect("inbox health mutex poisoned") .acknowledged - .contains(relay) + .get(relay) + .is_some_and(|&at| at >= connected_at) } /// How long the current outage has been running, or zero if listening. @@ -609,14 +630,20 @@ fn now_secs() -> i64 { /// the timeout machinery for no reason. /// /// A relay counts as serving the inbox only when **it** has said so, by -/// answering the REQ with an `EOSE` the event loop recorded. The SDK's own -/// subscription map is not evidence: it records what Mostro sent, so a relay -/// that keeps the connection open and quietly drops the REQ still appears -/// subscribed there — and the daemon would resume timeouts while deaf. It also -/// means a relay re-subscribed during this audit does not count until it -/// answers, which costs one interval before recovery is declared and keeps the -/// error on the safe side: the timeout clock stays frozen slightly longer than -/// strictly needed rather than restarting too early. +/// answering the REQ with an `EOSE` the event loop recorded, *on the websocket +/// session it is currently on*. The SDK's own subscription map is not +/// evidence: it records what Mostro sent, so a relay that keeps the connection +/// open and quietly drops the REQ still appears subscribed there — and the +/// daemon would resume timeouts while deaf. Neither is an acknowledgement from +/// an earlier connection: the SDK re-sends the REQ by itself after a reconnect +/// and the relay may ignore that one, which is why the check is +/// [`InboxHealth::has_acknowledged_since`] against the relay's `connected_at` +/// rather than a plain membership test. +/// +/// It also means a relay re-subscribed during this audit does not count until +/// it answers, which costs one interval before recovery is declared and keeps +/// the error on the safe side: the timeout clock stays frozen slightly longer +/// than strictly needed rather than restarting too early. pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscription) -> InboxStatus { check_inbox_health_with(client, subscription, InboxHealth::global()).await } @@ -641,11 +668,12 @@ async fn check_inbox_health_with( continue; } let registered = relay.subscription(subscription.id()).await.is_some(); + let connected_at = relay.stats().connected_at().as_secs() as i64; // Without a health record there is nowhere to have stored an // acknowledgement, so fall back to registration alone. let acknowledged = health .as_ref() - .map(|h| h.has_acknowledged(url)) + .map(|h| h.has_acknowledged_since(url, connected_at)) .unwrap_or(true); if registered && acknowledged { @@ -710,6 +738,13 @@ mod tests { RelayUrl::parse(url).expect("valid relay url") } + /// Whether `health` holds any acknowledgement for `url` at all, for the + /// tests that care about the record rather than the connection it belongs + /// to. + fn acked(health: &InboxHealth, url: &RelayUrl) -> bool { + health.has_acknowledged_since(url, 0) + } + #[test] fn filter_matches_the_subscription_the_daemon_has_always_used() { let key = pubkey(); @@ -1362,7 +1397,7 @@ mod tests { // The relay answered an earlier REQ. health.note_relay_acknowledged(&url); - assert!(health.has_acknowledged(&url)); + assert!(acked(&health, &url)); // Now it closes the subscription; the keeper re-sends the REQ. let closed = RelayMessage::Closed { @@ -1372,13 +1407,78 @@ mod tests { keeper.on_relay_message(&client, &url, &closed).await; assert!( - !health.has_acknowledged(&url), + !acked(&health, &url), "an EOSE from before the CLOSED must not vouch for the replacement REQ" ); relay.shutdown(); } + #[test] + fn an_acknowledgement_does_not_survive_the_connection_it_was_earned_on() { + // A websocket drop and reconnect leaves no trace the keeper can act + // on: there is no relay-status `ClientNotification` in nostr-sdk + // 0.45.1, and the SDK silently re-sends the REQ by itself + // (`should_resubscribe`). If the relay then ignores that replacement, + // the only thing standing between a deaf node and resumed slashing is + // the acknowledgement expiring with its session. + let health = InboxHealth::at(T0); + let url = relay_url("ws://relay.example"); + + health.note_relay_acknowledged_at(&url, T0 + 100); + + assert!(health.has_acknowledged_since(&url, T0 + 50)); + assert!( + health.has_acknowledged_since(&url, T0 + 100), + "an EOSE landing in the same second as the connect must still count" + ); + assert!( + !health.has_acknowledged_since(&url, T0 + 101), + "credit earned on a previous connection must not vouch for this one" + ); + } + + #[tokio::test] + async fn watchdog_does_not_trust_an_acknowledgement_from_a_previous_connection() { + use nostr_sdk::local_relay::LocalRelay; + + // The audit-level counterpart: the subscription is registered and the + // relay has an acknowledgement on file, but it predates the current + // websocket session, which is exactly what a reconnect leaves behind. + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + subscription.subscribe(&client).await.expect("subscribe"); + tokio::time::sleep(Duration::from_millis(500)).await; + + let health = Arc::new(InboxHealth::at(T0)); + health.note_relay_acknowledged_at(&url, T0); + + assert!( + client.subscriptions().await.contains_key(subscription.id()), + "precondition: the SDK has the subscription registered" + ); + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health.clone())).await, + InboxStatus::Blind, + "a stale acknowledgement plus a live registration must not read as listening" + ); + + // An EOSE on the current connection is what settles it. + health.note_relay_acknowledged(&url); + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health)).await, + InboxStatus::Listening + ); + + relay.shutdown(); + } + #[tokio::test] async fn watchdog_does_not_trust_a_relay_that_never_answered() { use nostr_sdk::local_relay::LocalRelay; @@ -1430,14 +1530,14 @@ mod tests { let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); let url = relay_url("ws://relay.example"); - assert!(!health.has_acknowledged(&url)); + assert!(!acked(&health, &url)); let eose = RelayMessage::EndOfStoredEvents(std::borrow::Cow::Owned(subscription.id().clone())); keeper.on_relay_message(&client, &url, &eose).await; assert!( - health.has_acknowledged(&url), + acked(&health, &url), "an EOSE for the inbox must be recorded as the relay serving it" ); @@ -1447,7 +1547,7 @@ mod tests { "not-the-inbox", ))); keeper.on_relay_message(&client, &other_url, &other).await; - assert!(!health.has_acknowledged(&other_url)); + assert!(!acked(&health, &other_url)); } #[tokio::test] From 4087027792ec0dfd12cade69c50f5cc2d0a41f17 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Fri, 21 Aug 2026 16:09:21 -0300 Subject: [PATCH 14/25] feat: stand down on auth-required and rate-limited closures to let SDK handle re-subscription --- docs/EVENT_ROUTING.md | 2 + src/inbox.rs | 119 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 526dc753..1524d124 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -14,6 +14,8 @@ How Nostr events become actions and side effects. ### Recovering a lost ear A relay can end a subscription at any time by sending `CLOSED`. The nostr-sdk removes the subscription for almost every reason prefix, and a removed subscription is never re-REQ'd, not even after a reconnect — so without handling, one frame from one relay leaves the daemon running, connected, and unable to receive anything. +The exceptions are `auth-required` and `rate-limited`, which the SDK only *marks*: the subscription stays registered and the SDK re-sends the REQ itself, after the NIP-42 round-trip in the first case and on the next reconnect in the second. The keeper stands down on those two rather than racing it, and whatever the SDK does not get to — a `rate-limited` closure on a connection that never drops — falls to the watchdog below, which is the right pace for a relay that has just asked to be left alone. + Two mechanisms keep the subscription alive: - `InboxKeeper::on_relay_message` reacts to a `CLOSED` naming the inbox by re-sending the REQ to that relay, paced by a per-relay backoff (immediate first retry, doubling to a five-minute ceiling, cleared when the relay answers with `EOSE`). diff --git a/src/inbox.rs b/src/inbox.rs index 1d89df9c..25e0f362 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -37,6 +37,11 @@ //! relay that refuses the inbox on principle is retried at a decreasing rate //! instead of being hammered. //! +//! Two reason prefixes are the exception. `auth-required` and `rate-limited` +//! only *mark* the subscription, leaving it registered for the SDK to re-send +//! by itself, so the keeper stands down on those and lets it: see +//! [`is_provisional_closure`]. +//! //! Not every way of losing the ear announces itself with a frame, though: the //! notification channel silently drops messages when the consumer falls //! behind, a REQ can fail to go out, a relay can be added after startup. @@ -203,6 +208,28 @@ impl InboxKeeper { subscription_id, message, } if subscription_id.as_ref() == self.subscription.id() => { + if is_provisional_closure(message) { + // Not the keeper's to answer: the SDK only *marks* these + // two prefixes and re-sends the REQ itself — after the + // NIP-42 round-trip for `auth-required`, on the next + // reconnect for `rate-limited`. Re-issuing the REQ here + // would drop the entry the SDK is about to re-send, race + // its AUTH, and arm a backoff against a relay that is + // behaving exactly as the protocol says it should. + // + // Whatever the SDK does not get to — a `rate-limited` + // closure on a connection that never drops, an + // `auth-required` one this node cannot answer because it + // has no keys — is left to [`check_inbox_health`]: the + // relay is not acknowledged, so the next audit re-sends + // the REQ. That is the right pace for a relay that has + // just asked to be left alone. + info!( + "Relay {relay_url} closed the Mostro inbox subscription provisionally \ + (\"{message}\"); recovery is the SDK's or the watchdog's" + ); + return; + } warn!("Relay {relay_url} closed the Mostro inbox subscription: \"{message}\""); self.resubscribe(client, relay_url).await; } @@ -272,6 +299,24 @@ impl InboxKeeper { } } +/// Whether a `CLOSED` reason means "not now" rather than "not ever". +/// +/// These are the two prefixes nostr-sdk 0.45.1 maps to `MarkAsClosed` instead +/// of `Remove` (`relay/inner.rs`, the `RelayMessage::Closed` arm), keeping the +/// subscription registered so it can be re-sent without the keeper's help. +/// Every other reason — and no reason at all — removes it, which is what +/// [`InboxKeeper`] exists to undo. +/// +/// `auth-required` is only marked when an authenticator is configured; without +/// one the SDK removes it and no re-REQ follows, but a node in that state has +/// no Nostr keys at all, so the watchdog's pace is the appropriate response. +fn is_provisional_closure(message: &str) -> bool { + matches!( + MachineReadablePrefix::parse(message), + Some(MachineReadablePrefix::AuthRequired) | Some(MachineReadablePrefix::RateLimited) + ) +} + /// Re-send the inbox REQ to one relay. /// /// Shared by the event-loop keeper (reacting to a `CLOSED`) and the watchdog @@ -883,6 +928,80 @@ mod tests { ); } + #[test] + fn only_the_two_prefixes_the_sdk_recovers_from_are_provisional() { + // Mirrors the `RelayMessage::Closed` arm of nostr-sdk 0.45.1: these + // two map to `MarkAsClosed`, everything else to `Remove`. A future + // bump that changes the split has to change this list with it. + assert!(is_provisional_closure( + "auth-required: we only serve authenticated users" + )); + assert!(is_provisional_closure("rate-limited: slow down")); + + for permanent in [ + "blocked: you are banned", + "restricted: not for you", + "error: go away", + "invalid: bad filter", + "unsupported: no such filter", + "pow: 24 bits required", + "duplicate: already have it", + "", + "we are closing this one", + ] { + assert!( + !is_provisional_closure(permanent), + "{permanent:?} removes the subscription, so the keeper has to re-send the REQ" + ); + } + } + + #[tokio::test] + async fn a_provisional_closure_is_left_to_the_sdk() { + let client = crate::util::mostro_nostr_client_options(None).build(); + let health = Arc::new(InboxHealth::at(T0)); + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + let relay = relay_url("ws://relay.example"); + + health.note_relay_acknowledged(&relay); + + for reason in ["auth-required: please auth", "rate-limited: slow down"] { + let closed = RelayMessage::Closed { + subscription_id: std::borrow::Cow::Owned(subscription.id().clone()), + message: std::borrow::Cow::Borrowed(reason), + }; + keeper.on_relay_message(&client, &relay, &closed).await; + } + + assert!( + keeper.backoff.is_empty(), + "a relay the SDK will re-REQ by itself must not be put on the keeper's backoff" + ); + assert!( + acked(&health, &relay), + "the subscription is still registered and still answered, so the credit stands" + ); + } + + #[tokio::test] + async fn a_permanent_closure_is_still_the_keepers_to_answer() { + let client = crate::util::mostro_nostr_client_options(None).build(); + let mut keeper = keeper(); + let relay = relay_url("ws://relay.example"); + + let closed = RelayMessage::Closed { + subscription_id: std::borrow::Cow::Owned(keeper.subscription.id().clone()), + message: std::borrow::Cow::Borrowed("blocked: you are banned"), + }; + keeper.on_relay_message(&client, &relay, &closed).await; + + assert!( + keeper.backoff.contains_key(&relay), + "a CLOSED the SDK removes the subscription for must still arm the keeper" + ); + } + #[tokio::test] async fn frames_for_other_subscriptions_are_ignored() { let client = crate::util::mostro_nostr_client_options(None).build(); From 23aada2f582f1f0cac06103f8272103b98b13f0f Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Fri, 21 Aug 2026 16:20:43 -0300 Subject: [PATCH 15/25] feat: consolidate inbox re-subscribe pacing in InboxHealth to prevent watchdog bypass Move backoff state from InboxKeeper to InboxHealth so the event loop and watchdog share one per-relay budget. The watchdog's 30-second cadence would otherwise put a hard floor under RESUBSCRIBE_MAX_BACKOFF by re-sending unconditionally on every pass. Sharing the budget lets a relay that merely lost the inbox recover on the next audit while one that refuses it tapers to five minutes instead of drawing a REQ every 30 seconds indefinitely. --- docs/EVENT_ROUTING.md | 4 +- src/app.rs | 4 +- src/inbox.rs | 363 ++++++++++++++++++++++++++++++------------ src/scheduler.rs | 7 +- 4 files changed, 275 insertions(+), 103 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 1524d124..c7b3851a 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -18,9 +18,11 @@ The exceptions are `auth-required` and `rate-limited`, which the SDK only *marks Two mechanisms keep the subscription alive: -- `InboxKeeper::on_relay_message` reacts to a `CLOSED` naming the inbox by re-sending the REQ to that relay, paced by a per-relay backoff (immediate first retry, doubling to a five-minute ceiling, cleared when the relay answers with `EOSE`). +- `InboxKeeper::on_relay_message` reacts to a `CLOSED` naming the inbox by re-sending the REQ to that relay. - `check_inbox_health`, run every 30 seconds by `job_inbox_watchdog`, audits each connected read relay and re-subscribes any that is no longer serving the inbox. This covers the losses that produce no frame the loop can see: a notification channel that dropped messages under lag, a REQ that failed to go out, a relay added after startup. +Both go through `resubscribe_relay`, which owns the pacing so neither can bypass it: one per-relay budget in `InboxHealth`, immediate first retry, doubling to a five-minute ceiling, reset when the relay answers with `EOSE`. Because the doublings start well below the 30-second audit interval, a relay that merely lost the inbox is re-subscribed on the next pass, while one that refuses it on principle converges on the five-minute figure rather than drawing a REQ every 30 seconds indefinitely. + Health is judged by the subscription, never by traffic volume: an instance with no trades in flight is legitimately silent. A relay counts as serving the inbox only once **it** has said so, by answering the REQ with an `EOSE` that the event loop recorded in `InboxHealth`. The SDK's own subscription map is not evidence — it records what Mostro sent, so a relay that holds the connection open and quietly drops the REQ still appears subscribed there, and the daemon would resume order timeouts while deaf. The same rule means a relay re-subscribed during an audit does not count until it answers, which costs one interval before recovery is declared and errs toward keeping the timeout clock frozen slightly longer than strictly needed. diff --git a/src/app.rs b/src/app.rs index ec79c984..4e10e2ee 100644 --- a/src/app.rs +++ b/src/app.rs @@ -466,7 +466,7 @@ pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { // The inbox identity is derived here rather than passed around (see // `crate::inbox`). let subscription = InboxSubscription::new(my_keys.public_key(), accepted_kind); - let mut keeper = InboxKeeper::new(subscription.clone()); + let keeper = InboxKeeper::new(subscription.clone()); let mut subscribed = false; loop { @@ -547,7 +547,7 @@ pub async fn run_cashu(ctx: AppContext) -> Result<()> { let pow_first_contact = ctx.settings().mostro.effective_pow_first_contact(); let is_v2 = accepted_kind.as_u16() == crate::config::constants::DM_EVENT_KIND; let subscription = InboxSubscription::new(my_keys.public_key(), accepted_kind); - let mut keeper = InboxKeeper::new(subscription.clone()); + let keeper = InboxKeeper::new(subscription.clone()); let mut subscribed = false; loop { diff --git a/src/inbox.rs b/src/inbox.rs index 25e0f362..2a1b1797 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -79,6 +79,13 @@ const RESUBSCRIBE_INITIAL_BACKOFF: Duration = Duration::from_secs(2); /// and the operator has to intervene. Retrying every five minutes keeps the /// door open for a config change on their side without generating traffic that /// looks like an attack. +/// +/// This is a real ceiling because [`check_inbox_health`] draws on the same +/// per-relay budget rather than re-sending on every pass: an audit every +/// `INBOX_WATCHDOG_INTERVAL` would otherwise put a hard floor of thirty +/// seconds under it. The doublings still start well below that interval, so a +/// relay that merely lost the inbox is re-subscribed on the next pass and only +/// a persistently refusing one reaches this figure. const RESUBSCRIBE_MAX_BACKOFF: Duration = Duration::from_secs(300); /// Per-relay re-subscribe pacing. @@ -165,15 +172,15 @@ impl InboxSubscription { /// Keeps the inbox subscription alive across relay-initiated closures. /// /// Lives in the event loop, which is the only consumer of the notification -/// stream — hence the plain `&mut self` state rather than a lock. +/// stream. All of its mutable state — acknowledgements and re-subscribe +/// pacing — is in [`InboxHealth`], because [`check_inbox_health`] runs from a +/// different task and has to see and share the very same facts. pub struct InboxKeeper { subscription: InboxSubscription, - /// Only holds relays that are currently failing; a relay that accepts the - /// REQ is dropped from the map, so the steady state is empty. - backoff: HashMap, - /// Where relay acknowledgements are recorded. The event loop is the only - /// place an `EOSE` can be observed, but the watchdog is what acts on it, - /// so the fact has to be shared rather than kept here. + /// Where relay acknowledgements and re-subscribe pacing are recorded. The + /// event loop is the only place an `EOSE` can be observed, but the + /// watchdog is what acts on it, so the facts have to be shared rather than + /// kept here. health: Option>, } @@ -185,7 +192,6 @@ impl InboxKeeper { pub fn with_health(subscription: InboxSubscription, health: Option>) -> Self { Self { subscription, - backoff: HashMap::new(), health, } } @@ -198,7 +204,7 @@ impl InboxKeeper { /// backoff. Everything else (`OK`, `NOTICE`, other subscriptions' frames) /// is not this module's business. pub async fn on_relay_message( - &mut self, + &self, client: &Client, relay_url: &RelayUrl, message: &RelayMessage<'_>, @@ -241,23 +247,18 @@ impl InboxKeeper { // it fail before is over, so the next failure deserves a prompt // retry again. if let Some(health) = &self.health { - health.note_relay_acknowledged(relay_url); - } - if self.backoff.remove(relay_url).is_some() { - info!("Inbox subscription re-established on relay {relay_url}"); + if health.note_relay_acknowledged(relay_url) { + info!("Inbox subscription re-established on relay {relay_url}"); + } } } _ => {} } } - /// Re-issue the inbox REQ to a single relay, subject to backoff. - async fn resubscribe(&mut self, client: &Client, relay_url: &RelayUrl) { - if !self.allow_attempt(relay_url, Instant::now()) { - debug!("Skipping inbox re-subscribe on relay {relay_url}: backing off"); - return; - } - + /// Re-issue the inbox REQ to a single relay. Pacing is + /// [`resubscribe_relay`]'s job, so the watchdog cannot bypass it. + async fn resubscribe(&self, client: &Client, relay_url: &RelayUrl) { let relay = match client.relay(relay_url).await { Ok(Some(relay)) => relay, Ok(None) => { @@ -272,31 +273,6 @@ impl InboxKeeper { resubscribe_relay(&relay, &self.subscription, self.health.as_deref()).await; } - - /// Whether a re-subscribe to `relay` may go out at `now`, arming the next - /// delay when it may. The first failure for a relay always passes. - fn allow_attempt(&mut self, relay: &RelayUrl, now: Instant) -> bool { - match self.backoff.get_mut(relay) { - None => { - self.backoff.insert( - relay.clone(), - RelayBackoff { - next_attempt_at: now + RESUBSCRIBE_INITIAL_BACKOFF, - delay: RESUBSCRIBE_INITIAL_BACKOFF, - }, - ); - true - } - Some(state) => { - if now < state.next_attempt_at { - return false; - } - state.delay = (state.delay * 2).min(RESUBSCRIBE_MAX_BACKOFF); - state.next_attempt_at = now + state.delay; - true - } - } - } } /// Whether a `CLOSED` reason means "not now" rather than "not ever". @@ -317,23 +293,38 @@ fn is_provisional_closure(message: &str) -> bool { ) } -/// Re-send the inbox REQ to one relay. +/// Re-send the inbox REQ to one relay, returning whether one actually went out. /// /// Shared by the event-loop keeper (reacting to a `CLOSED`) and the watchdog /// (finding an ear that went missing without one), so both recover a relay the -/// same way. +/// same way — and, just as importantly, pace it the same way. Backoff and +/// acknowledgement bookkeeping are part of the operation rather than something +/// callers remember to do: /// -/// Invalidating the relay's acknowledgement is part of the operation, not -/// something callers remember to do: from the moment a fresh REQ goes out, an -/// earlier `EOSE` says nothing about whether the relay is serving *this* one. -/// A relay that answers, then closes the subscription, then quietly ignores -/// the replacement would otherwise keep its stale credit and read as healthy. +/// - **Pacing.** Both callers draw on one per-relay budget in [`InboxHealth`]. +/// The watchdog would otherwise re-send unconditionally on every pass, +/// putting a hard floor of `INBOX_WATCHDOG_INTERVAL` under a ceiling that +/// claims to be [`RESUBSCRIBE_MAX_BACKOFF`]. Sharing it keeps a transient +/// failure recovering on the very next audit while a relay that refuses the +/// inbox on principle tapers to one REQ every five minutes. +/// - **Acknowledgement.** From the moment a fresh REQ goes out, an earlier +/// `EOSE` says nothing about whether the relay is serving *this* one. A +/// relay that answers, then closes the subscription, then quietly ignores +/// the replacement would otherwise keep its stale credit and read as +/// healthy. async fn resubscribe_relay( relay: &Relay, subscription: &InboxSubscription, health: Option<&InboxHealth>, -) { +) -> bool { if let Some(health) = health { + if !health.allow_resubscribe(relay.url()) { + debug!( + "Skipping inbox re-subscribe on relay {}: backing off", + relay.url() + ); + return false; + } health.note_relay_resubscribed(relay.url()); } @@ -355,6 +346,8 @@ async fn resubscribe_relay( relay.url() ), } + + true } /// Whether the daemon can currently hear anything at all. @@ -432,6 +425,11 @@ struct HealthState { /// connection says nothing about the current one — see /// [`InboxHealth::has_acknowledged_since`]. acknowledged: HashMap, + /// Re-subscribe pacing, drawn on by the event loop and the watchdog alike. + /// + /// Only holds relays that are currently failing; a relay that answers the + /// REQ is dropped from the map, so the steady state is empty. + backoff: HashMap, } impl HealthState { @@ -470,6 +468,7 @@ impl InboxHealth { installed_at, windows: Vec::new(), acknowledged: HashMap::new(), + backoff: HashMap::new(), }), } } @@ -588,16 +587,55 @@ impl InboxHealth { /// Record that `relay` answered the inbox REQ (an `EOSE` for our /// subscription), which is the only evidence that it is really serving it. - pub fn note_relay_acknowledged(&self, relay: &RelayUrl) { - self.note_relay_acknowledged_at(relay, now_secs()); + /// + /// Whatever was making the relay fail is over, so its pacing is reset too + /// and the next failure earns a prompt retry again. Returns whether the + /// relay was being backed off, which is what distinguishes a recovery + /// worth logging from the steady state. + pub fn note_relay_acknowledged(&self, relay: &RelayUrl) -> bool { + self.note_relay_acknowledged_at(relay, now_secs()) } - fn note_relay_acknowledged_at(&self, relay: &RelayUrl, at: i64) { - self.state - .lock() - .expect("inbox health mutex poisoned") - .acknowledged - .insert(relay.clone(), at); + fn note_relay_acknowledged_at(&self, relay: &RelayUrl, at: i64) -> bool { + let mut state = self.state.lock().expect("inbox health mutex poisoned"); + state.acknowledged.insert(relay.clone(), at); + state.backoff.remove(relay).is_some() + } + + /// Whether a re-subscribe to `relay` may go out now, arming the next delay + /// when it may. The first failure for a relay always passes. + /// + /// This is the single pacing budget the event loop and the watchdog share. + /// Under the watchdog's 30-second cadence the doubling only starts to bite + /// once the delay outgrows the interval — so a relay that lost the inbox + /// once is re-subscribed on the very next pass, and only one that keeps + /// refusing tapers to [`RESUBSCRIBE_MAX_BACKOFF`]. + pub fn allow_resubscribe(&self, relay: &RelayUrl) -> bool { + self.allow_resubscribe_at(relay, Instant::now()) + } + + fn allow_resubscribe_at(&self, relay: &RelayUrl, now: Instant) -> bool { + let mut state = self.state.lock().expect("inbox health mutex poisoned"); + match state.backoff.get_mut(relay) { + None => { + state.backoff.insert( + relay.clone(), + RelayBackoff { + next_attempt_at: now + RESUBSCRIBE_INITIAL_BACKOFF, + delay: RESUBSCRIBE_INITIAL_BACKOFF, + }, + ); + true + } + Some(pacing) => { + if now < pacing.next_attempt_at { + return false; + } + pacing.delay = (pacing.delay * 2).min(RESUBSCRIBE_MAX_BACKOFF); + pacing.next_attempt_at = now + pacing.delay; + true + } + } } /// Forget `relay`'s acknowledgement, because the REQ has just been sent @@ -729,8 +767,9 @@ async fn check_inbox_health_with( // failed to go out, a relay re-added after startup — or one that // took the REQ and never answered it. warn!("Relay {url} is connected but not serving the Mostro inbox; re-subscribing"); - resubscribe_relay(relay, subscription, health.as_deref()).await; - retried += 1; + if resubscribe_relay(relay, subscription, health.as_deref()).await { + retried += 1; + } } } @@ -775,8 +814,25 @@ mod tests { Keys::generate().public_key() } - fn keeper() -> InboxKeeper { - InboxKeeper::new(InboxSubscription::new(pubkey(), Kind::GiftWrap)) + /// A keeper backed by its own health record: all of its state — pacing and + /// acknowledgements alike — now lives there, so tests need the handle too. + fn keeper() -> (InboxKeeper, Arc) { + let health = Arc::new(InboxHealth::at(T0)); + let keeper = InboxKeeper::with_health( + InboxSubscription::new(pubkey(), Kind::GiftWrap), + Some(health.clone()), + ); + (keeper, health) + } + + /// Whether `health` is currently pacing re-subscribes to `url`. + fn backing_off(health: &InboxHealth, url: &RelayUrl) -> bool { + health + .state + .lock() + .expect("inbox health mutex poisoned") + .backoff + .contains_key(url) } fn relay_url(url: &str) -> RelayUrl { @@ -847,46 +903,53 @@ mod tests { #[test] fn first_closure_from_a_relay_retries_immediately() { - let mut keeper = keeper(); + let health = InboxHealth::at(T0); let relay = relay_url("ws://relay.example"); assert!( - keeper.allow_attempt(&relay, Instant::now()), + health.allow_resubscribe_at(&relay, Instant::now()), "a first CLOSED must be answered at once: every delay is deaf time" ); } #[test] fn repeat_closures_are_paced_and_back_off() { - let mut keeper = keeper(); + let health = InboxHealth::at(T0); let relay = relay_url("ws://relay.example"); let start = Instant::now(); - assert!(keeper.allow_attempt(&relay, start)); + assert!(health.allow_resubscribe_at(&relay, start)); // A relay that closes again right away must not pull a second REQ. - assert!(!keeper.allow_attempt(&relay, start)); - assert!(!keeper.allow_attempt(&relay, start + Duration::from_secs(1))); + assert!(!health.allow_resubscribe_at(&relay, start)); + assert!(!health.allow_resubscribe_at(&relay, start + Duration::from_secs(1))); // Past the first delay it retries, and the next wait is longer. - assert!(keeper.allow_attempt(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF)); - assert!(!keeper.allow_attempt(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 2)); - assert!(keeper.allow_attempt(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 3)); + assert!(health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF)); + assert!(!health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 2)); + assert!(health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 3)); } #[test] fn backoff_is_capped() { - let mut keeper = keeper(); + let health = InboxHealth::at(T0); let relay = relay_url("ws://relay.example"); let mut now = Instant::now(); // Drive it well past the ceiling. for _ in 0..20 { - assert!(keeper.allow_attempt(&relay, now)); + assert!(health.allow_resubscribe_at(&relay, now)); now += RESUBSCRIBE_MAX_BACKOFF * 2; } assert_eq!( - keeper.backoff.get(&relay).expect("state kept").delay, + health + .state + .lock() + .unwrap() + .backoff + .get(&relay) + .expect("state kept") + .delay, RESUBSCRIBE_MAX_BACKOFF, "a hostile relay must still be retried every {RESUBSCRIBE_MAX_BACKOFF:?}" ); @@ -894,15 +957,87 @@ mod tests { #[test] fn backoff_is_per_relay() { - let mut keeper = keeper(); + let health = InboxHealth::at(T0); let hostile = relay_url("ws://hostile.example"); let healthy = relay_url("ws://healthy.example"); let now = Instant::now(); - assert!(keeper.allow_attempt(&hostile, now)); - assert!(!keeper.allow_attempt(&hostile, now)); + assert!(health.allow_resubscribe_at(&hostile, now)); + assert!(!health.allow_resubscribe_at(&hostile, now)); // One misbehaving relay must not delay recovery on another. - assert!(keeper.allow_attempt(&healthy, now)); + assert!(health.allow_resubscribe_at(&healthy, now)); + } + + #[test] + fn an_acknowledgement_clears_the_pacing_for_the_next_failure() { + let health = InboxHealth::at(T0); + let relay = relay_url("ws://relay.example"); + let now = Instant::now(); + + assert!(health.allow_resubscribe_at(&relay, now)); + assert!(!health.allow_resubscribe_at(&relay, now)); + + assert!( + health.note_relay_acknowledged(&relay), + "clearing a live backoff entry is what marks a recovery" + ); + assert!( + health.allow_resubscribe_at(&relay, now), + "a relay that answered starts over: the next failure is a fresh one" + ); + + // The steady state has nothing to clear, so nothing to report either. + health.note_relay_acknowledged(&relay); + assert!(!health.note_relay_acknowledged(&relay)); + } + + #[test] + fn the_watchdog_cadence_recovers_promptly_and_only_then_tapers() { + // The point of sharing one budget: an audit every + // `INBOX_WATCHDOG_INTERVAL` must still re-subscribe a relay that + // simply lost the inbox, while a relay that refuses it converges on + // the advertised ceiling instead of drawing a REQ every 30 seconds + // forever. + let health = InboxHealth::at(T0); + let relay = relay_url("ws://hostile.example"); + let tick = Duration::from_secs(crate::scheduler::INBOX_WATCHDOG_INTERVAL); + let mut now = Instant::now(); + + assert!( + health.allow_resubscribe_at(&relay, now), + "the pass that first notices the loss must act on it" + ); + for pass in 1..=3 { + now += tick; + assert!( + health.allow_resubscribe_at(&relay, now), + "pass {pass}: a delay still under the audit interval must not skip a retry" + ); + } + + // Once the doubling outgrows the interval, passes start being skipped. + let mut attempts = 0; + for _ in 0..40 { + now += tick; + if health.allow_resubscribe_at(&relay, now) { + attempts += 1; + } + } + assert!( + attempts < 40, + "a relay that keeps refusing must stop drawing a REQ on every pass" + ); + assert_eq!( + health + .state + .lock() + .unwrap() + .backoff + .get(&relay) + .expect("state kept") + .delay, + RESUBSCRIBE_MAX_BACKOFF + ); } // ───────────────────────── control-plane handling ───────────────────────── @@ -910,12 +1045,11 @@ mod tests { #[tokio::test] async fn eose_for_the_inbox_clears_the_backoff() { let client = crate::util::mostro_nostr_client_options(None).build(); - let mut keeper = keeper(); + let (keeper, health) = keeper(); let relay = relay_url("ws://relay.example"); - let now = Instant::now(); - assert!(keeper.allow_attempt(&relay, now)); - assert!(keeper.backoff.contains_key(&relay)); + assert!(health.allow_resubscribe(&relay)); + assert!(backing_off(&health, &relay)); let eose = RelayMessage::EndOfStoredEvents(std::borrow::Cow::Owned( keeper.subscription.id().clone(), @@ -923,7 +1057,7 @@ mod tests { keeper.on_relay_message(&client, &relay, &eose).await; assert!( - !keeper.backoff.contains_key(&relay), + !backing_off(&health, &relay), "an accepted REQ must reset the pacing for the next failure" ); } @@ -961,7 +1095,7 @@ mod tests { let client = crate::util::mostro_nostr_client_options(None).build(); let health = Arc::new(InboxHealth::at(T0)); let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); - let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + let keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); let relay = relay_url("ws://relay.example"); health.note_relay_acknowledged(&relay); @@ -975,8 +1109,8 @@ mod tests { } assert!( - keeper.backoff.is_empty(), - "a relay the SDK will re-REQ by itself must not be put on the keeper's backoff" + !backing_off(&health, &relay), + "a relay the SDK will re-REQ by itself must not be put on the shared backoff" ); assert!( acked(&health, &relay), @@ -986,26 +1120,40 @@ mod tests { #[tokio::test] async fn a_permanent_closure_is_still_the_keepers_to_answer() { + use nostr_sdk::local_relay::LocalRelay; + + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let (keeper, health) = keeper(); let client = crate::util::mostro_nostr_client_options(None).build(); - let mut keeper = keeper(); - let relay = relay_url("ws://relay.example"); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + health.note_relay_acknowledged(&url); let closed = RelayMessage::Closed { subscription_id: std::borrow::Cow::Owned(keeper.subscription.id().clone()), message: std::borrow::Cow::Borrowed("blocked: you are banned"), }; - keeper.on_relay_message(&client, &relay, &closed).await; + keeper.on_relay_message(&client, &url, &closed).await; assert!( - keeper.backoff.contains_key(&relay), + backing_off(&health, &url), "a CLOSED the SDK removes the subscription for must still arm the keeper" ); + assert!( + !acked(&health, &url), + "the replacement REQ has yet to be answered, so the old EOSE cannot vouch for it" + ); + + relay.shutdown(); } #[tokio::test] async fn frames_for_other_subscriptions_are_ignored() { let client = crate::util::mostro_nostr_client_options(None).build(); - let mut keeper = keeper(); + let (keeper, health) = keeper(); let relay = relay_url("ws://relay.example"); // Mostro's price provider and NIP-33 queries share these relays; their @@ -1017,7 +1165,7 @@ mod tests { keeper.on_relay_message(&client, &relay, &other).await; assert!( - keeper.backoff.is_empty(), + !backing_off(&health, &relay), "a CLOSED for another subscription must not be treated as an inbox failure" ); } @@ -1093,7 +1241,7 @@ mod tests { publisher.add_relay(url.clone()).await.expect("add_relay"); publisher.connect().await; - let mut keeper = InboxKeeper::new(subscription.clone()); + let keeper = InboxKeeper::new(subscription.clone()); let wanted = wrap_for(mostro.public_key()); let wanted_id = wanted.id; let mut published = false; @@ -1512,7 +1660,7 @@ mod tests { let health = Arc::new(InboxHealth::at(T0)); let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); - let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + let keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); // The relay answered an earlier REQ. health.note_relay_acknowledged(&url); @@ -1646,7 +1794,7 @@ mod tests { let client = crate::util::mostro_nostr_client_options(None).build(); let health = Arc::new(InboxHealth::at(T0)); let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); - let mut keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + let keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); let url = relay_url("ws://relay.example"); assert!(!acked(&health, &url)); @@ -1708,18 +1856,35 @@ mod tests { client.connect().await; subscription.subscribe(&client).await.expect("subscribe"); + let health = Arc::new(InboxHealth::at(T0)); + // However many rounds it runs, a relay that keeps closing the inbox // never makes the node look healthy — this is what keeps the timeout // machinery paused while trade messages are being lost. for round in 0..3 { tokio::time::sleep(Duration::from_millis(500)).await; assert_eq!( - check_inbox_health(&client, &subscription).await, + check_inbox_health_with(&client, &subscription, Some(health.clone())).await, InboxStatus::Blind, "round {round}: a relay that refuses every REQ must never read as listening" ); } + // The audit draws on the same budget the event loop does, so a relay + // that refuses on principle is paced towards `RESUBSCRIBE_MAX_BACKOFF` + // instead of being handed a REQ on every pass, forever. Skipping the + // retry must not soften the verdict: the node is still deaf here. + assert!( + backing_off(&health, &url), + "repeated refusals must accumulate on the shared re-subscribe budget" + ); + assert!(!health.allow_resubscribe(&url)); + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health)).await, + InboxStatus::Blind, + "a pass that backs off instead of re-sending must still report the inbox deaf" + ); + relay.shutdown(); } diff --git a/src/scheduler.rs b/src/scheduler.rs index 0f0f01cc..8ce7bd28 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -77,7 +77,12 @@ const MAX_UNCONFIRMED_INBOX_PAUSE_SECS: i64 = 3 * 3600; /// Short enough that a lost ear is measured in seconds rather than the 60s /// timeout tick, long enough that it is not a source of traffic on its own. /// Hardcoded like the other maintenance intervals in this module. -const INBOX_WATCHDOG_INTERVAL: u64 = 30; +/// +/// Visible to `crate::inbox` because the two pacing knobs interact: the audit +/// draws on the same per-relay budget as the event loop, so this interval is +/// what decides how many doublings pass before `RESUBSCRIBE_MAX_BACKOFF` is +/// the thing actually limiting retries. +pub(crate) const INBOX_WATCHDOG_INTERVAL: u64 = 30; /// Audit the daemon's Nostr inbox and re-subscribe any relay that stopped /// serving it (see `crate::inbox`). From 71bb1b48a31aeb7b34c9a2a42739af6f60d39f30 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Fri, 21 Aug 2026 16:23:46 -0300 Subject: [PATCH 16/25] docs: correct inbox subscription origin from main to event loop in documentation Update references across docs and comments to reflect that the inbox subscription is sent by `app::run` / `app::run_cashu` after taking the notification stream, not by `main` at startup. This clarifies the timing dependency where the receiver must exist before the REQ to catch the relay's EOSE. --- docs/EVENT_ROUTING.md | 2 +- docs/TRANSPORT_V2_SPEC.md | 2 +- src/config/mod.rs | 2 +- src/scheduler.rs | 4 ++-- src/util.rs | 11 ++++++----- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index c7b3851a..282bd1a6 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -8,7 +8,7 @@ How Nostr events become actions and side effects. ## The Inbox Subscription - Source: `src/inbox.rs` -- Every trade message reaches Mostro over a single long-lived subscription, created at startup by `main` with a stable id (`InboxSubscription`) so that later frames can be attributed to it. Its filter is p-tagged to the node, restricted to the configured transport's event kind, and carries `limit(0)`: only live traffic is wanted. +- Every trade message reaches Mostro over a single long-lived subscription, sent by the event loop (`app::run` / `app::run_cashu`) right after it takes its notification stream — a receiver created after the REQ would miss the relay's `EOSE` — with a stable id (`InboxSubscription`) so that later frames can be attributed to it. Its filter is p-tagged to the node, restricted to the configured transport's event kind, and carries `limit(0)`: only live traffic is wanted. - `run` consumes the whole notification stream, not just events. `ClientNotification::Message` carries the relay control plane and goes to `InboxKeeper`; `ClientNotification::Shutdown` ends the loop. ### Recovering a lost ear diff --git a/docs/TRANSPORT_V2_SPEC.md b/docs/TRANSPORT_V2_SPEC.md index cc476864..e6c8c675 100644 --- a/docs/TRANSPORT_V2_SPEC.md +++ b/docs/TRANSPORT_V2_SPEC.md @@ -196,7 +196,7 @@ Minimal daemon integration; **zero handler changes** by design: - `[expiration] dm_days` knob (default 30) in `ExpirationSettings` and the `get_expiration_timestamp_for_kind` fallback (`DM_EVENT_KIND = 14` in `src/config/constants.rs`). -- `src/main.rs` — subscription filter uses `transport.event_kind()`. +- `src/inbox.rs` — subscription filter uses `transport.event_kind()`. - `src/app.rs` — event loop accepts only the configured kind and unwraps via `unwrap_incoming()`. - `src/util.rs send_dm()` — wraps via `wrap_message_with(transport, …)`; diff --git a/src/config/mod.rs b/src/config/mod.rs index d141f552..24a71ee9 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -37,7 +37,7 @@ pub static NOSTR_CLIENT: OnceLock = OnceLock::new(); /// /// Kept separate from [`NOSTR_CLIENT`] so `verify_subscriptions(true)` (and /// the REQ `limit` it enforces) applies only to price fetches — not to the -/// daemon's long-lived `.limit(0)` inbox subscription in `main.rs`, where +/// daemon's long-lived `.limit(0)` inbox subscription (`crate::inbox`), where /// pre-EOSE filter verification would reject matching trade messages /// (hermeme, PR #841). pub static PRICE_NOSTR_CLIENT: OnceLock = OnceLock::new(); diff --git a/src/scheduler.rs b/src/scheduler.rs index 8ce7bd28..490f4371 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -99,8 +99,8 @@ async fn job_inbox_watchdog(ctx: AppContext) { tokio::spawn(async move { loop { - // Sleep first: at startup `main` has just subscribed, and a REQ - // still in flight would look exactly like a missing one. + // Sleep first: at startup the event loop has just subscribed, and a + // REQ still in flight would look exactly like a missing one. tokio::time::sleep(tokio::time::Duration::from_secs(INBOX_WATCHDOG_INTERVAL)).await; crate::inbox::check_inbox_health(ctx.nostr_client(), &subscription).await; } diff --git a/src/util.rs b/src/util.rs index 1150d65c..1d2704a5 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1379,9 +1379,10 @@ pub async fn connect_nostr() -> Result { let nostr_settings = Settings::get_nostr(); // Daemon inbox client: shared size limits, but **no** - // `verify_subscriptions`. The long-lived `.limit(0)` subscription in - // `main.rs` must not count pre-EOSE frames against a zero limit — that - // would drop matching trade messages before dispatch (hermeme, PR #841). + // `verify_subscriptions`. The long-lived `.limit(0)` inbox subscription + // (`crate::inbox`) must not count pre-EOSE frames against a zero limit — + // that would drop matching trade messages before dispatch (hermeme, PR + // #841). // Price queries use [`connect_price_nostr`] / [`PRICE_NOSTR_CLIENT`] with // verification enabled instead. let client = mostro_nostr_client_options(nip42_identity()).build(); @@ -3354,7 +3355,7 @@ mod tests { #[test] fn nostr_client_policies_scope_verify_to_price_only() { - // Daemon inbox must not enable verify_subscriptions (main.rs limit(0)). + // Daemon inbox must not enable verify_subscriptions (inbox limit(0)). assert!( !daemon_nostr_client_policy().verify_subscriptions, "daemon client must leave verify_subscriptions off" @@ -3483,7 +3484,7 @@ mod tests { use nostr_sdk::local_relay::MockRelay; use std::time::Duration; - // Clean mock (no random flood): mirrors main.rs `.limit(0)` inbox — + // Clean mock (no random flood): mirrors the `.limit(0)` inbox — // history is skipped, but live matching events after EOSE must arrive. // Daemon options intentionally omit verify_subscriptions so pre-EOSE // frames are not counted against limit 0 (hermeme, PR #841). From 6a09b08b8975df2312b3a010d47949a7b4d7ec14 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 27 Aug 2026 16:09:03 -0300 Subject: [PATCH 17/25] fix(inbox): drop the relay's credit on a provisional CLOSED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `auth-required` and `rate-limited` are the two prefixes nostr-sdk maps to `MarkAsClosed` rather than `Remove`, so the keeper stands down and lets the SDK re-send the REQ itself. It stood down on the health verdict too, and that was wrong: `MarkAsClosed` only sets `closed = true` and leaves the entry in the subscription map, so `relay.subscription(id)` still returns `Some`. With the relay's earlier `EOSE` also intact, the audit counted it as listening permanently. The SDK does not always get to the re-send. `Relay::resubscribe` has two callers in 0.45.1 — `post_connection` and the ingester's post-AUTH success path — and neither fires for a `rate-limited` closure on a connection that never drops (there is no retry timer, only the next reconnect) or for an `auth-required` whose AUTH the relay then rejects. In both cases the node was deaf on that relay for the life of the connection while the watchdog reported a healthy inbox and `job_cancel_orders` kept slashing bonds over messages it could not receive. Drop the acknowledgement so the audit judges the relay on the replacement REQ instead of the `EOSE` that preceded the closure. The happy path is unchanged: the SDK re-sends, the relay answers, and the credit is back one audit interval later at worst. `note_relay_resubscribed` is now `note_relay_unacknowledged`, which fits both callers. --- docs/EVENT_ROUTING.md | 4 +- src/inbox.rs | 117 +++++++++++++++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 20 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 282bd1a6..9fde862b 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -14,7 +14,9 @@ How Nostr events become actions and side effects. ### Recovering a lost ear A relay can end a subscription at any time by sending `CLOSED`. The nostr-sdk removes the subscription for almost every reason prefix, and a removed subscription is never re-REQ'd, not even after a reconnect — so without handling, one frame from one relay leaves the daemon running, connected, and unable to receive anything. -The exceptions are `auth-required` and `rate-limited`, which the SDK only *marks*: the subscription stays registered and the SDK re-sends the REQ itself, after the NIP-42 round-trip in the first case and on the next reconnect in the second. The keeper stands down on those two rather than racing it, and whatever the SDK does not get to — a `rate-limited` closure on a connection that never drops — falls to the watchdog below, which is the right pace for a relay that has just asked to be left alone. +The exceptions are `auth-required` and `rate-limited`, which the SDK only *marks*: the subscription stays registered and the SDK re-sends the REQ itself, after the NIP-42 round-trip in the first case and on the next reconnect in the second. The keeper stands down on the REQ for those two rather than racing it. + +It does not stand down on the health verdict. `MarkAsClosed` leaves the entry in the SDK's subscription map, so the relay keeps *looking* subscribed, and the SDK does not always get around to re-sending: `rate-limited` has no retry timer at all (only the next reconnect, which never comes on a healthy connection), and an `auth-required` whose AUTH the relay then rejects ends without a re-subscribe. The relay's acknowledgement is therefore dropped either way, which hands the case to the watchdog below — the right pace for a relay that has just asked to be left alone, and the difference between recovering and being silently deaf for the life of the connection. Two mechanisms keep the subscription alive: diff --git a/src/inbox.rs b/src/inbox.rs index 2a1b1797..c419dabd 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -39,8 +39,11 @@ //! //! Two reason prefixes are the exception. `auth-required` and `rate-limited` //! only *mark* the subscription, leaving it registered for the SDK to re-send -//! by itself, so the keeper stands down on those and lets it: see -//! [`is_provisional_closure`]. +//! by itself, so the keeper stands down on the REQ and lets it: see +//! [`is_provisional_closure`]. It does not stand down on the *verdict* — the +//! relay's acknowledgement is dropped either way, because a subscription the +//! SDK left registered is not one a relay is answering, and the SDK does not +//! always get around to re-sending it. //! //! Not every way of losing the ear announces itself with a frame, though: the //! notification channel silently drops messages when the consumer falls @@ -215,21 +218,33 @@ impl InboxKeeper { message, } if subscription_id.as_ref() == self.subscription.id() => { if is_provisional_closure(message) { - // Not the keeper's to answer: the SDK only *marks* these - // two prefixes and re-sends the REQ itself — after the - // NIP-42 round-trip for `auth-required`, on the next - // reconnect for `rate-limited`. Re-issuing the REQ here + // The REQ is not the keeper's to re-send: the SDK only + // *marks* these two prefixes and re-sends it itself — + // after the NIP-42 round-trip for `auth-required`, on the + // next reconnect for `rate-limited`. Re-issuing it here // would drop the entry the SDK is about to re-send, race // its AUTH, and arm a backoff against a relay that is // behaving exactly as the protocol says it should. // - // Whatever the SDK does not get to — a `rate-limited` - // closure on a connection that never drops, an - // `auth-required` one this node cannot answer because it - // has no keys — is left to [`check_inbox_health`]: the - // relay is not acknowledged, so the next audit re-sends - // the REQ. That is the right pace for a relay that has - // just asked to be left alone. + // The *health verdict* is another matter, and must not + // stand down with it. `MarkAsClosed` leaves the entry in + // the SDK's subscription map, so the relay still reads as + // registered; with its earlier `EOSE` also intact the + // audit would count it as serving the inbox forever — + // including in the two cases the SDK never gets to: a + // `rate-limited` closure on a connection that never drops + // (`Relay::resubscribe` only runs on reconnect, and there + // is no retry timer), and an `auth-required` one whose + // AUTH the relay then rejects (the ingester reports + // `AuthenticationFailed` and returns without re-sending). + // Dropping the credit costs nothing on the happy path — + // the replacement REQ is answered and the credit comes + // back one audit later at worst — and turns both dead ends + // into a re-subscribe under the shared backoff instead of + // silent deafness. + if let Some(health) = &self.health { + health.note_relay_unacknowledged(relay_url); + } info!( "Relay {relay_url} closed the Mostro inbox subscription provisionally \ (\"{message}\"); recovery is the SDK's or the watchdog's" @@ -325,7 +340,7 @@ async fn resubscribe_relay( ); return false; } - health.note_relay_resubscribed(relay.url()); + health.note_relay_unacknowledged(relay.url()); } // A `CLOSED` does not always remove the subscription: rate-limited and @@ -638,9 +653,16 @@ impl InboxHealth { } } - /// Forget `relay`'s acknowledgement, because the REQ has just been sent - /// again and has yet to be answered. - pub fn note_relay_resubscribed(&self, relay: &RelayUrl) { + /// Forget `relay`'s acknowledgement: whatever it said about serving the + /// inbox no longer applies. + /// + /// Two callers, one meaning. A fresh REQ has gone out and has yet to be + /// answered ([`resubscribe_relay`]); or the relay closed the subscription + /// provisionally, so the entry the SDK left registered is not evidence of + /// anything until the replacement REQ is answered ([`InboxKeeper::on_relay_message`]). + /// In both cases the audit has to judge the relay on the new evidence + /// rather than on the old `EOSE`. + pub fn note_relay_unacknowledged(&self, relay: &RelayUrl) { self.state .lock() .expect("inbox health mutex poisoned") @@ -1113,11 +1135,68 @@ mod tests { "a relay the SDK will re-REQ by itself must not be put on the shared backoff" ); assert!( - acked(&health, &relay), - "the subscription is still registered and still answered, so the credit stands" + !acked(&health, &relay), + "the relay stopped answering the REQ it acknowledged, so the credit cannot stand" ); } + /// The SDK does not always get around to re-sending a provisionally closed + /// subscription: `rate-limited` has no retry timer at all (only the next + /// reconnect, which never comes on a healthy connection), and an + /// `auth-required` whose AUTH the relay then rejects ends the ingester's + /// post-auth path without a `resubscribe()`. `MarkAsClosed` leaves the + /// entry registered throughout, so registration alone would report a relay + /// that stopped serving the inbox as healthy for the life of the + /// connection — the exact silent deafness this module exists to prevent. + #[tokio::test] + async fn a_provisional_closure_the_sdk_never_answers_is_caught_by_the_audit() { + use nostr_sdk::local_relay::LocalRelay; + + let relay = LocalRelay::builder().build(); + relay.run().await.expect("run local relay"); + let url = relay.url().await; + + let subscription = InboxSubscription::new(pubkey(), Kind::GiftWrap); + let client = crate::util::mostro_nostr_client_options(None).build(); + client.add_relay(url.clone()).await.expect("add_relay"); + client.connect().await; + subscription.subscribe(&client).await.expect("subscribe"); + tokio::time::sleep(Duration::from_millis(500)).await; + + let health = Arc::new(InboxHealth::at(T0)); + let keeper = InboxKeeper::with_health(subscription.clone(), Some(health.clone())); + health.note_relay_acknowledged(&url); + + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health.clone())).await, + InboxStatus::Listening, + "precondition: an answered REQ on a live connection is a healthy inbox" + ); + + let closed = RelayMessage::Closed { + subscription_id: std::borrow::Cow::Owned(subscription.id().clone()), + message: std::borrow::Cow::Borrowed("rate-limited: slow down"), + }; + keeper.on_relay_message(&client, &url, &closed).await; + + let sdk_relay = client + .relay(&url) + .await + .expect("relay lookup") + .expect("relay in pool"); + assert!( + sdk_relay.subscription(subscription.id()).await.is_some(), + "precondition: the entry the SDK leaves registered is what used to vouch for the relay" + ); + assert_eq!( + check_inbox_health_with(&client, &subscription, Some(health)).await, + InboxStatus::Blind, + "a relay that closed the inbox and has not answered since is not serving it" + ); + + relay.shutdown(); + } + #[tokio::test] async fn a_permanent_closure_is_still_the_keepers_to_answer() { use nostr_sdk::local_relay::LocalRelay; From 4c6200741cadf6cf1f84c65c3dbea6a1bfcdbc08 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 27 Aug 2026 16:09:16 -0300 Subject: [PATCH 18/25] fix(scheduler): select timeout candidates on the nominal deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `find_order_by_seconds` took a `grace` parameter and subtracted it from the cut-off. The predicate is `taken_at < now - exp_seconds - grace`, so grace moved the deadline further into the past and *narrowed* the selection: an order was returned only once it had waited longer than `exp_seconds + max_blind_seconds`. Every surviving row therefore already satisfied the scheduler's per-order check, and the `continue` that credits an order for the downtime it actually waited through could never fire. What shipped was the single global allowance the design rejects — with `max_blind_seconds` summing every outage inside the seven-day retention window, a node with flapping relays postponed every order's deadline by hours of unrelated downtime. Drop the parameter and let the query select on the nominal deadline alone, deliberately over-selecting, with the scheduler's loop as the only place a credit is applied. It can only ever spare, so nothing eligible is lost. Tests: `find_order_by_seconds_selects_on_the_nominal_deadline_alone` pins the cut-off, and `an_order_taken_after_an_outage_is_not_credited_for_it` pins the per-order table — an order that waited through a 300s outage expires 300s late, one taken after recovery expires at its nominal deadline. --- docs/EVENT_ROUTING.md | 2 +- src/db.rs | 90 +++++++++++++++++++++---------------------- src/inbox.rs | 46 +++++++++++++++++++++- src/scheduler.rs | 33 ++++++++-------- 4 files changed, 106 insertions(+), 65 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 9fde862b..04b12c04 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -37,7 +37,7 @@ The daemon and price clients are built with a `SignerAuthenticator` over the nod Because those messages are lost rather than delayed, order timeouts cannot be trusted while the inbox is down — a user who answered on time would look silent. `job_cancel_orders` therefore skips its tick entirely unless an audit has confirmed the daemon is listening (`InboxHealth::is_confirmed_listening`): no slash, no refund, no republish. Startup counts as unconfirmed, since the daemon subscribes before the watchdog's first pass. -Once the inbox recovers, each order is credited the downtime **it** waited through. `InboxHealth` keeps the wall-clock windows during which the node was deaf; `blind_seconds_since(taken_at)` intersects them with the order's own wait. An order already waiting when a relay went quiet is owed all of that outage; one taken after it ended is owed nothing. The query widens its window by `max_blind_seconds` so no eligible order is missed, and the exact per-order figure decides. +Once the inbox recovers, each order is credited the downtime **it** waited through. `InboxHealth` keeps the wall-clock windows during which the node was deaf; `blind_seconds_since(taken_at)` intersects them with the order's own wait. An order already waiting when a relay went quiet is owed all of that outage; one taken after it ended is owed nothing. `find_order_by_seconds` selects on the nominal deadline alone — the credit is applied only in the scheduler's loop, order by order, where it can only ever spare. The credit has to be per order rather than one global allowance: a single figure either under-credits an order that waited through the whole outage or hands the same credit to one taken long afterwards. diff --git a/src/db.rs b/src/db.rs index e9eef59c..b170936d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -10,7 +10,6 @@ use std::collections::HashSet; use std::fs::{set_permissions, Permissions}; use std::path::Path; use std::sync::Arc; -use std::time::Duration; use uuid::Uuid; // Constants for status filtering used across restore session functions @@ -537,20 +536,19 @@ pub async fn find_order_by_date(pool: &SqlitePool) -> Result, MostroE /// Orders whose waiting deadline has passed and are therefore candidates for /// the timeout job. /// -/// `grace` widens the window by the **most** any order could be owed for time -/// the daemon spent unable to receive anything (see -/// [`crate::inbox::InboxHealth::max_blind_seconds`]), so that no order the -/// caller may still have to spare is filtered out here. It deliberately -/// over-selects: what a given order is actually owed depends on when it began -/// waiting, which this query cannot express, so the caller applies the exact -/// per-order figure to the rows returned. -pub async fn find_order_by_seconds( - pool: &SqlitePool, - grace: Duration, -) -> Result, MostroError> { +/// The nominal deadline is the only thing this query knows about. Compensation +/// for time the daemon spent unable to receive anything is **not** applied +/// here: what an order is owed depends on which outages overlap its own wait, +/// which this predicate cannot express. So the selection deliberately +/// over-selects — every order past its wall-clock deadline — and the caller +/// spares the ones it must, order by order (see `scheduler::job_cancel_orders` +/// and [`crate::inbox::InboxHealth::blind_seconds_since`]). Narrowing the +/// window here would put those rows out of reach and make the per-order credit +/// unreachable. +pub async fn find_order_by_seconds(pool: &SqlitePool) -> Result, MostroError> { let mostro_settings = Settings::get_mostro(); let exp_seconds = mostro_settings.expiration_seconds as u64; - let expire_time = Timestamp::now() - exp_seconds - grace.as_secs(); + let expire_time = Timestamp::now() - exp_seconds; let order = sqlx::query_as::<_, Order>( r#" SELECT * @@ -5042,7 +5040,7 @@ mod migration_and_query_tests { ) .await; - let stale = find_order_by_seconds(&pool, Duration::ZERO).await.unwrap(); + let stale = find_order_by_seconds(&pool).await.unwrap(); assert_eq!(stale.len(), 1); assert_eq!(stale[0].id, stale_id); } @@ -5086,61 +5084,63 @@ mod migration_and_query_tests { let after = Order::by_id(&pool, id).await.unwrap().unwrap(); assert!(after.taken_at > 0, "take must persist taken_at"); - let stale = find_order_by_seconds(&pool, Duration::ZERO).await.unwrap(); + let stale = find_order_by_seconds(&pool).await.unwrap(); assert!( stale.is_empty(), "a just-taken order must not be timeout-cancel eligible" ); } + /// Regression: the query must select on the nominal deadline alone. + /// + /// It used to subtract the largest outage the node had seen from the + /// cut-off, which narrows the selection rather than widening it — every + /// surviving row was then already past `deadline + max_blind_seconds`, so + /// the scheduler's per-order credit could never spare anything and the + /// global allowance the design rejects was what actually shipped. An order + /// one second past its deadline has to reach the caller for the per-order + /// figure to have anything to decide about. #[tokio::test] - async fn find_order_by_seconds_grace_spares_orders_the_daemon_could_not_hear() { + async fn find_order_by_seconds_selects_on_the_nominal_deadline_alone() { init_test_settings(); let pool = migrated_pool().await; let exp_seconds = Settings::get_mostro().expiration_seconds as i64; - // Taken one minute past the deadline: late by wall time, and the - // caller is about to say the node was deaf for longer than that. - let taken_at = Timestamp::now().as_secs() as i64 - exp_seconds - 60; - let order_id = Uuid::new_v4(); + let now = Timestamp::now().as_secs() as i64; + + // Barely late: one second past the wall-clock deadline. + let late_id = Uuid::new_v4(); insert_order( &pool, - order_id, + late_id, "sell", "waiting-buyer-invoice", Some(HEX_KEY_A), Some(HEX_KEY_B), HEX_KEY_B, - taken_at, + now - exp_seconds - 1, + ) + .await; + // Barely not late: one second short of it. + insert_order( + &pool, + Uuid::new_v4(), + "buy", + "waiting-payment", + Some(HEX_KEY_A), + Some(HEX_KEY_B), + HEX_KEY_A, + now - exp_seconds + 1, ) .await; - // No grace: the order is treated as late, which is what would cancel - // the escrow and slash the bond. - let without_grace = find_order_by_seconds(&pool, Duration::ZERO).await.unwrap(); - assert_eq!(without_grace.len(), 1); - assert_eq!(without_grace[0].id, order_id); - - // Owed more downtime than the order is late by: not late at all. - let with_grace = find_order_by_seconds(&pool, Duration::from_secs(300)) - .await - .unwrap(); - assert!( - with_grace.is_empty(), - "an order cannot be late for a window the daemon spent unable to listen" - ); - - // A grace smaller than the overshoot still selects it: the query only - // has to avoid filtering out rows the caller may spare, and the caller - // decides from each order's own downtime. - let smaller_grace = find_order_by_seconds(&pool, Duration::from_secs(30)) - .await - .unwrap(); + let stale = find_order_by_seconds(&pool).await.unwrap(); assert_eq!( - smaller_grace.len(), + stale.len(), 1, - "grace only postpones the deadline, it does not remove it" + "the cut-off is the nominal deadline, neither widened nor narrowed" ); + assert_eq!(stale[0].id, late_id); } #[tokio::test] diff --git a/src/inbox.rs b/src/inbox.rs index c419dabd..c6794064 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -587,8 +587,12 @@ impl InboxHealth { state.windows.iter().map(|w| w.overlap(from, to, to)).sum() } - /// Upper bound on what any order could be owed, for callers that need to - /// widen a query before applying the exact per-order figure. + /// Upper bound on what any order could be owed. + /// + /// No decision is taken on this figure — an order's credit is always the + /// downtime that overlaps its own wait ([`Self::blind_seconds_since`]). + /// It exists so the timeout job can tell an operator, in one line, how + /// much downtime is in play this tick. pub fn max_blind_seconds(&self) -> i64 { let now = now_secs(); let state = self.state.lock().expect("inbox health mutex poisoned"); @@ -1575,6 +1579,44 @@ mod tests { ); } + /// Regression: the credit is per order, so an order taken *after* an + /// outage ended must expire at its nominal deadline. + /// + /// The timeout job used to widen `find_order_by_seconds` by + /// [`InboxHealth::max_blind_seconds`], which narrows the selection rather + /// than widening it — every surviving row was already past + /// `deadline + max_blind_seconds`, the per-order check could never spare + /// anything, and what shipped was the global allowance this design + /// rejects. That allowance grows with every outage in the retention + /// window, so a node with flapping relays would postpone every deadline by + /// hours of unrelated downtime. + #[test] + fn an_order_taken_after_an_outage_is_not_credited_for_it() { + let health = InboxHealth::at(T0); + // One outage: [T0, T0+300]. + health.observe(InboxStatus::Blind, T0); + health.observe(InboxStatus::Listening, T0 + 300); + + let exp_seconds = 900i64; + let late_at = |taken_at: i64, now: i64| { + let owed = health.blind_seconds_between(taken_at, now); + (now - taken_at) >= exp_seconds + owed + }; + + // A: waited through the whole outage, owed all 300s. + assert!(!late_at(T0, T0 + 1_199)); + assert!(late_at(T0, T0 + 1_200)); + + // B: taken after recovery, owed nothing — even though the node's total + // downtime is the same 300s the global allowance would have handed it. + assert_eq!(health.max_blind_seconds(), 300); + assert!(!late_at(T0 + 400, T0 + 1_299)); + assert!( + late_at(T0 + 400, T0 + 1_300), + "an order that never lost a second must expire at its nominal deadline" + ); + } + #[test] fn consecutive_outages_accumulate_their_debt() { let health = InboxHealth::at(T0); diff --git a/src/scheduler.rs b/src/scheduler.rs index 1eadf4d1..ba8a16b4 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -19,7 +19,6 @@ use nostr_sdk::prelude::EventBuilder; use nostr_sdk::prelude::{FinalizeEvent, Kind as NostrKind, Nip65Tag, Tag}; use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; use tokio::sync::RwLock; use tracing::{debug, error, info, warn}; use util::{enqueue_order_msg, get_nostr_relays, send_dm, update_order_event}; @@ -576,34 +575,34 @@ async fn job_cancel_orders(ctx: AppContext) { } } - // Compensation for inbox downtime is per order, and applied in two - // steps. The query widens its window by the most any order could - // be owed, so nothing eligible is missed; the exact figure — the - // downtime that overlaps *this* order's own wait — then decides. - // A single global allowance cannot do this: it would either - // under-credit an order that waited through the whole outage or - // hand the same credit to one taken long after it ended. + // Compensation for inbox downtime is per order, and this loop is + // the only place it is applied. `find_order_by_seconds` selects on + // the nominal deadline alone — deliberately over-selecting — and + // the exact figure, the downtime that overlaps *this* order's own + // wait, decides below. A single global allowance cannot do this: + // it would either under-credit an order that waited through the + // whole outage or hand the same credit to one taken long after it + // ended. Widening the query by the largest outage seen would do + // the latter, and narrowing it would put the rows this credit is + // meant to spare out of reach entirely. // // The credit is skipped once the pause bound is passed. By then // the outage is hours deep, so every waiting order would be owed // more than its deadline and none would ever be unwound — which is // the state this branch exists to escape. Nobody is punished for // it: `blameless` releases the bonds instead of settling them. - let max_grace = if blameless { - 0 - } else { - health.as_ref().map(|h| h.max_blind_seconds()).unwrap_or(0) - }; + let max_grace = health + .as_ref() + .filter(|_| !blameless) + .map(|h| h.max_blind_seconds()) + .unwrap_or(0); if max_grace > 0 { info!( "scheduler_timeout: up to {max_grace}s of inbox downtime is credited against order deadlines" ); } - if let Ok(older_orders_list) = - crate::db::find_order_by_seconds(pool, Duration::from_secs(max_grace.max(0) as u64)) - .await - { + if let Ok(older_orders_list) = crate::db::find_order_by_seconds(pool).await { for order in older_orders_list.into_iter() { // The tick-start snapshot may be stale by the time this // iteration is reached — re-read and re-confirm before From 0d258414cc73830c39fcbddaf2c535ce294c7fcd Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 27 Aug 2026 16:09:26 -0300 Subject: [PATCH 19/25] fix(bond): propagate a failed blameless release on timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `release_on_timeout_without_slashing` returned `()`. It funnelled into `release_bonds_for_order_or_warn` / `release_taker_bonds_for_order_or_warn`, whose documented contract is to log and swallow, so a DB failure was invisible to the scheduler: execution fell through to cancel/republish and the order left its waiting state. That takes it out of `find_order_by_seconds`'s eligibility window with the bond still `Locked` and no later tick that would ever look at it again. This is the failure the sibling `Err` arm four lines down exists to avoid — `slash_or_release_on_timeout` returns a `Result` specifically so the call site can `continue`. The blameless path had the same exposure with none of the protection, and it fires on every waiting order in one pass, hours into an outage, which is exactly when a DB under stress is most likely to fail. Return `Result` and let the caller decide. Adds `release_taker_bonds_for_order` alongside the existing `_or_warn` wrapper, which now delegates to it; the wrappers stay for the call sites that genuinely cannot retry. --- docs/EVENT_ROUTING.md | 2 +- src/app/bond/flow.rs | 20 +++++-- src/app/bond/slash.rs | 127 +++++++++++++++++++++++++++++++++++++++++- src/scheduler.rs | 16 +++++- 4 files changed, 154 insertions(+), 11 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 04b12c04..6dfe4bfe 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -41,7 +41,7 @@ Once the inbox recovers, each order is credited the downtime **it** waited throu The credit has to be per order rather than one global allowance: a single figure either under-credits an order that waited through the whole outage or hands the same credit to one taken long afterwards. -Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped, since by then every waiting order would be owed more than its deadline and nothing would ever be unwound. +Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped, since by then every waiting order would be owed more than its deadline and nothing would ever be unwound. A failed release keeps the order in its waiting state so the next tick retries, exactly as the slashing path does: cancelling first would take the order out of `find_order_by_seconds`'s eligibility window with the bond still `Locked` and nothing left to look at it again. ## Dispatch - Router: `src/app.rs:handle_message_action` diff --git a/src/app/bond/flow.rs b/src/app/bond/flow.rs index 53b70497..2a26fd39 100644 --- a/src/app/bond/flow.rs +++ b/src/app/bond/flow.rs @@ -642,17 +642,25 @@ pub async fn release_bonds_for_order_or_warn( } } -/// Like [`release_bonds_for_order_or_warn`] but **retains the maker's -/// bond** — the waiting-timeout republish path (see [`release_active_bonds`]). -/// The maker's `Locked` bond stays put because the order returns to the -/// book with the maker still committed; only the abandoning taker side is -/// released. +/// Like [`release_bonds_for_order`] but **retains the maker's bond** — the +/// waiting-timeout republish path (see [`release_active_bonds`]). The maker's +/// `Locked` bond stays put because the order returns to the book with the +/// maker still committed; only the abandoning taker side is released. +pub async fn release_taker_bonds_for_order( + pool: &Pool, + order_id: Uuid, +) -> Result<(), MostroError> { + release_active_bonds(pool, order_id, true).await +} + +/// Best-effort [`release_taker_bonds_for_order`], for the call sites that +/// cannot act on the failure anyway (see [`release_bonds_for_order_or_warn`]). pub async fn release_taker_bonds_for_order_or_warn( pool: &Pool, order_id: Uuid, context: &'static str, ) { - if let Err(e) = release_active_bonds(pool, order_id, true).await { + if let Err(e) = release_taker_bonds_for_order(pool, order_id).await { warn!("{context}: bond release failed for {}: {}", order_id, e); } } diff --git a/src/app/bond/slash.rs b/src/app/bond/slash.rs index 2f074073..42444d44 100644 --- a/src/app/bond/slash.rs +++ b/src/app/bond/slash.rs @@ -60,7 +60,8 @@ use super::db::{ find_range_root_order, }; use super::flow::{ - release_bond, release_bonds_for_order_or_warn, release_taker_bonds_for_order_or_warn, + release_bond, release_bonds_for_order, release_bonds_for_order_or_warn, + release_taker_bonds_for_order, release_taker_bonds_for_order_or_warn, }; use super::math::compute_node_share; use super::model::Bond; @@ -491,8 +492,23 @@ async fn release_on_timeout(pool: &Pool, order_id: Uuid, republishes: bo /// evidence of abandonment, so every bond involved is released rather than /// settled — the republish-vs-cancel distinction is honoured exactly as in /// [`slash_or_release_on_timeout`]. -pub async fn release_on_timeout_without_slashing(pool: &Pool, order: &Order) { - release_on_timeout(pool, order.id, order_republishes_on_timeout(order)).await; +/// +/// Unlike [`release_on_timeout`] this **propagates** a failure, for the same +/// reason [`slash_or_release_on_timeout`] returns a `Result`: the caller is +/// about to persist the order out of `find_order_by_seconds`'s waiting-state +/// eligibility window, so a swallowed error would leave the bond `Locked` +/// with no later tick to look at it again. It also fires on *every* waiting +/// order in one pass, hours into an outage — precisely when a DB under stress +/// is most likely to fail. +pub async fn release_on_timeout_without_slashing( + pool: &Pool, + order: &Order, +) -> Result<(), MostroError> { + if order_republishes_on_timeout(order) { + release_taker_bonds_for_order(pool, order.id).await + } else { + release_bonds_for_order(pool, order.id).await + } } pub async fn slash_or_release_on_timeout( @@ -2279,6 +2295,111 @@ mod tests { ); } + // ── blameless unwind (inbox outage past the pause bound) ──────────────── + + #[tokio::test] + async fn blameless_timeout_release_frees_the_bonds_and_reports_success() { + // Buy order in WaitingBuyerInvoice: the maker is responsible, so the + // order dies rather than returning to the book and every bond is + // released. The caller needs the `Ok` before it may cancel/republish. + let pool = setup_pool().await; + let order = waiting_order( + Kind::Buy, + maker_pk(), + taker_pk(), + Status::WaitingBuyerInvoice, + ); + insert_order_row(&pool, &order).await; + let bond = insert_bond(&pool, order.id, taker_pk(), BondState::Locked).await; + + release_on_timeout_without_slashing(&pool, &order) + .await + .expect("release succeeds against a healthy DB"); + + assert_eq!( + read_bond_state(&pool, bond.id).await, + BondState::Released.to_string() + ); + } + + #[tokio::test] + async fn blameless_timeout_release_retains_the_maker_bond_on_a_republish() { + // Sell order in WaitingBuyerInvoice: the taker is responsible, the + // order goes back to the book, and the maker is still committed to it. + // Not slashing anyone does not change that distinction. + let pool = setup_pool().await; + let order = waiting_order( + Kind::Sell, + maker_pk(), + taker_pk(), + Status::WaitingBuyerInvoice, + ); + insert_order_row(&pool, &order).await; + let maker_bond = insert_bond_with_role( + &pool, + order.id, + maker_pk(), + BondRole::Maker, + BondState::Locked, + ) + .await; + let taker_bond = insert_bond_with_role( + &pool, + order.id, + taker_pk(), + BondRole::Taker, + BondState::Locked, + ) + .await; + + release_on_timeout_without_slashing(&pool, &order) + .await + .expect("release succeeds against a healthy DB"); + + assert_eq!( + read_bond_state(&pool, taker_bond.id).await, + BondState::Released.to_string() + ); + assert_eq!( + read_bond_state(&pool, maker_bond.id).await, + BondState::Locked.to_string(), + "the maker's commitment follows the order back to the book" + ); + } + + #[tokio::test] + async fn blameless_timeout_release_propagates_a_db_failure() { + // Regression: this used to funnel into the `_or_warn` helpers, whose + // documented contract is to swallow the error. The scheduler then fell + // through to cancel/republish, persisting the order out of + // `find_order_by_seconds`'s waiting-state eligibility window with the + // bond still `Locked` and no tick left that would ever look at it + // again. The failure has to reach the caller so it can stay eligible. + let pool = setup_pool().await; + let order = waiting_order( + Kind::Buy, + maker_pk(), + taker_pk(), + Status::WaitingBuyerInvoice, + ); + insert_order_row(&pool, &order).await; + sqlx::query("PRAGMA foreign_keys = OFF") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DROP TABLE bonds") + .execute(&pool) + .await + .unwrap(); + + assert!( + release_on_timeout_without_slashing(&pool, &order) + .await + .is_err(), + "a bond lookup failure must reach the caller, not just a log line" + ); + } + #[tokio::test] async fn timeout_slash_sell_buyer_silent_slashes_taker_bond() { // sell order, WaitingBuyerInvoice: the buyer is responsible and on diff --git a/src/scheduler.rs b/src/scheduler.rs index ba8a16b4..fc980dc4 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -725,7 +725,21 @@ async fn job_cancel_orders(ctx: AppContext) { // CLTV expiry — but every bond is released rather than // settled. if blameless { - bond::release_on_timeout_without_slashing(pool, &order).await; + // Same exposure as the `Err` arm below, and the + // same answer: the cancel/republish that follows + // persists the order out of the waiting-state + // eligibility window, so a dropped release would + // leave the bond `Locked` with no tick that will + // ever look at it again. Stay eligible and retry. + if let Err(e) = + bond::release_on_timeout_without_slashing(pool, &order).await + { + tracing::warn!( + "scheduler_timeout: blameless bond release failed for {} ({}); skipping cancel/republish so next tick retries", + order.id, e + ); + continue; + } } else { match bond::slash_or_release_on_timeout( pool, From e5c9b2b8573eaa4c4700509583dc7d2126e949f1 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 27 Aug 2026 16:09:37 -0300 Subject: [PATCH 20/25] fix(scheduler): supervise the inbox watchdog and recover a poisoned lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two coupled robustness gaps around the health record. The watchdog was a fire-and-forget `tokio::spawn` with no `JoinHandle` and nothing that would notice if the task died. A panic inside `check_inbox_health` froze the verdict at whatever it last was, and frozen at `Listening` is the dangerous direction: `is_confirmed_listening()` stays true forever and `job_cancel_orders` resumes slashing against an inbox nobody audits any more, with no log line since the panic is in a detached task. Each audit now runs in a task of its own, so a panic surfaces as a `JoinError` the loop logs and moves past. Every accessor on `InboxHealth` did `.lock().expect("inbox health mutex poisoned")`. `HealthState` is plain data with no invariant a panic could leave half-applied and no user code runs under the guard, so poisoning carries nothing worth acting on — but propagating it does: `is_confirmed_listening()` is read by the timeout job on every tick, so one panic there would kill that task for the life of the process, leaving hold invoices to ride to CLTV expiry and bonds unresolved. A private `state()` helper recovers with `unwrap_or_else(|e| e.into_inner())` and replaces the twelve copies of the `expect`. --- src/inbox.rs | 62 ++++++++++++++++++++++-------------------------- src/scheduler.rs | 20 +++++++++++++++- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/inbox.rs b/src/inbox.rs index c6794064..b9574c3e 100644 --- a/src/inbox.rs +++ b/src/inbox.rs @@ -54,7 +54,7 @@ //! daemon can tell whether Mostro is currently able to hear anything at all. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; use std::time::{Duration, Instant}; use nostr_sdk::prelude::*; @@ -476,6 +476,21 @@ impl InboxHealth { Self::at(now_secs()) } + /// The shared state, recovering rather than propagating a poisoned lock. + /// + /// [`HealthState`] is plain data with no invariant a panic could leave + /// half-applied, and no user code runs under the guard — so poisoning + /// carries no information worth acting on. Propagating it would, though: + /// every consumer of this record is a maintenance job, and + /// [`Self::is_confirmed_listening`] is read by the timeout job on every + /// tick. A panic there kills that task for the life of the process, which + /// stops timeouts permanently — hold invoices ride to CLTV expiry and + /// bonds never resolve. Taking the data as it stands is strictly better + /// than that. + fn state(&self) -> MutexGuard<'_, HealthState> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + fn at(installed_at: i64) -> Self { Self { state: Mutex::new(HealthState { @@ -502,7 +517,7 @@ impl InboxHealth { /// Record the current observation, returning the resulting status. fn observe(&self, status: InboxStatus, now: i64) -> InboxStatus { - let mut state = self.state.lock().expect("inbox health mutex poisoned"); + let mut state = self.state(); let first_verdict = state.verdict.is_none(); state.verdict = Some(status); @@ -544,11 +559,7 @@ impl InboxHealth { /// known to be listening, which is the question a caller about to act on a /// user's silence should be asking. See [`Self::is_confirmed_listening`]. pub fn is_blind(&self) -> bool { - self.state - .lock() - .expect("inbox health mutex poisoned") - .blind_now() - .is_some() + self.state().blind_now().is_some() } /// Whether an audit has actually confirmed that Mostro can hear. @@ -560,11 +571,7 @@ impl InboxHealth { /// working inbox would otherwise look healthy and start cancelling orders /// and slashing bonds over messages it was never in a position to receive. pub fn is_confirmed_listening(&self) -> bool { - self.state - .lock() - .expect("inbox health mutex poisoned") - .verdict - == Some(InboxStatus::Listening) + self.state().verdict == Some(InboxStatus::Listening) } /// Seconds the inbox was deaf between `from` and now. @@ -583,7 +590,7 @@ impl InboxHealth { } fn blind_seconds_between(&self, from: i64, to: i64) -> i64 { - let state = self.state.lock().expect("inbox health mutex poisoned"); + let state = self.state(); state.windows.iter().map(|w| w.overlap(from, to, to)).sum() } @@ -595,7 +602,7 @@ impl InboxHealth { /// much downtime is in play this tick. pub fn max_blind_seconds(&self) -> i64 { let now = now_secs(); - let state = self.state.lock().expect("inbox health mutex poisoned"); + let state = self.state(); state .windows .iter() @@ -616,7 +623,7 @@ impl InboxHealth { } fn note_relay_acknowledged_at(&self, relay: &RelayUrl, at: i64) -> bool { - let mut state = self.state.lock().expect("inbox health mutex poisoned"); + let mut state = self.state(); state.acknowledged.insert(relay.clone(), at); state.backoff.remove(relay).is_some() } @@ -634,7 +641,7 @@ impl InboxHealth { } fn allow_resubscribe_at(&self, relay: &RelayUrl, now: Instant) -> bool { - let mut state = self.state.lock().expect("inbox health mutex poisoned"); + let mut state = self.state(); match state.backoff.get_mut(relay) { None => { state.backoff.insert( @@ -667,11 +674,7 @@ impl InboxHealth { /// In both cases the audit has to judge the relay on the new evidence /// rather than on the old `EOSE`. pub fn note_relay_unacknowledged(&self, relay: &RelayUrl) { - self.state - .lock() - .expect("inbox health mutex poisoned") - .acknowledged - .remove(relay); + self.state().acknowledged.remove(relay); } /// Whether `relay` answered the inbox REQ *on its current connection*. @@ -686,9 +689,7 @@ impl InboxHealth { /// The comparison is inclusive so that an `EOSE` landing in the same /// second as the connect still counts. pub fn has_acknowledged_since(&self, relay: &RelayUrl, connected_at: i64) -> bool { - self.state - .lock() - .expect("inbox health mutex poisoned") + self.state() .acknowledged .get(relay) .is_some_and(|&at| at >= connected_at) @@ -697,9 +698,7 @@ impl InboxHealth { /// How long the current outage has been running, or zero if listening. pub fn blind_for_secs(&self) -> i64 { let now = now_secs(); - self.state - .lock() - .expect("inbox health mutex poisoned") + self.state() .blind_now() .map(|w| (now - w.start).max(0)) .unwrap_or(0) @@ -713,7 +712,7 @@ impl InboxHealth { /// that is not coming. pub fn unconfirmed_for_secs(&self) -> i64 { let now = now_secs(); - let state = self.state.lock().expect("inbox health mutex poisoned"); + let state = self.state(); if state.verdict == Some(InboxStatus::Listening) && state.blind_now().is_none() { return 0; } @@ -853,12 +852,7 @@ mod tests { /// Whether `health` is currently pacing re-subscribes to `url`. fn backing_off(health: &InboxHealth, url: &RelayUrl) -> bool { - health - .state - .lock() - .expect("inbox health mutex poisoned") - .backoff - .contains_key(url) + health.state().backoff.contains_key(url) } fn relay_url(url: &str) -> RelayUrl { diff --git a/src/scheduler.rs b/src/scheduler.rs index fc980dc4..e4dc28bf 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -91,6 +91,13 @@ pub(crate) const INBOX_WATCHDOG_INTERVAL: u64 = 30; /// consumer lags, and some ways of losing a subscription produce no frame at /// all. This job is the backstop, and the only thing that notices when *every* /// relay has gone quiet. +/// +/// Every guarantee the inbox machinery makes is downstream of this loop still +/// running, so each audit runs in a task of its own. A panic inside one is +/// then a `JoinError` this loop can log and move past, instead of the silent +/// end of the watchdog: with the loop gone the verdict would freeze at +/// whatever it last was, and frozen at `Listening` means `job_cancel_orders` +/// resumes slashing bonds against an inbox nobody is auditing any more. async fn job_inbox_watchdog(ctx: AppContext) { #[allow(deprecated)] let event_kind = ctx.settings().mostro.transport.event_kind(); @@ -101,7 +108,18 @@ async fn job_inbox_watchdog(ctx: AppContext) { // Sleep first: at startup the event loop has just subscribed, and a // REQ still in flight would look exactly like a missing one. tokio::time::sleep(tokio::time::Duration::from_secs(INBOX_WATCHDOG_INTERVAL)).await; - crate::inbox::check_inbox_health(ctx.nostr_client(), &subscription).await; + + let client = ctx.nostr_client().clone(); + let subscription = subscription.clone(); + let audit = tokio::spawn(async move { + crate::inbox::check_inbox_health(&client, &subscription).await; + }); + if let Err(e) = audit.await { + error!( + "scheduler_inbox_watchdog: audit task ended abnormally ({e}); retrying in \ + {INBOX_WATCHDOG_INTERVAL}s" + ); + } } }); } From 311c2f51edaed74f6057b4f4445b4b1460d6863c Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Thu, 27 Aug 2026 16:10:40 -0300 Subject: [PATCH 21/25] refactor(inbox): move the health record into its own module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `src/inbox.rs` had reached 1934 lines (808 implementation, 1126 tests), well past the 800-line guideline the repo follows elsewhere, and it held two genuinely separate concerns. `src/inbox/health.rs` now holds `InboxHealth`, `BlindWindow` and the per-relay pacing — the half `scheduler.rs` actually depends on. `src/inbox/mod.rs` keeps the subscription, the keeper and the watchdog audit. The module is private and re-exports `InboxHealth`, `InboxStatus` and `InstallError`, so `crate::inbox::` paths are unchanged for callers. Behaviour is identical: the test helper that reached into the backoff map is now `InboxHealth::is_backing_off` under `cfg(test)`, and `at`, `observe`, `note_relay_acknowledged_at` and `now_secs` widen from private to `pub(super)` so the keeper's half can still reach them. --- docs/EVENT_ROUTING.md | 2 +- docs/TRANSPORT_V2_SPEC.md | 2 +- src/inbox/health.rs | 871 +++++++++++++++++++++++++++++++++ src/{inbox.rs => inbox/mod.rs} | 863 +------------------------------- 4 files changed, 891 insertions(+), 847 deletions(-) create mode 100644 src/inbox/health.rs rename src/{inbox.rs => inbox/mod.rs} (59%) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 6dfe4bfe..19ea21c1 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -7,7 +7,7 @@ How Nostr events become actions and side effects. - Steps: POW check → signature verify → recency guard → NIP-59 unwrap → parse `mostro_core::Message` → inner verify → `check_trade_index` → dispatch. ## The Inbox Subscription -- Source: `src/inbox.rs` +- Source: `src/inbox/mod.rs` (subscription, keeper, watchdog audit) and `src/inbox/health.rs` (the health record the scheduler reads) - Every trade message reaches Mostro over a single long-lived subscription, sent by the event loop (`app::run` / `app::run_cashu`) right after it takes its notification stream — a receiver created after the REQ would miss the relay's `EOSE` — with a stable id (`InboxSubscription`) so that later frames can be attributed to it. Its filter is p-tagged to the node, restricted to the configured transport's event kind, and carries `limit(0)`: only live traffic is wanted. - `run` consumes the whole notification stream, not just events. `ClientNotification::Message` carries the relay control plane and goes to `InboxKeeper`; `ClientNotification::Shutdown` ends the loop. diff --git a/docs/TRANSPORT_V2_SPEC.md b/docs/TRANSPORT_V2_SPEC.md index de18519f..0ae39bc8 100644 --- a/docs/TRANSPORT_V2_SPEC.md +++ b/docs/TRANSPORT_V2_SPEC.md @@ -196,7 +196,7 @@ Minimal daemon integration; **zero handler changes** by design: - `[expiration] dm_days` knob (default 30) in `ExpirationSettings` and the `get_expiration_timestamp_for_kind` fallback (`DM_EVENT_KIND = 14` in `src/config/constants.rs`). -- `src/inbox.rs` — subscription filter uses `transport.event_kind()`. +- `src/inbox/mod.rs` — subscription filter uses `transport.event_kind()`. - `src/app.rs` — event loop accepts only the configured kind and unwraps via `unwrap_incoming()`. - `src/util.rs send_dm()` — wraps via `wrap_message_with(transport, …)`; diff --git a/src/inbox/health.rs b/src/inbox/health.rs new file mode 100644 index 00000000..a87ca144 --- /dev/null +++ b/src/inbox/health.rs @@ -0,0 +1,871 @@ +//! Whether the daemon's ear is open, and when it was not. +//! +//! [`InboxHealth`] is the half of the inbox the rest of the daemon reads. The +//! subscription machinery next door ([`super`]) decides what is true — which +//! relays answered the REQ, which stopped — and records it here; the scheduler +//! asks this record whether Mostro was in a position to hear at all before it +//! acts on a user's silence. +//! +//! Two things live here for that reason. The **outage log**: every stretch +//! during which no relay was serving the inbox, kept so an order can be +//! credited for exactly the downtime that overlaps its own wait. And the +//! **per-relay pacing**: which relays have acknowledged the subscription, and +//! when another REQ may go out to one that has not — shared by the event loop +//! and the watchdog so neither can bypass the other's backoff. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; +use std::time::{Duration, Instant}; + +use nostr_sdk::prelude::*; + +/// Delay before a *second* consecutive re-subscribe to the same relay. +/// +/// The first `CLOSED` is answered immediately — the common case is a transient +/// refusal, and every second of delay is a second of deaf node. Backoff only +/// starts mattering when a relay keeps closing the inbox. +const RESUBSCRIBE_INITIAL_BACKOFF: Duration = Duration::from_secs(2); + +/// Ceiling for the per-relay re-subscribe delay. +/// +/// A relay that has refused the inbox for five minutes straight is not having +/// a hiccup — it is configured to refuse us (NIP-42, a pubkey allowlist, a ban) +/// and the operator has to intervene. Retrying every five minutes keeps the +/// door open for a config change on their side without generating traffic that +/// looks like an attack. +/// +/// This is a real ceiling because [`super::check_inbox_health`] draws on the +/// same per-relay budget rather than re-sending on every pass: an audit every +/// `INBOX_WATCHDOG_INTERVAL` would otherwise put a hard floor of thirty +/// seconds under it. The doublings still start well below that interval, so a +/// relay that merely lost the inbox is re-subscribed on the next pass and only +/// a persistently refusing one reaches this figure. +const RESUBSCRIBE_MAX_BACKOFF: Duration = Duration::from_secs(300); + +/// Per-relay re-subscribe pacing. +#[derive(Debug)] +struct RelayBackoff { + /// Earliest instant at which another REQ may go out to this relay. + next_attempt_at: Instant, + /// Delay applied after the next attempt; doubles up to the ceiling. + delay: Duration, +} + +/// Whether the daemon can currently hear anything at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InboxStatus { + /// At least one connected relay is serving the inbox subscription. + Listening, + /// No connected relay is serving it: every message sent to Mostro right + /// now is being lost. + Blind, +} + +/// Process-wide inbox health. `None` until [`InboxHealth::install_global`] +/// runs at startup; consumers treat an absent health record as "listening", so +/// unit tests that never install it behave as before. +static INBOX_HEALTH: OnceLock> = OnceLock::new(); + +/// Why [`InboxHealth::install_global`] refused. Mirrors `spam_gate::InstallError`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InstallError { + /// A health record is already installed. + AlreadyInstalled, +} + +/// One stretch during which the daemon could not hear. +/// +/// Timestamps are wall-clock seconds, the same base as an order's `taken_at`, +/// because that is what these windows are ultimately intersected against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BlindWindow { + start: i64, + /// `None` while the outage is still running. + end: Option, +} + +impl BlindWindow { + /// Seconds of this window that fall inside `[from, to]`. + fn overlap(&self, from: i64, to: i64, now: i64) -> i64 { + let end = self.end.unwrap_or(now); + (end.min(to) - self.start.max(from)).max(0) + } +} + +/// Windows that ended longer ago than this are dropped: no order the timeout +/// job can still be looking at was waiting back then, so they can no longer +/// change any verdict. Generous next to `expiration_seconds` (900s by default) +/// so the bound is never the reason a user loses compensation. +const BLIND_WINDOW_RETENTION_SECS: i64 = 7 * 24 * 3600; + +/// Hard cap on retained windows, so a relay flapping in a tight loop cannot +/// grow this unboundedly between prunes. +const MAX_BLIND_WINDOWS: usize = 512; + +#[derive(Debug)] +struct HealthState { + /// The last verdict an audit reached. `None` until the first one runs: + /// startup is not evidence that the inbox works, and must not be read as + /// such (see [`InboxHealth::is_confirmed_listening`]). + verdict: Option, + /// When the record was installed, which is the earliest moment an outage + /// discovered by the first audit could have begun. + installed_at: i64, + /// Every outage this process has seen, oldest first, pruned by age. + windows: Vec, + /// Relay to the wall-clock second its last `EOSE` for the inbox arrived. + /// + /// A relay is only credited with serving the inbox once it says so. The + /// SDK's own subscription map cannot stand in for this: it records what + /// *we* sent, so a relay that holds the connection open and quietly + /// ignores the REQ still looks subscribed there. + /// + /// The timestamp is what binds the credit to a single websocket session. + /// On reconnect the SDK re-sends the REQ by itself, with no frame this + /// module can observe, so an acknowledgement earned on the previous + /// connection says nothing about the current one — see + /// [`InboxHealth::has_acknowledged_since`]. + acknowledged: HashMap, + /// Re-subscribe pacing, drawn on by the event loop and the watchdog alike. + /// + /// Only holds relays that are currently failing; a relay that answers the + /// REQ is dropped from the map, so the steady state is empty. + backoff: HashMap, +} + +impl HealthState { + fn blind_now(&self) -> Option<&BlindWindow> { + self.windows.last().filter(|w| w.end.is_none()) + } +} + +/// Tracks whether the daemon's ear is open, and when it was not. +/// +/// The scheduler's timeout machinery reads this: an order is only "late" if +/// Mostro was in a position to hear from the user, and the ten-second replay +/// window means a message sent into a dead inbox is gone rather than delayed +/// (see the module docs). Punishing a user for a silence the node itself +/// caused would be unfair, so the timeout clock stops while the inbox is down. +#[derive(Debug)] +pub struct InboxHealth { + state: Mutex, +} + +impl Default for InboxHealth { + fn default() -> Self { + Self::new() + } +} + +impl InboxHealth { + pub fn new() -> Self { + Self::at(now_secs()) + } + + /// The shared state, recovering rather than propagating a poisoned lock. + /// + /// [`HealthState`] is plain data with no invariant a panic could leave + /// half-applied, and no user code runs under the guard — so poisoning + /// carries no information worth acting on. Propagating it would, though: + /// every consumer of this record is a maintenance job, and + /// [`Self::is_confirmed_listening`] is read by the timeout job on every + /// tick. A panic there kills that task for the life of the process, which + /// stops timeouts permanently — hold invoices ride to CLTV expiry and + /// bonds never resolve. Taking the data as it stands is strictly better + /// than that. + fn state(&self) -> MutexGuard<'_, HealthState> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + pub(super) fn at(installed_at: i64) -> Self { + Self { + state: Mutex::new(HealthState { + verdict: None, + installed_at, + windows: Vec::new(), + acknowledged: HashMap::new(), + backoff: HashMap::new(), + }), + } + } + + /// Install as the process-wide health record. + pub fn install_global(self) -> Result<(), InstallError> { + INBOX_HEALTH + .set(Arc::new(self)) + .map_err(|_| InstallError::AlreadyInstalled) + } + + /// The process-wide health record, if one was installed. + pub fn global() -> Option> { + INBOX_HEALTH.get().cloned() + } + + /// Record the current observation, returning the resulting status. + pub(super) fn observe(&self, status: InboxStatus, now: i64) -> InboxStatus { + let mut state = self.state(); + let first_verdict = state.verdict.is_none(); + state.verdict = Some(status); + + match (status, state.blind_now().is_some()) { + (InboxStatus::Blind, false) => { + // A first audit that finds the inbox deaf has found an outage + // that was already running: the node has not heard anything + // since it came up, so that is when the outage began. + let start = if first_verdict { + state.installed_at + } else { + now + }; + state.windows.push(BlindWindow { start, end: None }); + } + (InboxStatus::Listening, true) => { + if let Some(open) = state.windows.last_mut() { + open.end = Some(now); + } + } + _ => {} + } + + state.windows.retain(|w| match w.end { + Some(end) => end > now - BLIND_WINDOW_RETENTION_SECS, + None => true, + }); + if state.windows.len() > MAX_BLIND_WINDOWS { + let excess = state.windows.len() - MAX_BLIND_WINDOWS; + state.windows.drain(..excess); + } + + status + } + + /// Whether the inbox is deaf right now. + /// + /// A record that has never been audited is not blind — but neither is it + /// known to be listening, which is the question a caller about to act on a + /// user's silence should be asking. See [`Self::is_confirmed_listening`]. + pub fn is_blind(&self) -> bool { + self.state().blind_now().is_some() + } + + /// Whether an audit has actually confirmed that Mostro can hear. + /// + /// This is the predicate for anything that punishes a user for not + /// answering. It is deliberately false before the first audit: the daemon + /// subscribes at startup and the watchdog's first pass comes later, so + /// between the two there is a window in which a node that never obtained a + /// working inbox would otherwise look healthy and start cancelling orders + /// and slashing bonds over messages it was never in a position to receive. + pub fn is_confirmed_listening(&self) -> bool { + self.state().verdict == Some(InboxStatus::Listening) + } + + /// Seconds the inbox was deaf between `from` and now. + /// + /// This is the compensation a single order is owed, and it is computed per + /// order on purpose. A deadline is wall-clock, but the user is answering a + /// node that has to be listening for the answer to land — and because a + /// message sent into a dead inbox is lost rather than queued (the + /// ten-second replay window, see the module docs), they must send it again + /// once the ear is back. So an order's clock effectively stops for exactly + /// the outages that overlap its own waiting period: an order that was + /// already waiting through an outage is owed all of it, one taken + /// afterwards is owed nothing. + pub fn blind_seconds_since(&self, from: i64) -> i64 { + self.blind_seconds_between(from, now_secs()) + } + + fn blind_seconds_between(&self, from: i64, to: i64) -> i64 { + let state = self.state(); + state.windows.iter().map(|w| w.overlap(from, to, to)).sum() + } + + /// Upper bound on what any order could be owed. + /// + /// No decision is taken on this figure — an order's credit is always the + /// downtime that overlaps its own wait ([`Self::blind_seconds_since`]). + /// It exists so the timeout job can tell an operator, in one line, how + /// much downtime is in play this tick. + pub fn max_blind_seconds(&self) -> i64 { + let now = now_secs(); + let state = self.state(); + state + .windows + .iter() + .map(|w| w.end.unwrap_or(now) - w.start) + .sum::() + .max(0) + } + + /// Record that `relay` answered the inbox REQ (an `EOSE` for our + /// subscription), which is the only evidence that it is really serving it. + /// + /// Whatever was making the relay fail is over, so its pacing is reset too + /// and the next failure earns a prompt retry again. Returns whether the + /// relay was being backed off, which is what distinguishes a recovery + /// worth logging from the steady state. + pub fn note_relay_acknowledged(&self, relay: &RelayUrl) -> bool { + self.note_relay_acknowledged_at(relay, now_secs()) + } + + pub(super) fn note_relay_acknowledged_at(&self, relay: &RelayUrl, at: i64) -> bool { + let mut state = self.state(); + state.acknowledged.insert(relay.clone(), at); + state.backoff.remove(relay).is_some() + } + + /// Whether a re-subscribe to `relay` may go out now, arming the next delay + /// when it may. The first failure for a relay always passes. + /// + /// This is the single pacing budget the event loop and the watchdog share. + /// Under the watchdog's 30-second cadence the doubling only starts to bite + /// once the delay outgrows the interval — so a relay that lost the inbox + /// once is re-subscribed on the very next pass, and only one that keeps + /// refusing tapers to [`RESUBSCRIBE_MAX_BACKOFF`]. + pub fn allow_resubscribe(&self, relay: &RelayUrl) -> bool { + self.allow_resubscribe_at(relay, Instant::now()) + } + + fn allow_resubscribe_at(&self, relay: &RelayUrl, now: Instant) -> bool { + let mut state = self.state(); + match state.backoff.get_mut(relay) { + None => { + state.backoff.insert( + relay.clone(), + RelayBackoff { + next_attempt_at: now + RESUBSCRIBE_INITIAL_BACKOFF, + delay: RESUBSCRIBE_INITIAL_BACKOFF, + }, + ); + true + } + Some(pacing) => { + if now < pacing.next_attempt_at { + return false; + } + pacing.delay = (pacing.delay * 2).min(RESUBSCRIBE_MAX_BACKOFF); + pacing.next_attempt_at = now + pacing.delay; + true + } + } + } + + /// Forget `relay`'s acknowledgement: whatever it said about serving the + /// inbox no longer applies. + /// + /// Two callers, one meaning. A fresh REQ has gone out and has yet to be + /// answered (`resubscribe_relay`); or the relay closed the subscription + /// provisionally, so the entry the SDK left registered is not evidence of + /// anything until the replacement REQ is answered + /// (`InboxKeeper::on_relay_message`). In both cases the audit has to judge + /// the relay on the new evidence rather than on the old `EOSE`. + pub fn note_relay_unacknowledged(&self, relay: &RelayUrl) { + self.state().acknowledged.remove(relay); + } + + /// Whether `relay` answered the inbox REQ *on its current connection*. + /// + /// `connected_at` is when the websocket the audit is looking at was + /// established. An acknowledgement older than that was earned on a session + /// that no longer exists: the SDK re-sends the REQ on reconnect of its own + /// accord (`should_resubscribe`), emitting nothing this module can see, so + /// a relay that comes back and then quietly ignores the replacement would + /// otherwise keep reading as healthy on the strength of its old `EOSE`. + /// + /// The comparison is inclusive so that an `EOSE` landing in the same + /// second as the connect still counts. + pub fn has_acknowledged_since(&self, relay: &RelayUrl, connected_at: i64) -> bool { + self.state() + .acknowledged + .get(relay) + .is_some_and(|&at| at >= connected_at) + } + + /// How long the current outage has been running, or zero if listening. + pub fn blind_for_secs(&self) -> i64 { + let now = now_secs(); + self.state() + .blind_now() + .map(|w| (now - w.start).max(0)) + .unwrap_or(0) + } + + /// How long it has been since an audit confirmed Mostro can hear. + /// + /// Zero while listening. Otherwise it counts from the start of the current + /// outage, or — if no audit has ever run — from startup, so a watchdog + /// that never reported cannot leave a caller waiting forever on a verdict + /// that is not coming. + pub fn unconfirmed_for_secs(&self) -> i64 { + let now = now_secs(); + let state = self.state(); + if state.verdict == Some(InboxStatus::Listening) && state.blind_now().is_none() { + return 0; + } + let since = state + .blind_now() + .map(|w| w.start) + .unwrap_or(state.installed_at); + (now - since).max(0) + } + + /// Whether re-subscribes to `relay` are currently being paced. + /// + /// Pacing is deliberately not observable in production — callers ask + /// [`Self::allow_resubscribe`], which also arms the next delay — but the + /// keeper's tests next door assert on it, and the map is private here. + #[cfg(test)] + pub(super) fn is_backing_off(&self, relay: &RelayUrl) -> bool { + self.state().backoff.contains_key(relay) + } +} + +/// Wall-clock seconds, the base an order's `taken_at` is recorded in. +pub(super) fn now_secs() -> i64 { + Timestamp::now().as_secs() as i64 +} + +/// Origin for the wall-clock arithmetic under test: health observations are +/// timestamp-based, so tests drive a fixed origin rather than the real clock. +/// Shared with the keeper's tests next door. +#[cfg(test)] +pub(super) const T0: i64 = 1_700_000_000; + +#[cfg(test)] +mod tests { + use super::*; + + fn relay_url(url: &str) -> RelayUrl { + RelayUrl::parse(url).expect("valid relay url") + } + + // ───────────────────────────── backoff pacing ───────────────────────────── + + #[test] + fn first_closure_from_a_relay_retries_immediately() { + let health = InboxHealth::at(T0); + let relay = relay_url("ws://relay.example"); + + assert!( + health.allow_resubscribe_at(&relay, Instant::now()), + "a first CLOSED must be answered at once: every delay is deaf time" + ); + } + + #[test] + fn repeat_closures_are_paced_and_back_off() { + let health = InboxHealth::at(T0); + let relay = relay_url("ws://relay.example"); + let start = Instant::now(); + + assert!(health.allow_resubscribe_at(&relay, start)); + // A relay that closes again right away must not pull a second REQ. + assert!(!health.allow_resubscribe_at(&relay, start)); + assert!(!health.allow_resubscribe_at(&relay, start + Duration::from_secs(1))); + + // Past the first delay it retries, and the next wait is longer. + assert!(health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF)); + assert!(!health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 2)); + assert!(health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 3)); + } + + #[test] + fn backoff_is_capped() { + let health = InboxHealth::at(T0); + let relay = relay_url("ws://relay.example"); + let mut now = Instant::now(); + + // Drive it well past the ceiling. + for _ in 0..20 { + assert!(health.allow_resubscribe_at(&relay, now)); + now += RESUBSCRIBE_MAX_BACKOFF * 2; + } + + assert_eq!( + health + .state + .lock() + .unwrap() + .backoff + .get(&relay) + .expect("state kept") + .delay, + RESUBSCRIBE_MAX_BACKOFF, + "a hostile relay must still be retried every {RESUBSCRIBE_MAX_BACKOFF:?}" + ); + } + + #[test] + fn backoff_is_per_relay() { + let health = InboxHealth::at(T0); + let hostile = relay_url("ws://hostile.example"); + let healthy = relay_url("ws://healthy.example"); + let now = Instant::now(); + + assert!(health.allow_resubscribe_at(&hostile, now)); + assert!(!health.allow_resubscribe_at(&hostile, now)); + // One misbehaving relay must not delay recovery on another. + assert!(health.allow_resubscribe_at(&healthy, now)); + } + + #[test] + fn an_acknowledgement_clears_the_pacing_for_the_next_failure() { + let health = InboxHealth::at(T0); + let relay = relay_url("ws://relay.example"); + let now = Instant::now(); + + assert!(health.allow_resubscribe_at(&relay, now)); + assert!(!health.allow_resubscribe_at(&relay, now)); + + assert!( + health.note_relay_acknowledged(&relay), + "clearing a live backoff entry is what marks a recovery" + ); + assert!( + health.allow_resubscribe_at(&relay, now), + "a relay that answered starts over: the next failure is a fresh one" + ); + + // The steady state has nothing to clear, so nothing to report either. + health.note_relay_acknowledged(&relay); + assert!(!health.note_relay_acknowledged(&relay)); + } + + #[test] + fn the_watchdog_cadence_recovers_promptly_and_only_then_tapers() { + // The point of sharing one budget: an audit every + // `INBOX_WATCHDOG_INTERVAL` must still re-subscribe a relay that + // simply lost the inbox, while a relay that refuses it converges on + // the advertised ceiling instead of drawing a REQ every 30 seconds + // forever. + let health = InboxHealth::at(T0); + let relay = relay_url("ws://hostile.example"); + let tick = Duration::from_secs(crate::scheduler::INBOX_WATCHDOG_INTERVAL); + let mut now = Instant::now(); + + assert!( + health.allow_resubscribe_at(&relay, now), + "the pass that first notices the loss must act on it" + ); + for pass in 1..=3 { + now += tick; + assert!( + health.allow_resubscribe_at(&relay, now), + "pass {pass}: a delay still under the audit interval must not skip a retry" + ); + } + + // Once the doubling outgrows the interval, passes start being skipped. + let mut attempts = 0; + for _ in 0..40 { + now += tick; + if health.allow_resubscribe_at(&relay, now) { + attempts += 1; + } + } + assert!( + attempts < 40, + "a relay that keeps refusing must stop drawing a REQ on every pass" + ); + assert_eq!( + health + .state + .lock() + .unwrap() + .backoff + .get(&relay) + .expect("state kept") + .delay, + RESUBSCRIBE_MAX_BACKOFF + ); + } + + // ───────────────────────────── health record ───────────────────────────── + + #[test] + fn health_records_an_outage_from_first_blindness_to_recovery() { + let health = InboxHealth::at(T0); + + assert!(!health.is_blind(), "a fresh record starts out listening"); + + health.observe(InboxStatus::Blind, T0); + assert!(health.is_blind()); + + // Staying blind must not restart the clock — the outage began at the + // first observation, and that is what an order is owed. + health.observe(InboxStatus::Blind, T0 + 30); + assert!(health.is_blind()); + + health.observe(InboxStatus::Listening, T0 + 90); + assert!(!health.is_blind()); + + assert_eq!( + health.blind_seconds_between(T0, T0 + 90), + 90, + "the recorded outage must span the whole blind window" + ); + } + + #[test] + fn health_is_not_listening_until_an_audit_says_so() { + let health = InboxHealth::at(T0); + + // Startup is not evidence. Between `main` subscribing and the + // watchdog's first pass, a node whose inbox never worked would + // otherwise process timeouts as if it had been listening all along. + assert!( + !health.is_confirmed_listening(), + "an unaudited record must not authorise acting on a user's silence" + ); + assert!( + !health.is_blind(), + "nor should it claim an outage it has not observed" + ); + + health.observe(InboxStatus::Listening, T0); + assert!(health.is_confirmed_listening()); + } + + #[test] + fn a_blind_first_audit_dates_the_outage_from_startup() { + let health = InboxHealth::at(T0); + + // The watchdog's first pass comes some time after boot. Finding the + // inbox deaf then means it was deaf for that whole stretch, not just + // from the moment somebody looked. + health.observe(InboxStatus::Blind, T0 + 30); + health.observe(InboxStatus::Listening, T0 + 90); + + assert_eq!( + health.blind_seconds_between(T0, T0 + 90), + 90, + "the outage must be dated from startup, not from the first audit" + ); + } + + #[test] + fn a_node_that_was_never_blind_owes_nothing() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + + assert_eq!(health.blind_seconds_between(T0, T0 + 10_000), 0); + assert_eq!(health.max_blind_seconds(), 0); + } + + // ──────────────────── what a single order is owed ──────────────────── + + #[test] + fn an_order_is_owed_only_the_downtime_it_waited_through() { + let health = InboxHealth::at(T0); + // One outage: [T0+100, T0+400], five minutes. + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 400); + + let now = T0 + 1_000; + + // Waiting since before it started: owed the whole outage. + assert_eq!(health.blind_seconds_between(T0, now), 300); + // Taken midway through: owed only the remainder. + assert_eq!(health.blind_seconds_between(T0 + 250, now), 150); + // Taken after it ended: owed nothing. This is what a single global + // allowance got wrong — it credited orders that never lost a second. + assert_eq!(health.blind_seconds_between(T0 + 500, now), 0); + } + + #[test] + fn compensation_does_not_evaporate_as_time_passes() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 400); + + // The debt an order carries is a property of when it waited, not of + // how long ago the outage was. A decaying allowance wore off at the + // same rate the deadline advanced, so it compensated almost nothing. + for probe in [400, 700, 5_000, 50_000] { + assert_eq!( + health.blind_seconds_between(T0, T0 + probe), + 300, + "an order waiting since T0 is owed the outage regardless of when we ask" + ); + } + } + + #[test] + fn an_order_waiting_through_an_outage_survives_its_nominal_deadline() { + // The regression in full: 900s timeout, an order taken at T0, and a + // 300s outage right at the start. Under the old decaying allowance + // this order was cancelled at ~T0+900, having had only 600s of + // listening time. + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Blind, T0); + health.observe(InboxStatus::Listening, T0 + 300); + + let exp_seconds = 900i64; + let late_at = |now: i64| { + let owed = health.blind_seconds_between(T0, now); + (now - T0) >= exp_seconds + owed + }; + + assert!(!late_at(T0 + 900), "cancelled after only 600s of listening"); + assert!(!late_at(T0 + 1_199)); + assert!( + late_at(T0 + 1_200), + "and it must still expire once it has had its full 900s" + ); + } + + /// Regression: the credit is per order, so an order taken *after* an + /// outage ended must expire at its nominal deadline. + /// + /// The timeout job used to widen `find_order_by_seconds` by + /// [`InboxHealth::max_blind_seconds`], which narrows the selection rather + /// than widening it — every surviving row was already past + /// `deadline + max_blind_seconds`, the per-order check could never spare + /// anything, and what shipped was the global allowance this design + /// rejects. That allowance grows with every outage in the retention + /// window, so a node with flapping relays would postpone every deadline by + /// hours of unrelated downtime. + #[test] + fn an_order_taken_after_an_outage_is_not_credited_for_it() { + let health = InboxHealth::at(T0); + // One outage: [T0, T0+300]. + health.observe(InboxStatus::Blind, T0); + health.observe(InboxStatus::Listening, T0 + 300); + + let exp_seconds = 900i64; + let late_at = |taken_at: i64, now: i64| { + let owed = health.blind_seconds_between(taken_at, now); + (now - taken_at) >= exp_seconds + owed + }; + + // A: waited through the whole outage, owed all 300s. + assert!(!late_at(T0, T0 + 1_199)); + assert!(late_at(T0, T0 + 1_200)); + + // B: taken after recovery, owed nothing — even though the node's total + // downtime is the same 300s the global allowance would have handed it. + assert_eq!(health.max_blind_seconds(), 300); + assert!(!late_at(T0 + 400, T0 + 1_299)); + assert!( + late_at(T0 + 400, T0 + 1_300), + "an order that never lost a second must expire at its nominal deadline" + ); + } + + #[test] + fn consecutive_outages_accumulate_their_debt() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 200); + health.observe(InboxStatus::Blind, T0 + 240); + health.observe(InboxStatus::Listening, T0 + 290); + + assert_eq!( + health.blind_seconds_between(T0, T0 + 1_000), + 150, + "an order waiting through both outages is owed both" + ); + assert_eq!( + health.blind_seconds_between(T0 + 210, T0 + 1_000), + 50, + "one taken between them is owed only the second" + ); + } + + #[test] + fn an_ongoing_outage_counts_up_to_now() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + + assert_eq!(health.blind_seconds_between(T0, T0 + 400), 300); + assert_eq!(health.blind_seconds_between(T0, T0 + 900), 800); + } + + #[test] + fn stale_windows_are_pruned() { + let health = InboxHealth::at(T0); + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Blind, T0 + 100); + health.observe(InboxStatus::Listening, T0 + 200); + + // Far past the retention horizon, the old window is dropped rather + // than accumulating for the life of the process. + let much_later = T0 + BLIND_WINDOW_RETENTION_SECS + 1_000; + health.observe(InboxStatus::Listening, much_later); + + assert_eq!(health.blind_seconds_between(T0, much_later), 0); + assert!(health.state.lock().expect("lock").windows.is_empty()); + } + + #[test] + fn unconfirmed_time_counts_from_the_outage_or_from_startup() { + // What bounds how long the timeout job may defer. It has to answer + // even when no audit ever ran, or a watchdog that died would park the + // job on a verdict that is never coming. + let never_audited = InboxHealth::at(now_secs() - 120); + assert!( + never_audited.unconfirmed_for_secs() >= 120, + "with no verdict at all, the clock runs from startup" + ); + + let healthy = InboxHealth::at(now_secs()); + healthy.observe(InboxStatus::Listening, now_secs()); + assert_eq!( + healthy.unconfirmed_for_secs(), + 0, + "a confirmed inbox owes no waiting" + ); + + let blind = InboxHealth::at(now_secs() - 600); + blind.observe(InboxStatus::Listening, now_secs() - 600); + blind.observe(InboxStatus::Blind, now_secs() - 300); + assert!( + (300..=310).contains(&blind.unconfirmed_for_secs()), + "while blind it runs from the start of the outage, got {}", + blind.unconfirmed_for_secs() + ); + } + + #[test] + fn health_ignores_repeated_healthy_observations() { + let health = InboxHealth::at(T0); + + health.observe(InboxStatus::Listening, T0); + health.observe(InboxStatus::Listening, T0 + 30); + + assert!(!health.is_blind()); + assert_eq!( + health.blind_seconds_between(T0, T0 + 30), + 0, + "a node that was never blind has no outage to compensate for" + ); + } + + #[test] + fn an_acknowledgement_does_not_survive_the_connection_it_was_earned_on() { + // A websocket drop and reconnect leaves no trace the keeper can act + // on: there is no relay-status `ClientNotification` in nostr-sdk + // 0.45.1, and the SDK silently re-sends the REQ by itself + // (`should_resubscribe`). If the relay then ignores that replacement, + // the only thing standing between a deaf node and resumed slashing is + // the acknowledgement expiring with its session. + let health = InboxHealth::at(T0); + let url = relay_url("ws://relay.example"); + + health.note_relay_acknowledged_at(&url, T0 + 100); + + assert!(health.has_acknowledged_since(&url, T0 + 50)); + assert!( + health.has_acknowledged_since(&url, T0 + 100), + "an EOSE landing in the same second as the connect must still count" + ); + assert!( + !health.has_acknowledged_since(&url, T0 + 101), + "credit earned on a previous connection must not vouch for this one" + ); + } +} diff --git a/src/inbox.rs b/src/inbox/mod.rs similarity index 59% rename from src/inbox.rs rename to src/inbox/mod.rs index b9574c3e..bf2b6940 100644 --- a/src/inbox.rs +++ b/src/inbox/mod.rs @@ -52,14 +52,20 @@ //! whether it is still serving the subscription, re-subscribes the ones that //! are not, and records the verdict in [`InboxHealth`] so the rest of the //! daemon can tell whether Mostro is currently able to hear anything at all. +//! +//! The health record itself — the outage log the scheduler reads, and the +//! per-relay pacing both recovery paths draw on — is in [`health`]. -use std::collections::HashMap; -use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; -use std::time::{Duration, Instant}; +use std::sync::Arc; use nostr_sdk::prelude::*; use tracing::{debug, error, info, warn}; +mod health; + +use health::now_secs; +pub use health::{InboxHealth, InboxStatus, InstallError}; + /// Subscription id used for the daemon inbox. /// /// Fixed rather than the SDK's per-call random id, so a `CLOSED` frame can be @@ -68,38 +74,6 @@ use tracing::{debug, error, info, warn}; /// tag already names this node. const INBOX_SUBSCRIPTION_ID: &str = "mostro-inbox"; -/// Delay before a *second* consecutive re-subscribe to the same relay. -/// -/// The first `CLOSED` is answered immediately — the common case is a transient -/// refusal, and every second of delay is a second of deaf node. Backoff only -/// starts mattering when a relay keeps closing the inbox. -const RESUBSCRIBE_INITIAL_BACKOFF: Duration = Duration::from_secs(2); - -/// Ceiling for the per-relay re-subscribe delay. -/// -/// A relay that has refused the inbox for five minutes straight is not having -/// a hiccup — it is configured to refuse us (NIP-42, a pubkey allowlist, a ban) -/// and the operator has to intervene. Retrying every five minutes keeps the -/// door open for a config change on their side without generating traffic that -/// looks like an attack. -/// -/// This is a real ceiling because [`check_inbox_health`] draws on the same -/// per-relay budget rather than re-sending on every pass: an audit every -/// `INBOX_WATCHDOG_INTERVAL` would otherwise put a hard floor of thirty -/// seconds under it. The doublings still start well below that interval, so a -/// relay that merely lost the inbox is re-subscribed on the next pass and only -/// a persistently refusing one reaches this figure. -const RESUBSCRIBE_MAX_BACKOFF: Duration = Duration::from_secs(300); - -/// Per-relay re-subscribe pacing. -#[derive(Debug)] -struct RelayBackoff { - /// Earliest instant at which another REQ may go out to this relay. - next_attempt_at: Instant, - /// Delay applied after the next attempt; doubles up to the ceiling. - delay: Duration, -} - /// The daemon's inbox: the subscription every trade message arrives on. #[derive(Debug, Clone)] pub struct InboxSubscription { @@ -319,7 +293,7 @@ fn is_provisional_closure(message: &str) -> bool { /// - **Pacing.** Both callers draw on one per-relay budget in [`InboxHealth`]. /// The watchdog would otherwise re-send unconditionally on every pass, /// putting a hard floor of `INBOX_WATCHDOG_INTERVAL` under a ceiling that -/// claims to be [`RESUBSCRIBE_MAX_BACKOFF`]. Sharing it keeps a transient +/// claims to be `RESUBSCRIBE_MAX_BACKOFF`. Sharing it keeps a transient /// failure recovering on the very next audit while a relay that refuses the /// inbox on principle tapers to one REQ every five minutes. /// - **Acknowledgement.** From the moment a fresh REQ goes out, an earlier @@ -365,370 +339,6 @@ async fn resubscribe_relay( true } -/// Whether the daemon can currently hear anything at all. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InboxStatus { - /// At least one connected relay is serving the inbox subscription. - Listening, - /// No connected relay is serving it: every message sent to Mostro right - /// now is being lost. - Blind, -} - -/// Process-wide inbox health. `None` until [`InboxHealth::install_global`] -/// runs at startup; consumers treat an absent health record as "listening", so -/// unit tests that never install it behave as before. -static INBOX_HEALTH: OnceLock> = OnceLock::new(); - -/// Why [`InboxHealth::install_global`] refused. Mirrors `spam_gate::InstallError`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InstallError { - /// A health record is already installed. - AlreadyInstalled, -} - -/// One stretch during which the daemon could not hear. -/// -/// Timestamps are wall-clock seconds, the same base as an order's `taken_at`, -/// because that is what these windows are ultimately intersected against. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct BlindWindow { - start: i64, - /// `None` while the outage is still running. - end: Option, -} - -impl BlindWindow { - /// Seconds of this window that fall inside `[from, to]`. - fn overlap(&self, from: i64, to: i64, now: i64) -> i64 { - let end = self.end.unwrap_or(now); - (end.min(to) - self.start.max(from)).max(0) - } -} - -/// Windows that ended longer ago than this are dropped: no order the timeout -/// job can still be looking at was waiting back then, so they can no longer -/// change any verdict. Generous next to `expiration_seconds` (900s by default) -/// so the bound is never the reason a user loses compensation. -const BLIND_WINDOW_RETENTION_SECS: i64 = 7 * 24 * 3600; - -/// Hard cap on retained windows, so a relay flapping in a tight loop cannot -/// grow this unboundedly between prunes. -const MAX_BLIND_WINDOWS: usize = 512; - -#[derive(Debug)] -struct HealthState { - /// The last verdict an audit reached. `None` until the first one runs: - /// startup is not evidence that the inbox works, and must not be read as - /// such (see [`InboxHealth::is_confirmed_listening`]). - verdict: Option, - /// When the record was installed, which is the earliest moment an outage - /// discovered by the first audit could have begun. - installed_at: i64, - /// Every outage this process has seen, oldest first, pruned by age. - windows: Vec, - /// Relay to the wall-clock second its last `EOSE` for the inbox arrived. - /// - /// A relay is only credited with serving the inbox once it says so. The - /// SDK's own subscription map cannot stand in for this: it records what - /// *we* sent, so a relay that holds the connection open and quietly - /// ignores the REQ still looks subscribed there. - /// - /// The timestamp is what binds the credit to a single websocket session. - /// On reconnect the SDK re-sends the REQ by itself, with no frame this - /// module can observe, so an acknowledgement earned on the previous - /// connection says nothing about the current one — see - /// [`InboxHealth::has_acknowledged_since`]. - acknowledged: HashMap, - /// Re-subscribe pacing, drawn on by the event loop and the watchdog alike. - /// - /// Only holds relays that are currently failing; a relay that answers the - /// REQ is dropped from the map, so the steady state is empty. - backoff: HashMap, -} - -impl HealthState { - fn blind_now(&self) -> Option<&BlindWindow> { - self.windows.last().filter(|w| w.end.is_none()) - } -} - -/// Tracks whether the daemon's ear is open, and when it was not. -/// -/// The scheduler's timeout machinery reads this: an order is only "late" if -/// Mostro was in a position to hear from the user, and the ten-second replay -/// window means a message sent into a dead inbox is gone rather than delayed -/// (see the module docs). Punishing a user for a silence the node itself -/// caused would be unfair, so the timeout clock stops while the inbox is down. -#[derive(Debug)] -pub struct InboxHealth { - state: Mutex, -} - -impl Default for InboxHealth { - fn default() -> Self { - Self::new() - } -} - -impl InboxHealth { - pub fn new() -> Self { - Self::at(now_secs()) - } - - /// The shared state, recovering rather than propagating a poisoned lock. - /// - /// [`HealthState`] is plain data with no invariant a panic could leave - /// half-applied, and no user code runs under the guard — so poisoning - /// carries no information worth acting on. Propagating it would, though: - /// every consumer of this record is a maintenance job, and - /// [`Self::is_confirmed_listening`] is read by the timeout job on every - /// tick. A panic there kills that task for the life of the process, which - /// stops timeouts permanently — hold invoices ride to CLTV expiry and - /// bonds never resolve. Taking the data as it stands is strictly better - /// than that. - fn state(&self) -> MutexGuard<'_, HealthState> { - self.state.lock().unwrap_or_else(|e| e.into_inner()) - } - - fn at(installed_at: i64) -> Self { - Self { - state: Mutex::new(HealthState { - verdict: None, - installed_at, - windows: Vec::new(), - acknowledged: HashMap::new(), - backoff: HashMap::new(), - }), - } - } - - /// Install as the process-wide health record. - pub fn install_global(self) -> Result<(), InstallError> { - INBOX_HEALTH - .set(Arc::new(self)) - .map_err(|_| InstallError::AlreadyInstalled) - } - - /// The process-wide health record, if one was installed. - pub fn global() -> Option> { - INBOX_HEALTH.get().cloned() - } - - /// Record the current observation, returning the resulting status. - fn observe(&self, status: InboxStatus, now: i64) -> InboxStatus { - let mut state = self.state(); - let first_verdict = state.verdict.is_none(); - state.verdict = Some(status); - - match (status, state.blind_now().is_some()) { - (InboxStatus::Blind, false) => { - // A first audit that finds the inbox deaf has found an outage - // that was already running: the node has not heard anything - // since it came up, so that is when the outage began. - let start = if first_verdict { - state.installed_at - } else { - now - }; - state.windows.push(BlindWindow { start, end: None }); - } - (InboxStatus::Listening, true) => { - if let Some(open) = state.windows.last_mut() { - open.end = Some(now); - } - } - _ => {} - } - - state.windows.retain(|w| match w.end { - Some(end) => end > now - BLIND_WINDOW_RETENTION_SECS, - None => true, - }); - if state.windows.len() > MAX_BLIND_WINDOWS { - let excess = state.windows.len() - MAX_BLIND_WINDOWS; - state.windows.drain(..excess); - } - - status - } - - /// Whether the inbox is deaf right now. - /// - /// A record that has never been audited is not blind — but neither is it - /// known to be listening, which is the question a caller about to act on a - /// user's silence should be asking. See [`Self::is_confirmed_listening`]. - pub fn is_blind(&self) -> bool { - self.state().blind_now().is_some() - } - - /// Whether an audit has actually confirmed that Mostro can hear. - /// - /// This is the predicate for anything that punishes a user for not - /// answering. It is deliberately false before the first audit: the daemon - /// subscribes at startup and the watchdog's first pass comes later, so - /// between the two there is a window in which a node that never obtained a - /// working inbox would otherwise look healthy and start cancelling orders - /// and slashing bonds over messages it was never in a position to receive. - pub fn is_confirmed_listening(&self) -> bool { - self.state().verdict == Some(InboxStatus::Listening) - } - - /// Seconds the inbox was deaf between `from` and now. - /// - /// This is the compensation a single order is owed, and it is computed per - /// order on purpose. A deadline is wall-clock, but the user is answering a - /// node that has to be listening for the answer to land — and because a - /// message sent into a dead inbox is lost rather than queued (the - /// ten-second replay window, see the module docs), they must send it again - /// once the ear is back. So an order's clock effectively stops for exactly - /// the outages that overlap its own waiting period: an order that was - /// already waiting through an outage is owed all of it, one taken - /// afterwards is owed nothing. - pub fn blind_seconds_since(&self, from: i64) -> i64 { - self.blind_seconds_between(from, now_secs()) - } - - fn blind_seconds_between(&self, from: i64, to: i64) -> i64 { - let state = self.state(); - state.windows.iter().map(|w| w.overlap(from, to, to)).sum() - } - - /// Upper bound on what any order could be owed. - /// - /// No decision is taken on this figure — an order's credit is always the - /// downtime that overlaps its own wait ([`Self::blind_seconds_since`]). - /// It exists so the timeout job can tell an operator, in one line, how - /// much downtime is in play this tick. - pub fn max_blind_seconds(&self) -> i64 { - let now = now_secs(); - let state = self.state(); - state - .windows - .iter() - .map(|w| w.end.unwrap_or(now) - w.start) - .sum::() - .max(0) - } - - /// Record that `relay` answered the inbox REQ (an `EOSE` for our - /// subscription), which is the only evidence that it is really serving it. - /// - /// Whatever was making the relay fail is over, so its pacing is reset too - /// and the next failure earns a prompt retry again. Returns whether the - /// relay was being backed off, which is what distinguishes a recovery - /// worth logging from the steady state. - pub fn note_relay_acknowledged(&self, relay: &RelayUrl) -> bool { - self.note_relay_acknowledged_at(relay, now_secs()) - } - - fn note_relay_acknowledged_at(&self, relay: &RelayUrl, at: i64) -> bool { - let mut state = self.state(); - state.acknowledged.insert(relay.clone(), at); - state.backoff.remove(relay).is_some() - } - - /// Whether a re-subscribe to `relay` may go out now, arming the next delay - /// when it may. The first failure for a relay always passes. - /// - /// This is the single pacing budget the event loop and the watchdog share. - /// Under the watchdog's 30-second cadence the doubling only starts to bite - /// once the delay outgrows the interval — so a relay that lost the inbox - /// once is re-subscribed on the very next pass, and only one that keeps - /// refusing tapers to [`RESUBSCRIBE_MAX_BACKOFF`]. - pub fn allow_resubscribe(&self, relay: &RelayUrl) -> bool { - self.allow_resubscribe_at(relay, Instant::now()) - } - - fn allow_resubscribe_at(&self, relay: &RelayUrl, now: Instant) -> bool { - let mut state = self.state(); - match state.backoff.get_mut(relay) { - None => { - state.backoff.insert( - relay.clone(), - RelayBackoff { - next_attempt_at: now + RESUBSCRIBE_INITIAL_BACKOFF, - delay: RESUBSCRIBE_INITIAL_BACKOFF, - }, - ); - true - } - Some(pacing) => { - if now < pacing.next_attempt_at { - return false; - } - pacing.delay = (pacing.delay * 2).min(RESUBSCRIBE_MAX_BACKOFF); - pacing.next_attempt_at = now + pacing.delay; - true - } - } - } - - /// Forget `relay`'s acknowledgement: whatever it said about serving the - /// inbox no longer applies. - /// - /// Two callers, one meaning. A fresh REQ has gone out and has yet to be - /// answered ([`resubscribe_relay`]); or the relay closed the subscription - /// provisionally, so the entry the SDK left registered is not evidence of - /// anything until the replacement REQ is answered ([`InboxKeeper::on_relay_message`]). - /// In both cases the audit has to judge the relay on the new evidence - /// rather than on the old `EOSE`. - pub fn note_relay_unacknowledged(&self, relay: &RelayUrl) { - self.state().acknowledged.remove(relay); - } - - /// Whether `relay` answered the inbox REQ *on its current connection*. - /// - /// `connected_at` is when the websocket the audit is looking at was - /// established. An acknowledgement older than that was earned on a session - /// that no longer exists: the SDK re-sends the REQ on reconnect of its own - /// accord (`should_resubscribe`), emitting nothing this module can see, so - /// a relay that comes back and then quietly ignores the replacement would - /// otherwise keep reading as healthy on the strength of its old `EOSE`. - /// - /// The comparison is inclusive so that an `EOSE` landing in the same - /// second as the connect still counts. - pub fn has_acknowledged_since(&self, relay: &RelayUrl, connected_at: i64) -> bool { - self.state() - .acknowledged - .get(relay) - .is_some_and(|&at| at >= connected_at) - } - - /// How long the current outage has been running, or zero if listening. - pub fn blind_for_secs(&self) -> i64 { - let now = now_secs(); - self.state() - .blind_now() - .map(|w| (now - w.start).max(0)) - .unwrap_or(0) - } - - /// How long it has been since an audit confirmed Mostro can hear. - /// - /// Zero while listening. Otherwise it counts from the start of the current - /// outage, or — if no audit has ever run — from startup, so a watchdog - /// that never reported cannot leave a caller waiting forever on a verdict - /// that is not coming. - pub fn unconfirmed_for_secs(&self) -> i64 { - let now = now_secs(); - let state = self.state(); - if state.verdict == Some(InboxStatus::Listening) && state.blind_now().is_none() { - return 0; - } - let since = state - .blind_now() - .map(|w| w.start) - .unwrap_or(state.installed_at); - (now - since).max(0) - } -} - -/// Wall-clock seconds, the base an order's `taken_at` is recorded in. -fn now_secs() -> i64 { - Timestamp::now().as_secs() as i64 -} - /// Check every read relay, re-subscribing any that is not serving the inbox, /// and record the verdict in the process-wide [`InboxHealth`]. /// @@ -833,7 +443,9 @@ async fn check_inbox_health_with( #[cfg(test)] mod tests { + use super::health::T0; use super::*; + use std::time::Duration; fn pubkey() -> PublicKey { Keys::generate().public_key() @@ -850,11 +462,6 @@ mod tests { (keeper, health) } - /// Whether `health` is currently pacing re-subscribes to `url`. - fn backing_off(health: &InboxHealth, url: &RelayUrl) -> bool { - health.state().backoff.contains_key(url) - } - fn relay_url(url: &str) -> RelayUrl { RelayUrl::parse(url).expect("valid relay url") } @@ -919,147 +526,6 @@ mod tests { .contains(&Kind::PrivateDirectMessage)); } - // ───────────────────────────── backoff pacing ───────────────────────────── - - #[test] - fn first_closure_from_a_relay_retries_immediately() { - let health = InboxHealth::at(T0); - let relay = relay_url("ws://relay.example"); - - assert!( - health.allow_resubscribe_at(&relay, Instant::now()), - "a first CLOSED must be answered at once: every delay is deaf time" - ); - } - - #[test] - fn repeat_closures_are_paced_and_back_off() { - let health = InboxHealth::at(T0); - let relay = relay_url("ws://relay.example"); - let start = Instant::now(); - - assert!(health.allow_resubscribe_at(&relay, start)); - // A relay that closes again right away must not pull a second REQ. - assert!(!health.allow_resubscribe_at(&relay, start)); - assert!(!health.allow_resubscribe_at(&relay, start + Duration::from_secs(1))); - - // Past the first delay it retries, and the next wait is longer. - assert!(health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF)); - assert!(!health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 2)); - assert!(health.allow_resubscribe_at(&relay, start + RESUBSCRIBE_INITIAL_BACKOFF * 3)); - } - - #[test] - fn backoff_is_capped() { - let health = InboxHealth::at(T0); - let relay = relay_url("ws://relay.example"); - let mut now = Instant::now(); - - // Drive it well past the ceiling. - for _ in 0..20 { - assert!(health.allow_resubscribe_at(&relay, now)); - now += RESUBSCRIBE_MAX_BACKOFF * 2; - } - - assert_eq!( - health - .state - .lock() - .unwrap() - .backoff - .get(&relay) - .expect("state kept") - .delay, - RESUBSCRIBE_MAX_BACKOFF, - "a hostile relay must still be retried every {RESUBSCRIBE_MAX_BACKOFF:?}" - ); - } - - #[test] - fn backoff_is_per_relay() { - let health = InboxHealth::at(T0); - let hostile = relay_url("ws://hostile.example"); - let healthy = relay_url("ws://healthy.example"); - let now = Instant::now(); - - assert!(health.allow_resubscribe_at(&hostile, now)); - assert!(!health.allow_resubscribe_at(&hostile, now)); - // One misbehaving relay must not delay recovery on another. - assert!(health.allow_resubscribe_at(&healthy, now)); - } - - #[test] - fn an_acknowledgement_clears_the_pacing_for_the_next_failure() { - let health = InboxHealth::at(T0); - let relay = relay_url("ws://relay.example"); - let now = Instant::now(); - - assert!(health.allow_resubscribe_at(&relay, now)); - assert!(!health.allow_resubscribe_at(&relay, now)); - - assert!( - health.note_relay_acknowledged(&relay), - "clearing a live backoff entry is what marks a recovery" - ); - assert!( - health.allow_resubscribe_at(&relay, now), - "a relay that answered starts over: the next failure is a fresh one" - ); - - // The steady state has nothing to clear, so nothing to report either. - health.note_relay_acknowledged(&relay); - assert!(!health.note_relay_acknowledged(&relay)); - } - - #[test] - fn the_watchdog_cadence_recovers_promptly_and_only_then_tapers() { - // The point of sharing one budget: an audit every - // `INBOX_WATCHDOG_INTERVAL` must still re-subscribe a relay that - // simply lost the inbox, while a relay that refuses it converges on - // the advertised ceiling instead of drawing a REQ every 30 seconds - // forever. - let health = InboxHealth::at(T0); - let relay = relay_url("ws://hostile.example"); - let tick = Duration::from_secs(crate::scheduler::INBOX_WATCHDOG_INTERVAL); - let mut now = Instant::now(); - - assert!( - health.allow_resubscribe_at(&relay, now), - "the pass that first notices the loss must act on it" - ); - for pass in 1..=3 { - now += tick; - assert!( - health.allow_resubscribe_at(&relay, now), - "pass {pass}: a delay still under the audit interval must not skip a retry" - ); - } - - // Once the doubling outgrows the interval, passes start being skipped. - let mut attempts = 0; - for _ in 0..40 { - now += tick; - if health.allow_resubscribe_at(&relay, now) { - attempts += 1; - } - } - assert!( - attempts < 40, - "a relay that keeps refusing must stop drawing a REQ on every pass" - ); - assert_eq!( - health - .state - .lock() - .unwrap() - .backoff - .get(&relay) - .expect("state kept") - .delay, - RESUBSCRIBE_MAX_BACKOFF - ); - } - // ───────────────────────── control-plane handling ───────────────────────── #[tokio::test] @@ -1069,7 +535,7 @@ mod tests { let relay = relay_url("ws://relay.example"); assert!(health.allow_resubscribe(&relay)); - assert!(backing_off(&health, &relay)); + assert!(health.is_backing_off(&relay)); let eose = RelayMessage::EndOfStoredEvents(std::borrow::Cow::Owned( keeper.subscription.id().clone(), @@ -1077,7 +543,7 @@ mod tests { keeper.on_relay_message(&client, &relay, &eose).await; assert!( - !backing_off(&health, &relay), + !health.is_backing_off(&relay), "an accepted REQ must reset the pacing for the next failure" ); } @@ -1129,7 +595,7 @@ mod tests { } assert!( - !backing_off(&health, &relay), + !health.is_backing_off(&relay), "a relay the SDK will re-REQ by itself must not be put on the shared backoff" ); assert!( @@ -1216,7 +682,7 @@ mod tests { keeper.on_relay_message(&client, &url, &closed).await; assert!( - backing_off(&health, &url), + health.is_backing_off(&url), "a CLOSED the SDK removes the subscription for must still arm the keeper" ); assert!( @@ -1242,7 +708,7 @@ mod tests { keeper.on_relay_message(&client, &relay, &other).await; assert!( - !backing_off(&health, &relay), + !health.is_backing_off(&relay), "a CLOSED for another subscription must not be treated as an inbox failure" ); } @@ -1433,275 +899,6 @@ mod tests { relay.shutdown(); } - // ───────────────────────────── health record ───────────────────────────── - - /// Health observations are wall-clock based, so tests drive a fixed origin - /// rather than the real clock. - const T0: i64 = 1_700_000_000; - - #[test] - fn health_records_an_outage_from_first_blindness_to_recovery() { - let health = InboxHealth::at(T0); - - assert!(!health.is_blind(), "a fresh record starts out listening"); - - health.observe(InboxStatus::Blind, T0); - assert!(health.is_blind()); - - // Staying blind must not restart the clock — the outage began at the - // first observation, and that is what an order is owed. - health.observe(InboxStatus::Blind, T0 + 30); - assert!(health.is_blind()); - - health.observe(InboxStatus::Listening, T0 + 90); - assert!(!health.is_blind()); - - assert_eq!( - health.blind_seconds_between(T0, T0 + 90), - 90, - "the recorded outage must span the whole blind window" - ); - } - - #[test] - fn health_is_not_listening_until_an_audit_says_so() { - let health = InboxHealth::at(T0); - - // Startup is not evidence. Between `main` subscribing and the - // watchdog's first pass, a node whose inbox never worked would - // otherwise process timeouts as if it had been listening all along. - assert!( - !health.is_confirmed_listening(), - "an unaudited record must not authorise acting on a user's silence" - ); - assert!( - !health.is_blind(), - "nor should it claim an outage it has not observed" - ); - - health.observe(InboxStatus::Listening, T0); - assert!(health.is_confirmed_listening()); - } - - #[test] - fn a_blind_first_audit_dates_the_outage_from_startup() { - let health = InboxHealth::at(T0); - - // The watchdog's first pass comes some time after boot. Finding the - // inbox deaf then means it was deaf for that whole stretch, not just - // from the moment somebody looked. - health.observe(InboxStatus::Blind, T0 + 30); - health.observe(InboxStatus::Listening, T0 + 90); - - assert_eq!( - health.blind_seconds_between(T0, T0 + 90), - 90, - "the outage must be dated from startup, not from the first audit" - ); - } - - #[test] - fn a_node_that_was_never_blind_owes_nothing() { - let health = InboxHealth::at(T0); - health.observe(InboxStatus::Listening, T0); - - assert_eq!(health.blind_seconds_between(T0, T0 + 10_000), 0); - assert_eq!(health.max_blind_seconds(), 0); - } - - // ──────────────────── what a single order is owed ──────────────────── - - #[test] - fn an_order_is_owed_only_the_downtime_it_waited_through() { - let health = InboxHealth::at(T0); - // One outage: [T0+100, T0+400], five minutes. - health.observe(InboxStatus::Listening, T0); - health.observe(InboxStatus::Blind, T0 + 100); - health.observe(InboxStatus::Listening, T0 + 400); - - let now = T0 + 1_000; - - // Waiting since before it started: owed the whole outage. - assert_eq!(health.blind_seconds_between(T0, now), 300); - // Taken midway through: owed only the remainder. - assert_eq!(health.blind_seconds_between(T0 + 250, now), 150); - // Taken after it ended: owed nothing. This is what a single global - // allowance got wrong — it credited orders that never lost a second. - assert_eq!(health.blind_seconds_between(T0 + 500, now), 0); - } - - #[test] - fn compensation_does_not_evaporate_as_time_passes() { - let health = InboxHealth::at(T0); - health.observe(InboxStatus::Listening, T0); - health.observe(InboxStatus::Blind, T0 + 100); - health.observe(InboxStatus::Listening, T0 + 400); - - // The debt an order carries is a property of when it waited, not of - // how long ago the outage was. A decaying allowance wore off at the - // same rate the deadline advanced, so it compensated almost nothing. - for probe in [400, 700, 5_000, 50_000] { - assert_eq!( - health.blind_seconds_between(T0, T0 + probe), - 300, - "an order waiting since T0 is owed the outage regardless of when we ask" - ); - } - } - - #[test] - fn an_order_waiting_through_an_outage_survives_its_nominal_deadline() { - // The regression in full: 900s timeout, an order taken at T0, and a - // 300s outage right at the start. Under the old decaying allowance - // this order was cancelled at ~T0+900, having had only 600s of - // listening time. - let health = InboxHealth::at(T0); - health.observe(InboxStatus::Blind, T0); - health.observe(InboxStatus::Listening, T0 + 300); - - let exp_seconds = 900i64; - let late_at = |now: i64| { - let owed = health.blind_seconds_between(T0, now); - (now - T0) >= exp_seconds + owed - }; - - assert!(!late_at(T0 + 900), "cancelled after only 600s of listening"); - assert!(!late_at(T0 + 1_199)); - assert!( - late_at(T0 + 1_200), - "and it must still expire once it has had its full 900s" - ); - } - - /// Regression: the credit is per order, so an order taken *after* an - /// outage ended must expire at its nominal deadline. - /// - /// The timeout job used to widen `find_order_by_seconds` by - /// [`InboxHealth::max_blind_seconds`], which narrows the selection rather - /// than widening it — every surviving row was already past - /// `deadline + max_blind_seconds`, the per-order check could never spare - /// anything, and what shipped was the global allowance this design - /// rejects. That allowance grows with every outage in the retention - /// window, so a node with flapping relays would postpone every deadline by - /// hours of unrelated downtime. - #[test] - fn an_order_taken_after_an_outage_is_not_credited_for_it() { - let health = InboxHealth::at(T0); - // One outage: [T0, T0+300]. - health.observe(InboxStatus::Blind, T0); - health.observe(InboxStatus::Listening, T0 + 300); - - let exp_seconds = 900i64; - let late_at = |taken_at: i64, now: i64| { - let owed = health.blind_seconds_between(taken_at, now); - (now - taken_at) >= exp_seconds + owed - }; - - // A: waited through the whole outage, owed all 300s. - assert!(!late_at(T0, T0 + 1_199)); - assert!(late_at(T0, T0 + 1_200)); - - // B: taken after recovery, owed nothing — even though the node's total - // downtime is the same 300s the global allowance would have handed it. - assert_eq!(health.max_blind_seconds(), 300); - assert!(!late_at(T0 + 400, T0 + 1_299)); - assert!( - late_at(T0 + 400, T0 + 1_300), - "an order that never lost a second must expire at its nominal deadline" - ); - } - - #[test] - fn consecutive_outages_accumulate_their_debt() { - let health = InboxHealth::at(T0); - health.observe(InboxStatus::Listening, T0); - health.observe(InboxStatus::Blind, T0 + 100); - health.observe(InboxStatus::Listening, T0 + 200); - health.observe(InboxStatus::Blind, T0 + 240); - health.observe(InboxStatus::Listening, T0 + 290); - - assert_eq!( - health.blind_seconds_between(T0, T0 + 1_000), - 150, - "an order waiting through both outages is owed both" - ); - assert_eq!( - health.blind_seconds_between(T0 + 210, T0 + 1_000), - 50, - "one taken between them is owed only the second" - ); - } - - #[test] - fn an_ongoing_outage_counts_up_to_now() { - let health = InboxHealth::at(T0); - health.observe(InboxStatus::Listening, T0); - health.observe(InboxStatus::Blind, T0 + 100); - - assert_eq!(health.blind_seconds_between(T0, T0 + 400), 300); - assert_eq!(health.blind_seconds_between(T0, T0 + 900), 800); - } - - #[test] - fn stale_windows_are_pruned() { - let health = InboxHealth::at(T0); - health.observe(InboxStatus::Listening, T0); - health.observe(InboxStatus::Blind, T0 + 100); - health.observe(InboxStatus::Listening, T0 + 200); - - // Far past the retention horizon, the old window is dropped rather - // than accumulating for the life of the process. - let much_later = T0 + BLIND_WINDOW_RETENTION_SECS + 1_000; - health.observe(InboxStatus::Listening, much_later); - - assert_eq!(health.blind_seconds_between(T0, much_later), 0); - assert!(health.state.lock().expect("lock").windows.is_empty()); - } - - #[test] - fn unconfirmed_time_counts_from_the_outage_or_from_startup() { - // What bounds how long the timeout job may defer. It has to answer - // even when no audit ever ran, or a watchdog that died would park the - // job on a verdict that is never coming. - let never_audited = InboxHealth::at(now_secs() - 120); - assert!( - never_audited.unconfirmed_for_secs() >= 120, - "with no verdict at all, the clock runs from startup" - ); - - let healthy = InboxHealth::at(now_secs()); - healthy.observe(InboxStatus::Listening, now_secs()); - assert_eq!( - healthy.unconfirmed_for_secs(), - 0, - "a confirmed inbox owes no waiting" - ); - - let blind = InboxHealth::at(now_secs() - 600); - blind.observe(InboxStatus::Listening, now_secs() - 600); - blind.observe(InboxStatus::Blind, now_secs() - 300); - assert!( - (300..=310).contains(&blind.unconfirmed_for_secs()), - "while blind it runs from the start of the outage, got {}", - blind.unconfirmed_for_secs() - ); - } - - #[test] - fn health_ignores_repeated_healthy_observations() { - let health = InboxHealth::at(T0); - - health.observe(InboxStatus::Listening, T0); - health.observe(InboxStatus::Listening, T0 + 30); - - assert!(!health.is_blind()); - assert_eq!( - health.blind_seconds_between(T0, T0 + 30), - 0, - "a node that was never blind has no outage to compensate for" - ); - } - // ──────────────────────────────── watchdog ──────────────────────────────── #[tokio::test] @@ -1796,30 +993,6 @@ mod tests { relay.shutdown(); } - #[test] - fn an_acknowledgement_does_not_survive_the_connection_it_was_earned_on() { - // A websocket drop and reconnect leaves no trace the keeper can act - // on: there is no relay-status `ClientNotification` in nostr-sdk - // 0.45.1, and the SDK silently re-sends the REQ by itself - // (`should_resubscribe`). If the relay then ignores that replacement, - // the only thing standing between a deaf node and resumed slashing is - // the acknowledgement expiring with its session. - let health = InboxHealth::at(T0); - let url = relay_url("ws://relay.example"); - - health.note_relay_acknowledged_at(&url, T0 + 100); - - assert!(health.has_acknowledged_since(&url, T0 + 50)); - assert!( - health.has_acknowledged_since(&url, T0 + 100), - "an EOSE landing in the same second as the connect must still count" - ); - assert!( - !health.has_acknowledged_since(&url, T0 + 101), - "credit earned on a previous connection must not vouch for this one" - ); - } - #[tokio::test] async fn watchdog_does_not_trust_an_acknowledgement_from_a_previous_connection() { use nostr_sdk::local_relay::LocalRelay; @@ -1990,7 +1163,7 @@ mod tests { // instead of being handed a REQ on every pass, forever. Skipping the // retry must not soften the verdict: the node is still deaf here. assert!( - backing_off(&health, &url), + health.is_backing_off(&url), "repeated refusals must accumulate on the shared re-subscribe budget" ); assert!(!health.allow_resubscribe(&url)); From fc3df0705b67111fb270bcf3e19932b9a662d7c0 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Fri, 28 Aug 2026 21:07:58 -0300 Subject: [PATCH 22/25] fix(scheduler): bound the per-order downtime credit to one expiration window Outage windows are retained for days, so an inbox that flaps could accrue credit faster than the clock runs it down and an order in a waiting state would never time out, holding its escrow to CLTV expiry with the taker's bond locked. Cap the credit at one expiration window, give no credit to orders whose taken_at was never persisted, and document both the bound and the fact that blameless-unwind notifications go out through the same relays that were down. --- docs/EVENT_ROUTING.md | 6 ++- src/inbox/mod.rs | 7 ++++ src/scheduler.rs | 87 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 19ea21c1..8cfaa4d8 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -41,7 +41,11 @@ Once the inbox recovers, each order is credited the downtime **it** waited throu The credit has to be per order rather than one global allowance: a single figure either under-credits an order that waited through the whole outage or hands the same credit to one taken long afterwards. -Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped, since by then every waiting order would be owed more than its deadline and nothing would ever be unwound. A failed release keeps the order in its waiting state so the next tick retries, exactly as the slashing path does: cancelling first would take the order out of `find_order_by_seconds`'s eligibility window with the bond still `Locked` and nothing left to look at it again. +The credit is also bounded, to one expiration window per order (`src/scheduler.rs`, `fn downtime_credit`). Outage windows are retained for days, so an inbox that flaps — blind long enough to keep accruing windows, listening just often enough to keep the tick running — would otherwise accrue credit faster than the clock runs it down and its orders would never time out, holding escrows to CLTV expiry: the state the three-hour bound below exists to prevent, reached through a path its continuous-stretch measure never sees. An order whose `taken_at` was never persisted (a value of zero) receives no credit: with no anchor there is no wait to intersect the windows with. + +Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped: the unwind is already blameless, so deferring it further would only keep escrows encumbered for longer. A failed release keeps the order in its waiting state so the next tick retries, exactly as the slashing path does: cancelling first would take the order out of `find_order_by_seconds`'s eligibility window with the bond still `Locked` and nothing left to look at it again. + +The unwind's notifications share the outage. The cancellation and republish messages go out through the same relays that stopped answering, so after a blameless unwind users may not receive them and will discover the outcome only by refreshing the order book once the relays are back. This is inherent — there is no second channel — but an operator recovering a node should expect a wave of "my order disappeared" reports rather than assume the messages were delivered. ## Dispatch - Router: `src/app.rs:handle_message_action` diff --git a/src/inbox/mod.rs b/src/inbox/mod.rs index bf2b6940..3fa623db 100644 --- a/src/inbox/mod.rs +++ b/src/inbox/mod.rs @@ -180,6 +180,13 @@ impl InboxKeeper { /// confirming it accepted the REQ and is the signal used to clear the /// backoff. Everything else (`OK`, `NOTICE`, other subscriptions' frames) /// is not this module's business. + /// + /// Awaiting this inline in the event loop is safe: the re-subscribe + /// bottoms out in `send_client_msg`, a `try_send` onto the relay's + /// transport channel with `wait_until_sent: None` (nostr-sdk 0.45.1, + /// `relay/inner.rs`) — no network round-trip, no blocking send, only + /// short locks on the subscription map — so a slow or dead relay cannot + /// stall the loop that every trade message flows through. pub async fn on_relay_message( &self, client: &Client, diff --git a/src/scheduler.rs b/src/scheduler.rs index e4dc28bf..530b69d7 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -534,6 +534,33 @@ async fn reconfirm_timeout_eligibility( (still_waiting && still_expired).then_some(fresh) } +/// Downtime credit for one order, bounded to one expiration window. +/// +/// `blind_overlap` is the retained inbox downtime overlapping the order's +/// wait (`InboxHealth::blind_seconds_since(taken_at)`), and it is what the +/// deadline is deferred by — but not verbatim. Windows are retained for +/// days, so an inbox that flaps — blind long enough to keep accruing +/// windows, listening just often enough for `is_confirmed_listening` to +/// keep this tick running — accrues credit faster than the clock runs it +/// down, and the order never times out: the hold invoice stays encumbered +/// until CLTV expiry and the taker's bond stays `Locked`. That is the state +/// `MAX_UNCONFIRMED_INBOX_PAUSE_SECS` exists to prevent, reached through a +/// path its continuous-stretch measure never sees, so the credit itself has +/// to carry a bound. One full window is the cap: past it the error is a +/// hastened cancellation, which stops costing anything once the user +/// re-sends, while the uncapped error is an escrow nothing ever unwinds. +/// +/// An order with no real anchor (`taken_at <= 0`; pre-trade CAS writes have +/// been seen to drop the field, see #866) gets no credit: there is no wait +/// to intersect the windows with, and its computed age is decades long, +/// beyond any bounded credit anyway. +fn downtime_credit(taken_at: i64, exp_seconds: u32, blind_overlap: i64) -> i64 { + if taken_at <= 0 { + return 0; + } + blind_overlap.min(exp_seconds as i64) +} + async fn job_cancel_orders(ctx: AppContext) { info!("Create a pool to connect to db"); @@ -597,22 +624,26 @@ async fn job_cancel_orders(ctx: AppContext) { // the only place it is applied. `find_order_by_seconds` selects on // the nominal deadline alone — deliberately over-selecting — and // the exact figure, the downtime that overlaps *this* order's own - // wait, decides below. A single global allowance cannot do this: + // wait (bounded to one expiration window, see `downtime_credit`), + // decides below. A single global allowance cannot do this: // it would either under-credit an order that waited through the // whole outage or hand the same credit to one taken long after it // ended. Widening the query by the largest outage seen would do // the latter, and narrowing it would put the rows this credit is // meant to spare out of reach entirely. // - // The credit is skipped once the pause bound is passed. By then - // the outage is hours deep, so every waiting order would be owed - // more than its deadline and none would ever be unwound — which is - // the state this branch exists to escape. Nobody is punished for - // it: `blameless` releases the bonds instead of settling them. + // The credit is skipped once the pause bound is passed. The + // unwind past the bound is blameless — bonds are released rather + // than settled, so nobody is punished — and deferring it any + // further would only keep escrows encumbered, which is the state + // this branch exists to escape. + // + // Capped like the per-order credit below, so the figure the + // operator reads is the one an order can actually receive. let max_grace = health .as_ref() .filter(|_| !blameless) - .map(|h| h.max_blind_seconds()) + .map(|h| h.max_blind_seconds().min(exp_seconds as i64)) .unwrap_or(0); if max_grace > 0 { info!( @@ -637,7 +668,11 @@ async fn job_cancel_orders(ctx: AppContext) { // through. Orders taken after the outage are owed nothing // and fall through unchanged. if let Some(health) = health.as_ref().filter(|_| !blameless) { - let owed = health.blind_seconds_since(order.taken_at); + let owed = downtime_credit( + order.taken_at, + exp_seconds, + health.blind_seconds_since(order.taken_at), + ); let waited = nostr_sdk::prelude::Timestamp::now().as_secs() as i64 - order.taken_at; if waited < exp_seconds as i64 + owed { @@ -1707,6 +1742,42 @@ mod tests { .collect() } + // ── downtime_credit ────────────────────────────────────────────────── + + /// An outage shorter than the deadline is credited in full: the order + /// gets back exactly the downtime that overlapped its wait. + #[test] + fn downtime_credit_passes_a_real_outage_through_unchanged() { + assert_eq!(downtime_credit(1_700_000_000, 900, 300), 300); + assert_eq!(downtime_credit(1_700_000_000, 900, 900), 900); + } + + /// The flapping-inbox hazard: windows are retained for days, so their sum + /// can exceed any deadline while `is_confirmed_listening` keeps the tick + /// running. Uncapped, `waited < exp + owed` would hold on every tick and + /// the order would never unwind — hold invoice encumbered until CLTV + /// expiry, taker's bond `Locked`. The credit is bounded to one expiration + /// window so the deadline always stays reachable. + #[test] + fn downtime_credit_is_capped_at_one_expiration_window() { + let exp: u32 = 900; + let owed = downtime_credit(1_700_000_000, exp, 7 * 24 * 3600); + assert_eq!(owed, exp as i64); + // An order that has waited two full windows is late even against the + // largest credit the cap allows. + let waited = 2 * exp as i64; + assert!(waited >= exp as i64 + owed); + } + + /// An order whose `taken_at` was never persisted has no anchor to + /// intersect the outage windows with; it gets no credit rather than a + /// meaningless one. + #[test] + fn downtime_credit_gives_an_unanchored_order_nothing() { + assert_eq!(downtime_credit(0, 900, 300), 0); + assert_eq!(downtime_credit(-5, 900, 300), 0); + } + // ── reconfirm_timeout_eligibility ──────────────────────────────────── /// A waiting order whose duty clock is genuinely past the window stays From b1055b933c37fd6c278a0cef13e8b013c7787100 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Wed, 2 Sep 2026 21:55:47 -0300 Subject: [PATCH 23/25] fix(scheduler): cap the downtime credit at the inbox pause ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capping the per-order credit at one expiration window spared nobody once an outage ran to two: on the first tick after recovery an order has waited the whole outage, so `waited < exp + min(D, exp)` is false for every order in a waiting state once D >= 2 * exp. Each one was cancelled and its bond slashed for a silence that was the node's, and `blameless` did not catch it — it arms only while the inbox is unconfirmed, which recovery clears before the tick runs. The result was an inverted risk profile: a four-hour outage unwound blamelessly and cost nobody anything, while a forty-five minute one slashed everyone who had been waiting through it. Cap at MAX_UNCONFIRMED_INBOX_PAUSE_SECS instead. The flapping hazard the cap exists for stays just as bounded — worst case an escrow is held one expiration window past the ceiling, against a CLTV horizon of about twenty-two hours — and no user is charged for downtime they sat through. --- docs/EVENT_ROUTING.md | 4 +- src/scheduler.rs | 106 ++++++++++++++++++++++++++++++++---------- 2 files changed, 84 insertions(+), 26 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 8cfaa4d8..51b337d7 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -41,7 +41,9 @@ Once the inbox recovers, each order is credited the downtime **it** waited throu The credit has to be per order rather than one global allowance: a single figure either under-credits an order that waited through the whole outage or hands the same credit to one taken long afterwards. -The credit is also bounded, to one expiration window per order (`src/scheduler.rs`, `fn downtime_credit`). Outage windows are retained for days, so an inbox that flaps — blind long enough to keep accruing windows, listening just often enough to keep the tick running — would otherwise accrue credit faster than the clock runs it down and its orders would never time out, holding escrows to CLTV expiry: the state the three-hour bound below exists to prevent, reached through a path its continuous-stretch measure never sees. An order whose `taken_at` was never persisted (a value of zero) receives no credit: with no anchor there is no wait to intersect the windows with. +The credit is also bounded, by the same three-hour ceiling that bounds the timeout pause below (`src/scheduler.rs`, `fn downtime_credit`). Outage windows are retained for days, so an inbox that flaps — blind long enough to keep accruing windows, listening just often enough to keep the tick running — would otherwise accrue credit faster than the clock runs it down and its orders would never time out, holding escrows to CLTV expiry: the state the three-hour bound exists to prevent, reached through a path its continuous-stretch measure never sees. An order whose `taken_at` was never persisted (a value of zero) receives no credit: with no anchor there is no wait to intersect the windows with. + +The ceiling is the tightest bound that is safe. One expiration window is the tempting cap and is the wrong one: an order is spared only while its wait is shorter than the deadline plus its credit, and on the first tick after an outage its wait already includes the whole outage — so a cap of one window spares nobody once the outage runs to two, and every order still waiting is cancelled and its bond slashed for a silence that was the node's. The blameless path does not catch that case either, because it arms only while the inbox is unconfirmed and recovery clears that before the tick runs. Capping at three hours instead bounds the flapping hazard just as firmly — an escrow is held at worst one expiration window past the ceiling, against a CLTV horizon of about twenty-two hours — without charging a user for downtime they sat through. Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped: the unwind is already blameless, so deferring it further would only keep escrows encumbered for longer. A failed release keeps the order in its waiting state so the next tick retries, exactly as the slashing path does: cancelling first would take the order out of `find_order_by_seconds`'s eligibility window with the bond still `Locked` and nothing left to look at it again. diff --git a/src/scheduler.rs b/src/scheduler.rs index 530b69d7..bcb39392 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -534,7 +534,7 @@ async fn reconfirm_timeout_eligibility( (still_waiting && still_expired).then_some(fresh) } -/// Downtime credit for one order, bounded to one expiration window. +/// Downtime credit for one order, bounded by the inbox pause ceiling. /// /// `blind_overlap` is the retained inbox downtime overlapping the order's /// wait (`InboxHealth::blind_seconds_since(taken_at)`), and it is what the @@ -546,19 +546,35 @@ async fn reconfirm_timeout_eligibility( /// until CLTV expiry and the taker's bond stays `Locked`. That is the state /// `MAX_UNCONFIRMED_INBOX_PAUSE_SECS` exists to prevent, reached through a /// path its continuous-stretch measure never sees, so the credit itself has -/// to carry a bound. One full window is the cap: past it the error is a -/// hastened cancellation, which stops costing anything once the user -/// re-sends, while the uncapped error is an escrow nothing ever unwinds. +/// to carry a bound. +/// +/// The bound is that same ceiling, and it cannot be tighter. One expiration +/// window looks like the natural cap and is the wrong one: an order is +/// spared only while `waited < exp + min(D, cap)`, and on the first tick +/// after an outage of length `D` it has waited `pre + D`, where `pre` is +/// what it had already waited when the outage began. A cap of `exp` +/// therefore spares nobody once `D >= 2 * exp` — every order in +/// `waiting-buyer-invoice` or `waiting-payment` is cancelled and the +/// responsible bond slashed, for a silence that was the node's. `blameless` +/// does not catch it: it arms only while the inbox is unconfirmed, and +/// recovery clears that before the tick runs. The risk profile ends up +/// inverted — a four-hour outage unwinds blamelessly and costs nobody +/// anything, while a forty-five minute one slashes everyone who was waiting +/// through it. Capping at `MAX_UNCONFIRMED_INBOX_PAUSE_SECS` keeps the +/// flapping hazard just as bounded — worst case an escrow is held for one +/// expiration window past the ceiling, against a CLTV horizon of about +/// twenty-two hours — without ever charging a user for downtime they sat +/// through. /// /// An order with no real anchor (`taken_at <= 0`; pre-trade CAS writes have /// been seen to drop the field, see #866) gets no credit: there is no wait /// to intersect the windows with, and its computed age is decades long, /// beyond any bounded credit anyway. -fn downtime_credit(taken_at: i64, exp_seconds: u32, blind_overlap: i64) -> i64 { +fn downtime_credit(taken_at: i64, blind_overlap: i64) -> i64 { if taken_at <= 0 { return 0; } - blind_overlap.min(exp_seconds as i64) + blind_overlap.min(MAX_UNCONFIRMED_INBOX_PAUSE_SECS) } async fn job_cancel_orders(ctx: AppContext) { @@ -624,7 +640,7 @@ async fn job_cancel_orders(ctx: AppContext) { // the only place it is applied. `find_order_by_seconds` selects on // the nominal deadline alone — deliberately over-selecting — and // the exact figure, the downtime that overlaps *this* order's own - // wait (bounded to one expiration window, see `downtime_credit`), + // wait (bounded by the pause ceiling, see `downtime_credit`), // decides below. A single global allowance cannot do this: // it would either under-credit an order that waited through the // whole outage or hand the same credit to one taken long after it @@ -643,7 +659,7 @@ async fn job_cancel_orders(ctx: AppContext) { let max_grace = health .as_ref() .filter(|_| !blameless) - .map(|h| h.max_blind_seconds().min(exp_seconds as i64)) + .map(|h| h.max_blind_seconds().min(MAX_UNCONFIRMED_INBOX_PAUSE_SECS)) .unwrap_or(0); if max_grace > 0 { info!( @@ -670,7 +686,6 @@ async fn job_cancel_orders(ctx: AppContext) { if let Some(health) = health.as_ref().filter(|_| !blameless) { let owed = downtime_credit( order.taken_at, - exp_seconds, health.blind_seconds_since(order.taken_at), ); let waited = @@ -1744,29 +1759,70 @@ mod tests { // ── downtime_credit ────────────────────────────────────────────────── - /// An outage shorter than the deadline is credited in full: the order - /// gets back exactly the downtime that overlapped its wait. + /// An outage the credit is not asked to bound is passed through in full: + /// the order gets back exactly the downtime that overlapped its wait. #[test] fn downtime_credit_passes_a_real_outage_through_unchanged() { - assert_eq!(downtime_credit(1_700_000_000, 900, 300), 300); - assert_eq!(downtime_credit(1_700_000_000, 900, 900), 900); + assert_eq!(downtime_credit(1_700_000_000, 300), 300); + assert_eq!(downtime_credit(1_700_000_000, 900), 900); + assert_eq!( + downtime_credit(1_700_000_000, MAX_UNCONFIRMED_INBOX_PAUSE_SECS), + MAX_UNCONFIRMED_INBOX_PAUSE_SECS + ); } /// The flapping-inbox hazard: windows are retained for days, so their sum /// can exceed any deadline while `is_confirmed_listening` keeps the tick /// running. Uncapped, `waited < exp + owed` would hold on every tick and /// the order would never unwind — hold invoice encumbered until CLTV - /// expiry, taker's bond `Locked`. The credit is bounded to one expiration - /// window so the deadline always stays reachable. + /// expiry, taker's bond `Locked`. The credit is bounded by the ceiling + /// that already bounds the timeout pause, so the deadline always stays + /// reachable. #[test] - fn downtime_credit_is_capped_at_one_expiration_window() { - let exp: u32 = 900; - let owed = downtime_credit(1_700_000_000, exp, 7 * 24 * 3600); - assert_eq!(owed, exp as i64); - // An order that has waited two full windows is late even against the - // largest credit the cap allows. - let waited = 2 * exp as i64; - assert!(waited >= exp as i64 + owed); + fn downtime_credit_is_capped_at_the_inbox_pause_ceiling() { + let owed = downtime_credit(1_700_000_000, 7 * 24 * 3600); + assert_eq!(owed, MAX_UNCONFIRMED_INBOX_PAUSE_SECS); + // An order still waiting one full window past the ceiling is late + // even against the largest credit the cap allows. + let exp: i64 = 900; + let waited = exp + MAX_UNCONFIRMED_INBOX_PAUSE_SECS; + assert!(waited >= exp + owed); + } + + /// The harm a tighter cap caused: an outage of two full expiration + /// windows, an order taken just as it began. On the first tick after + /// recovery the order has waited the whole outage, so capping the credit + /// at one window would have answered 1800s of enforced silence with 900s + /// of credit — cancelling the order and slashing the responsible bond for + /// a wait that was entirely the node's deafness. Every second of it is + /// owed back. + #[test] + fn an_outage_of_two_windows_spares_the_order_that_sat_through_it() { + let exp: i64 = 900; + let outage = 2 * exp; + let owed = downtime_credit(1_700_000_000, outage); + // Taken as the outage began, so its entire wait is downtime. + let waited = outage; + assert!( + waited < exp + owed, + "an order whose whole wait was inbox downtime must not be cancelled: \ + waited {waited}s against a deadline of {exp}s plus {owed}s of credit" + ); + } + + /// The bound still has to bite: once an order has waited its window on + /// top of the longest credit the cap allows, it is genuinely late and + /// unwinds normally. This is the property the cap exists for — the + /// deadline must stay reachable no matter how much downtime accrued. + #[test] + fn the_cap_keeps_the_deadline_reachable_after_the_longest_outage() { + let exp: i64 = 900; + let owed = downtime_credit(1_700_000_000, 30 * 24 * 3600); + let waited = exp + MAX_UNCONFIRMED_INBOX_PAUSE_SECS + 1; + assert!( + waited >= exp + owed, + "no amount of accrued downtime may put the deadline out of reach" + ); } /// An order whose `taken_at` was never persisted has no anchor to @@ -1774,8 +1830,8 @@ mod tests { /// meaningless one. #[test] fn downtime_credit_gives_an_unanchored_order_nothing() { - assert_eq!(downtime_credit(0, 900, 300), 0); - assert_eq!(downtime_credit(-5, 900, 300), 0); + assert_eq!(downtime_credit(0, 300), 0); + assert_eq!(downtime_credit(-5, 300), 0); } // ── reconfirm_timeout_eligibility ──────────────────────────────────── From afcd6c6596b38490ea15e1d265182e8c83f59c3c Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Wed, 2 Sep 2026 21:56:01 -0300 Subject: [PATCH 24/25] refactor(app): run both event loops from one implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run` and `run_cashu` differed in a single call — the dispatcher — and carried sixty duplicated lines of transport gating, subscription lifecycle and ClientNotification handling between them, which the inbox work had just widened. Fold them into `event_loop`, parameterised by a `Dispatcher` enum that carries the LND connection in Lightning mode. --- src/app.rs | 131 +++++++++++++++++++++-------------------------------- 1 file changed, 51 insertions(+), 80 deletions(-) diff --git a/src/app.rs b/src/app.rs index a0d07971..d67ed6bb 100644 --- a/src/app.rs +++ b/src/app.rs @@ -484,14 +484,26 @@ fn gate_for(is_v2: bool) -> Option<&'static SpamGate> { } } -/// Main event loop that processes incoming Nostr events. -/// Handles message verification, POW checking, and routes valid messages to appropriate handlers. +/// Which dispatcher a running [`event_loop`] hands a validated action to. /// -/// # Arguments -/// * `my_keys` - The node's keypair -/// * `client` - Nostr client instance -/// * `ln_client` - Lightning network connector -pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { +/// The two modes share everything else — transport gate, POW and spam +/// pre-validation, the inbox subscription lifecycle, the relay control plane — +/// and differ in exactly one call, so they run the same loop rather than two +/// copies of it that drift apart. +enum Dispatcher<'a> { + /// Lightning mode: the full order lifecycle, against an LND connection. + Lightning(&'a mut LndConnector), + /// Cashu mode (CF-5): no LND, so escrow actions are rejected. See + /// [`dispatch_cashu`]. + Cashu, +} + +/// The daemon's event loop: read the Nostr notification stream, validate +/// every incoming event, and route what survives to `dispatcher`. +/// +/// Handles message verification, POW checking, the inbox's control plane, and +/// re-attaching to the stream if it ends without a shutdown. +async fn event_loop(ctx: AppContext, mut dispatcher: Dispatcher<'_>) -> Result<()> { let my_keys = ctx.keys(); let client = ctx.nostr_client(); let pow = ctx.settings().mostro.pow; @@ -543,15 +555,23 @@ pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { else { continue; }; - let result = handle_message_action( - &action, - message.clone(), - &unwrapped, - my_keys, - ln_client, - &ctx, - ) - .await; + let result = match &mut dispatcher { + Dispatcher::Lightning(ln_client) => { + handle_message_action( + &action, + message.clone(), + &unwrapped, + my_keys, + ln_client, + &ctx, + ) + .await + } + Dispatcher::Cashu => { + dispatch_cashu(&action, message.clone(), &unwrapped, my_keys, &ctx) + .await + } + }; finalize_dispatch(result, message, unwrapped, &action).await; } ClientNotification::Message { relay_url, message } => { @@ -575,75 +595,26 @@ pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { } } -/// Cashu-mode event loop (CF-5). Mirrors [`run`]'s transport/validation -/// pipeline through the shared [`accept_event`]/[`finalize_dispatch`] helpers, -/// but dispatches through [`dispatch_cashu`] instead of -/// [`handle_message_action`] — there is no `ln_client` in Cashu mode. It -/// differs from `run` in exactly one line: the dispatch call. +/// Main event loop that processes incoming Nostr events. +/// Handles message verification, POW checking, and routes valid messages to appropriate handlers. +/// +/// # Arguments +/// * `ctx` - The application context (keys, settings, pool, Nostr client) +/// * `ln_client` - Lightning network connector +pub async fn run(ctx: AppContext, ln_client: &mut LndConnector) -> Result<()> { + event_loop(ctx, Dispatcher::Lightning(ln_client)).await +} + +/// Cashu-mode event loop (CF-5). Mirrors [`run`] exactly — same transport and +/// validation pipeline, same inbox handling — but dispatches through +/// [`dispatch_cashu`] instead of [`handle_message_action`], because there is +/// no `ln_client` in Cashu mode. /// /// During the foundation milestone every escrow/trade action is rejected with /// `CantDo(InvalidAction)`; the feature tracks replace those arms one at a time /// (see `docs/cashu/01-fundamentals.md` §6 action-ownership matrix). pub async fn run_cashu(ctx: AppContext) -> Result<()> { - let my_keys = ctx.keys(); - let client = ctx.nostr_client(); - let pow = ctx.settings().mostro.pow; - #[allow(deprecated)] - let accepted_kind = ctx.settings().mostro.transport.event_kind(); - let pow_first_contact = ctx.settings().mostro.effective_pow_first_contact(); - let gate = gate_for(accepted_kind.as_u16() == crate::config::constants::DM_EVENT_KIND); - let subscription = InboxSubscription::new(my_keys.public_key(), accepted_kind); - let keeper = InboxKeeper::new(subscription.clone()); - let mut subscribed = false; - - loop { - let mut notifications = client.notifications(); - - // Subscribe only once the stream exists — see `run`. - if !subscribed { - subscription.subscribe(client).await?; - subscribed = true; - } - - while let Some(notification) = notifications.next().await { - match notification { - ClientNotification::Event { event, .. } => { - let Some((action, message, unwrapped)) = accept_event( - &ctx, - &event, - my_keys, - pow, - pow_first_contact, - accepted_kind, - gate, - ) - .await - else { - continue; - }; - let result = - dispatch_cashu(&action, message.clone(), &unwrapped, my_keys, &ctx).await; - finalize_dispatch(result, message, unwrapped, &action).await; - } - ClientNotification::Message { relay_url, message } => { - keeper.on_relay_message(client, &relay_url, &message).await; - } - ClientNotification::Shutdown => return Ok(()), - } - } - - // The stream ended without a `Shutdown` frame. That frame can be - // missed — the SDK's notification channel silently drops messages when - // the consumer falls behind — and after a shutdown `notifications()` - // hands back an empty stream, so re-taking it unconditionally spins - // this loop at full tilt. Leave when the client is done, and pace the - // retry otherwise. - if client.is_shutdown() { - return Ok(()); - } - tracing::warn!("Nostr notification stream ended without a shutdown; re-attaching"); - tokio::time::sleep(NOTIFICATION_STREAM_RETRY).await; - } + event_loop(ctx, Dispatcher::Cashu).await } /// Route a validated action in Cashu mode (CF-5). From 5e49aef6cd1e5d4f3f7c9fbf381154fa5072bb25 Mon Sep 17 00:00:00 2001 From: Andrea Diaz Correia Date: Wed, 2 Sep 2026 21:56:10 -0300 Subject: [PATCH 25/25] docs: state the watchdog hand-off, the restart gap and the expiry gating Three places where the behaviour was right and the write-up was not: - The keeper's stand-down on a provisional CLOSED is scoped to the frame, not permanent; the watchdog's re-REQ one interval later is the intended backstop rather than an override, since an AUTH round-trip resolves well inside that interval and an answered relay is never touched. - Outage windows are process-local, so an order that waited through an outage preceding a restart is credited nothing for it. - Only job_cancel_orders is gated on inbox health, and why that is safe for job_expire_pending_older_orders as it stands today (see #926). --- docs/EVENT_ROUTING.md | 4 ++++ src/inbox/mod.rs | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/docs/EVENT_ROUTING.md b/docs/EVENT_ROUTING.md index 51b337d7..2f31c415 100644 --- a/docs/EVENT_ROUTING.md +++ b/docs/EVENT_ROUTING.md @@ -45,8 +45,12 @@ The credit is also bounded, by the same three-hour ceiling that bounds the timeo The ceiling is the tightest bound that is safe. One expiration window is the tempting cap and is the wrong one: an order is spared only while its wait is shorter than the deadline plus its credit, and on the first tick after an outage its wait already includes the whole outage — so a cap of one window spares nobody once the outage runs to two, and every order still waiting is cancelled and its bond slashed for a silence that was the node's. The blameless path does not catch that case either, because it arms only while the inbox is unconfirmed and recovery clears that before the tick runs. Capping at three hours instead bounds the flapping hazard just as firmly — an escrow is held at worst one expiration window past the ceiling, against a CLTV horizon of about twenty-two hours — without charging a user for downtime they sat through. +Outage history does not survive a restart. `InboxHealth` keeps its windows in memory, so an order that waited through an outage preceding a crash, deploy or restart is credited nothing for it once the daemon comes back: the per-order fairness described here holds within one process lifetime. Recording the gap across restarts needs persistence and a rule for how much a single restart may claim — a node deliberately offline for a week must not reopen as a week-long outage — and is tracked as follow-up work rather than solved here. + Deferring cannot be unconditional, though. The same pass that slashes a bond is the one that releases it and the one that cancels the seller's hold invoice, so waiting forever on a permanently broken inbox would leave escrows encumbered until CLTV expiry and honest takers' bonds locked indefinitely. After three hours without a confirmed inbox, timed-out orders are unwound anyway — but blamelessly: bonds are released rather than settled (`bond::release_on_timeout_without_slashing`), and the downtime credit is skipped: the unwind is already blameless, so deferring it further would only keep escrows encumbered for longer. A failed release keeps the order in its waiting state so the next tick retries, exactly as the slashing path does: cancelling first would take the order out of `find_order_by_seconds`'s eligibility window with the bond still `Locked` and nothing left to look at it again. +Only `job_cancel_orders` is gated on inbox health. `job_expire_pending_older_orders` runs throughout, and that is correct only because of what it does: it expires orders that were never taken and releases the bonds it touches, so the worst an outage costs there is a maker who has to republish. It takes no slash decision of its own — the one bond it can settle, a range maker bond at close, is carrying out a slash an earlier slice already decided. That is a property of the job as it stands today, not a licence: any future path there that settles a bond on a user's silence needs the same `is_confirmed_listening` gate this one has. A `TakeSell` lost to a blind inbox still expires an order unfairly, which is tracked separately (issue #926). + The unwind's notifications share the outage. The cancellation and republish messages go out through the same relays that stopped answering, so after a blameless unwind users may not receive them and will discover the outcome only by refreshing the order book once the relays are back. This is inherent — there is no second channel — but an operator recovering a node should expect a wave of "my order disappeared" reports rather than assume the messages were delivered. ## Dispatch diff --git a/src/inbox/mod.rs b/src/inbox/mod.rs index 3fa623db..b78352b8 100644 --- a/src/inbox/mod.rs +++ b/src/inbox/mod.rs @@ -199,13 +199,28 @@ impl InboxKeeper { message, } if subscription_id.as_ref() == self.subscription.id() => { if is_provisional_closure(message) { - // The REQ is not the keeper's to re-send: the SDK only - // *marks* these two prefixes and re-sends it itself — - // after the NIP-42 round-trip for `auth-required`, on the - // next reconnect for `rate-limited`. Re-issuing it here - // would drop the entry the SDK is about to re-send, race - // its AUTH, and arm a backoff against a relay that is - // behaving exactly as the protocol says it should. + // The REQ is not the keeper's to re-send *on this frame*: + // the SDK only *marks* these two prefixes and re-sends it + // itself — after the NIP-42 round-trip for + // `auth-required`, on the next reconnect for + // `rate-limited`. Re-issuing it here, in the microseconds + // after the frame arrives, would drop the entry the SDK is + // about to re-send, cut across its AUTH, and arm a backoff + // against a relay that is behaving exactly as the protocol + // says it should. + // + // The stand-down is scoped to that window and no further. + // `check_inbox_health` will re-send the REQ at the next + // audit if the relay still has not answered, and that is + // deliberate rather than an override of this branch: an + // AUTH round-trip completes in well under + // `INBOX_WATCHDOG_INTERVAL`, so a relay still + // unacknowledged a full interval later is one the SDK's + // own recovery did not reach — the `rate-limited` and + // rejected-AUTH dead ends below. A relay that *did* answer + // is acknowledged and the audit never touches it, so the + // backstop costs a redundant REQ only in the case where + // standing down permanently would mean silent deafness. // // The *health verdict* is another matter, and must not // stand down with it. `MarkAsClosed` leaves the entry in @@ -369,6 +384,14 @@ async fn resubscribe_relay( /// it answers, which costs one interval before recovery is declared and keeps /// the error on the safe side: the timeout clock stays frozen slightly longer /// than strictly needed rather than restarting too early. +/// +/// The audit re-sends to every unacknowledged relay, including one +/// [`InboxKeeper`] stood down on after a provisional `CLOSED`. That is the +/// intended hand-off, not a bypass: the keeper stands down for the instant the +/// frame arrives, so it does not cut across the SDK's own AUTH round-trip, and +/// by the time an audit comes round that round-trip has either produced an +/// `EOSE` — in which case the relay is acknowledged and left alone — or it +/// never will, which is exactly when the REQ has to come from here. pub async fn check_inbox_health(client: &Client, subscription: &InboxSubscription) -> InboxStatus { check_inbox_health_with(client, subscription, InboxHealth::global()).await }