From d040086cfa675ba8e75110c5a6fe65953f6b34c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:07:35 -0800 Subject: [PATCH 01/19] chore(quest): claim quest/m0/nightly/credential-established.md Quest-Claim: 16edfb3c8237bf9b From cc336d2a972797d9f8e71e313aebbfa27ff8fd5d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:15:31 -0800 Subject: [PATCH 02/19] feat(net): a reconnecting client is established by the peer accepting its credential Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 57 ++++++++++++-- rs/moq-net/src/auth/tokens.rs | 16 ++-- rs/moq-net/src/ietf/peer.rs | 2 +- rs/moq-net/src/lite/session.rs | 8 +- rs/moq-net/src/lite/setup.rs | 2 +- rs/moq-net/src/reconnect.rs | 43 ++++++---- rs/moq-net/src/reconnect/tests.rs | 126 +++++++++++++++++++++++++----- rs/moq-net/src/session.rs | 7 ++ 8 files changed, 206 insertions(+), 55 deletions(-) diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 49f224b806b..ec3aa00804f 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -230,8 +230,22 @@ struct Attached { /// gates can skip re-matching paths on every wakeup. epoch: u64, /// Whether the peer has accepted the connection, credential included: at once, unless - /// the session [awaits the peer's SETUP](Link::await_setup). - accepted: bool, + /// the session [awaits its answer](Link::await_acceptance) or the version + /// [has none](Link::silent). + acceptance: Acceptance, +} + +/// Whether a session's peer accepted the connection credential. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Acceptance { + /// With its handshake, or with the answer the session awaited. + Accepted, + /// Not yet: the peer's SETUP (a client without AUTH) or its AUTH_OK to the connection's + /// own credential (a lite-07 client) is still to come, and the peer can still refuse. + Awaiting, + /// Never said: the version has no answer, so the peer refuses only by closing the + /// session (a lite-03/04 client). + Silent, } /// One token as one session presents it. @@ -322,7 +336,7 @@ impl Attached { issued: BTreeMap::new(), scope: None, epoch: 0, - accepted: true, + acceptance: Acceptance::Accepted, }; attached.rescope(limit); attached @@ -1117,6 +1131,10 @@ impl Link { if !matches!(slot.phase, Phase::Pending { .. } | Phase::Granted(_)) { return; } + // The answer to the connection's own credential accepts the session. + if slot.setup && link.acceptance == Acceptance::Awaiting { + link.acceptance = Acceptance::Accepted; + } slot.phase = Phase::Granted(grant); link.replied = true; link.recompute(); @@ -1409,21 +1427,44 @@ impl Link { self.read(|link, _| link.closed.clone()) } - /// The peer accepts the connection credential only with its SETUP, which this client - /// reads after the session is live, while the server can still refuse it. - pub(crate) fn await_setup(&self) { - self.with(|link| link.accepted = false); + /// The peer accepts the connection credential only with an answer this client reads after + /// the session is live, while the server can still refuse it: its SETUP on a version + /// without AUTH, or the AUTH_OK to the connection's own credential on one with it. + pub(crate) fn await_acceptance(&self) { + self.with(|link| link.acceptance = Acceptance::Awaiting); + } + + /// The version never answers the connection credential: the peer refuses it only by + /// closing the session, however soon. + pub(crate) fn silent(&self) { + self.with(|link| link.acceptance = Acceptance::Silent); } /// The peer's SETUP arrived, so it accepted the connection credential. pub(crate) fn accepted(&self) { self.with_state(|state, link| { - attached(&mut state.links, link).accepted = true; + let link = attached(&mut state.links, link); + if link.acceptance == Acceptance::Awaiting { + link.acceptance = Acceptance::Accepted; + } // Latch a set this acceptance completed, before a later end can replace its answer. state.record(); }); } + /// Ready once the peer said it accepts the connection credential, which a version without + /// that answer never does. A close afterward leaves it accepted. + pub(crate) fn poll_accepted(&self, waiter: &kio::Waiter) -> Poll<()> { + let id = self.id(); + let _ = ready_or!(self.state().poll(waiter, |state| { + match state.links.get(&id).map(|link| link.acceptance) { + Some(Acceptance::Accepted) => Poll::Ready(()), + _ => Poll::Pending, + } + })); + Poll::Ready(()) + } + /// End the session: fail every pending token here, leaving the handle's tokens for the /// next session. A handle the session created for itself ends with it. pub(crate) fn close(&self, err: Error) { diff --git a/rs/moq-net/src/auth/tokens.rs b/rs/moq-net/src/auth/tokens.rs index ca35fc6a287..fc05fafec60 100644 --- a/rs/moq-net/src/auth/tokens.rs +++ b/rs/moq-net/src/auth/tokens.rs @@ -229,7 +229,7 @@ pub(super) struct Configured { /// How the latest dial's session answers its connection credential. enum Connected { /// The version has no AUTH, so the peer accepted it with its handshake, or with its - /// SETUP when the session [awaits it](super::Link::await_setup). + /// SETUP when the session [awaits it](super::Link::await_acceptance). Accepted { link: u64 }, /// Its own AUTH stream on that session. Slot { link: u64, id: u64 }, @@ -252,12 +252,14 @@ impl Configured { Connected::Slot { link, id } => (link, id), Connected::Accepted { link } => { let link = state.links.get(link)?; - return match (link.accepted, &link.closed) { - (true, _) => Some(Ok(())), + let awaiting = link.acceptance == super::Acceptance::Awaiting; + return match (awaiting, &link.closed) { + // Accepted, or a version that never answers, whose live session counts as accepting it. + (false, _) => Some(Ok(())), // Closed before accepting it: the close is the answer, since the connection // keeps redialing with the same credential and nothing else would answer. - (false, Some(err)) => Some(Err(err.clone())), - (false, None) => None, + (true, Some(err)) => Some(Err(err.clone())), + (true, None) => None, }; } }; @@ -791,7 +793,7 @@ mod tests { let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); set.commit(&presented); let link = Link::attach(&set.handle, false, false).unwrap(); - link.await_setup(); + link.await_acceptance(); set.connect(link.id(), None); link } @@ -823,7 +825,7 @@ mod tests { let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); set.commit(&presented); let link = Link::attach(&set.handle, false, false).unwrap(); - link.await_setup(); + link.await_acceptance(); link.close(Error::Session(SessionError::Unauthorized)); set.connect(link.id(), None); refused(&set, SessionError::Unauthorized); diff --git a/rs/moq-net/src/ietf/peer.rs b/rs/moq-net/src/ietf/peer.rs index 7f4d60fd228..ce2ee2b55df 100644 --- a/rs/moq-net/src/ietf/peer.rs +++ b/rs/moq-net/src/ietf/peer.rs @@ -54,7 +54,7 @@ impl PeerSetup { /// Tell `link` once the peer's SETUP arrives that it accepted the connection credential. pub fn accepting(mut self, link: crate::auth::Link) -> Self { - link.await_setup(); + link.await_acceptance(); self.accept = Some(link); self } diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 1ce14ebc946..392c229d73d 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -328,9 +328,15 @@ pub fn start(config: Config) -> Result { .offer_ietf(our_setup.ietf) .with_implementation(peer_implementation.clone()); // Without AUTH, the server checks a client's credential before its SETUP, which the - // client reads only after going live. + // client reads only after going live. With AUTH, the AUTH_OK to the connection's own + // credential accepts it. Lite-03/04 have neither, and lite-01/02 answered in the + // negotiation SETUP already read. if client && version.has_setup_stream() && !version.has_auth() { peer_setup_slot = peer_setup_slot.accepting(auth.clone()); + } else if client && version.has_auth() { + auth.await_acceptance(); + } else if client && matches!(version, Version::Lite03 | Version::Lite04) { + auth.silent(); } // A version without a Setup Stream advertises nothing, so there is nothing to wait for. if !version.has_setup_stream() { diff --git a/rs/moq-net/src/lite/setup.rs b/rs/moq-net/src/lite/setup.rs index 4985e2a47d6..5ae9e25a319 100644 --- a/rs/moq-net/src/lite/setup.rs +++ b/rs/moq-net/src/lite/setup.rs @@ -557,7 +557,7 @@ impl PeerSetup { /// Tell `link` once the peer's SETUP arrives that it accepted the connection credential. pub fn accepting(self, link: crate::auth::Link) -> Self { - link.await_setup(); + link.await_acceptance(); self.0.lock().accept = Some(link); self } diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index f1e3cde9e8d..2aab59a6d44 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -8,8 +8,11 @@ //! predecessor serving through its handover, and drops neither on the caller's behalf //! until it reports [`Action::Done`]. //! -//! The policy fails fast until a session is established, which takes staying up past -//! [`Backoff::healthy`] or ending in a GOAWAY: a failed walk, or a session closed before then, ends the policy +//! The policy fails fast until a session is established: the peer accepts the connection +//! credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the connection's own +//! credential), drains the session with a GOAWAY, or the session stays up past +//! [`Backoff::healthy`], the only bar on moq-lite 03/04, which answer the credential with +//! nothing. A failed walk, or a session closed before then, ends the policy //! with [`Action::Done`] carrying that error rather than retrying, so a setup mistake //! surfaces at once instead of looping. That covers a peer that refuses the session right //! after connecting, as a relay rejecting a token on moq-lite 03 or later does. A client @@ -56,9 +59,9 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// When to redial after a failure, and when to give up, once a session has been established. /// -/// A session is established once it stays up past `healthy` or the peer drains it with a -/// GOAWAY. Before then nothing retries: a failed walk, or a session closed sooner, ends the -/// policy. +/// A session is established once the peer accepts the connection credential, drains the +/// session with a GOAWAY, or the session stays up past `healthy`. Before then nothing +/// retries: a failed walk, or a session closed sooner, ends the policy. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter, @@ -86,8 +89,9 @@ pub struct Backoff { pub max: Option, /// How long a session must stay up to count as healthy. Must be nonzero. Defaults to 1 s. /// - /// The first healthy session, or one the peer drains with a GOAWAY, establishes the - /// policy, so failures retry from then on. + /// The first healthy session establishes the policy, so failures retry from then on. That + /// bar matters only where the peer never answers the connection credential, as on moq-lite + /// 03/04: an accepting peer establishes it sooner. /// A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. @@ -377,8 +381,8 @@ pub struct Reconnect { phase: Phase, /// The failures of the walk under way. walk: Walk, - /// Whether any session stayed up past the healthy bar or ended in a GOAWAY. Until one has, - /// a failed walk or a closed session ends the policy. + /// Whether any session's peer accepted its credential, or one stayed up past the healthy + /// bar or ended in a GOAWAY. Until then, a failed walk or a closed session ends the policy. established: bool, /// An old session kept serving after a GOAWAY so its in-flight groups finish. predecessor: Option, @@ -594,6 +598,11 @@ impl + From> Reco deadline => return Action::Wait(earliest(deadline.as_ref().map(|(at, _)| *at), handover)), }, Phase::Live(live) => { + // Checked before the close, which may follow the peer's acceptance in one wake: + // a session the peer accepted and then dropped is an outage, not a refusal. + if !self.established && live.session.poll_accepted(waiter).is_ready() { + self.established = true; + } let mut ended = None; // Checked before the close: a GOAWAY means the session is still up, and // migrating from it is not the same as reconnecting after it died. @@ -905,9 +914,9 @@ impl + From> Reco fn walk_failed(&mut self, now: Instant, err: E, verdict: Verdict) { self.walk = Walk::default(); - // Otherwise a walk fails fast until a session was established: a peer that never held - // one past the healthy bar is more likely misconfigured (an unknown host, an untrusted - // certificate, no version in common) than restarting, and only the caller can fix that. + // Otherwise a walk fails fast until a session was established: a peer that never + // accepted one is more likely misconfigured (an unknown host, an untrusted certificate, + // no version in common) than restarting, and only the caller can fix that. // After that, every failure retries, whatever its kind. if verdict == Verdict::Unusable || self.config.once || !self.established { return self.done(Err(err)); @@ -1098,11 +1107,11 @@ impl + From> Reco if self.config.once { return self.done(Err(err)); } - // Until a session has stayed up past the healthy bar, a close ends the policy as a - // failed walk does. It is how a peer refuses a session it already accepted, such as a - // rejected credential on moq-lite 03 or later, which has no SETUP reply to refuse in; - // redialing would offer the same credential forever. No error kind is read, and as - // for a refused dial, tokens replaced since change nothing. + // Until a session was established, a close ends the policy as a failed walk does. It + // is how a peer refuses a session that is already live, such as a rejected credential + // on moq-lite 03 or later, closed before the peer's SETUP or AUTH_OK; redialing would + // offer the same credential forever. No error kind is read, and as for a refused dial, + // tokens replaced since change nothing. if !self.established { return self.done(Err(err)); } diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 987b39e81d1..a87d73bcbf8 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -28,6 +28,8 @@ enum Peer { Goaway(String), /// Accept and keep serving. Serve, + /// Accept and keep serving, but never answer the connection credential. + Withhold, /// Fail the dial, judged as this. Fail(Verdict), /// Never answer. @@ -60,24 +62,35 @@ fn version() -> crate::Version { /// Run an in-memory MoQ handshake, returning the client and server sessions. async fn handshake() -> (Session, Session) { + let (client, server, _) = handshake_with(version(), false).await; + (client, server) +} + +/// Run an in-memory MoQ handshake over `version`, returning the client and server sessions. +/// With `withhold`, the server answers the client's tokens itself and never does, returning +/// the queue that keeps them unanswered while it is held. +async fn handshake_with(version: crate::Version, withhold: bool) -> (Session, Session, Option) { let (client, server) = crate::transport::memory::pair(crate::transport::memory::Config { - protocol: Some(version().alpn().to_string()), + protocol: Some(version.alpn().to_string()), ..Default::default() }); - let dial = crate::Client::new().with_versions(version().into()); - let serve = crate::Server::new().with_versions(version().into()); - futures::join!( + let dial = crate::Client::new().with_versions(version.into()); + let serve = crate::Server::new().with_versions(version.into()); + let (client, (server, requests)) = futures::join!( async { let (session, driver) = dial.connect(kio::clock::now(), client).await.unwrap(); drop(kio::clock::sim::spawn(crate::time::run(driver))); session }, async { - let (session, driver) = serve.accept(kio::clock::now(), server).await.unwrap(); + let mut handshake = serve.handshake(kio::clock::now(), server).await.unwrap(); + let requests = withhold.then(|| handshake.requests()); + let (session, driver) = handshake.ok().await.unwrap(); drop(kio::clock::sim::spawn(crate::time::run(driver))); - session + (session, requests) }, - ) + ); + (client, server, requests) } /// The next action that is not a wait, sleeping on the simulated clock meanwhile. @@ -116,6 +129,8 @@ struct Run { events: Vec<&'static str>, /// The server sides still serving. servers: Vec, + /// The auth queues of the servers withholding their answer. + withheld: Vec, } /// Drive `machine` against `peers` until it ends, or until `stop` says so. @@ -158,8 +173,9 @@ async fn drive( match peer { Peer::Hang => {} Peer::Fail(verdict) => machine.failed(kio::clock::now(), TestError::Dial, *verdict), - Peer::Close | Peer::Goaway(_) | Peer::Serve => { - let (client, server) = handshake().await; + Peer::Close | Peer::Goaway(_) | Peer::Serve | Peer::Withhold => { + let (client, server, withheld) = handshake_with(version(), matches!(peer, Peer::Withhold)).await; + run.withheld.extend(withheld); run.accepts.push((url, kio::clock::now())); match peer { Peer::Close => drop(server), @@ -340,11 +356,12 @@ async fn the_first_walk_fails_fast() { assert_eq!(kio::clock::now(), started, "gave up without waiting"); } -/// A first session closed before it was healthy ends the policy with its close reason, -/// like a failed first walk: it is how a peer refuses a session it already accepted, such -/// as a rejected credential on moq-lite 03 or later. Tokens replaced since change nothing. +/// A first session closed before the peer accepted its credential ends the policy with its +/// close reason, like a failed first walk: it is how a peer refuses a session that is +/// already live, such as a rejected credential on moq-lite 03 or later. Tokens replaced +/// since change nothing. #[kio::test] -async fn a_first_session_closed_before_healthy_fails_fast() { +async fn a_first_session_closed_before_acceptance_fails_fast() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); let started = kio::clock::now(); @@ -352,7 +369,7 @@ async fn a_first_session_closed_before_healthy_fails_fast() { .unwrap() .with_tokens(tokens.clone()); let mut run = Run::default(); - drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { !run.accepts.is_empty() }) .await; @@ -369,10 +386,11 @@ async fn a_first_session_closed_before_healthy_fails_fast() { assert!(run.events.is_empty(), "retried: {:?}", run.events); } -/// A session that stayed up past the healthy bar establishes the policy: its close, and -/// the failed walk after it, back off and retry like any later failure. +/// The peer accepting the connection credential establishes the policy however soon the +/// session then drops: its close, and the failed walk after it, back off and retry like any +/// later failure. #[kio::test] -async fn a_healthy_session_establishes_the_policy() { +async fn an_accepted_session_establishes_the_policy() { let peer = url("tcp://peer:1/"); let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); @@ -381,6 +399,64 @@ async fn a_healthy_session_establishes_the_policy() { }) .await; + let accepted = kio::clock::now(); + kio::wait(|waiter| machine.session().expect("live").poll_accepted(waiter)).await; + assert!( + kio::clock::now() - accepted < Config::default().backoff.healthy / 10, + "the AUTH_OK came late" + ); + run.servers.remove(0).abort(Error::Unauthorized); + let ended = drive(&mut machine, &[(peer, Peer::Fail(Verdict::Retry))], &mut run, |run| { + run.dials.len() == 3 + }) + .await; + assert!(ended.is_none(), "an established policy gave up"); + assert_eq!(run.events, ["offline", "retrying"]); +} + +/// A version that never answers the connection credential (moq-lite-04) keeps the uptime +/// bar: a session closed before it is a refusal, and one that outlives it establishes. +#[kio::test] +async fn a_silent_version_is_established_by_uptime() { + let lite04: crate::Version = "moq-lite-04".parse().unwrap(); + for healthy in [false, true] { + let mut machine = Reconnect::::new( + config(INITIAL, None), + vec![url("tcp://peer:1/")], + kio::clock::now(), + ) + .unwrap(); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + let (client, server, _) = handshake_with(lite04, false).await; + machine.connected(kio::clock::now(), client); + + let healthy_bar = Config::default().backoff.healthy; + kio::clock::sleep(if healthy { healthy_bar } else { healthy_bar / 2 }).await; + assert!( + machine.session().expect("live").poll_accepted(&kio::Waiter::noop()).is_pending(), + "moq-lite-04 has no answer to accept with" + ); + server.abort(Error::Unauthorized); + match next(&mut machine).await { + Action::Offline(_) => assert!(healthy, "a refusal before the bar was retried"), + Action::Done(res) => assert!(!healthy && res.is_err(), "a healthy session gave up: {res:?}"), + _ => panic!("unexpected action"), + } + } +} + +/// A session that stayed up past the healthy bar establishes the policy even when the peer +/// never answered the credential: its close, and the failed walk after it, retry. +#[kio::test] +async fn a_healthy_session_establishes_the_policy() { + let peer = url("tcp://peer:1/"); + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + kio::clock::sleep(Config::default().backoff.healthy).await; run.servers.remove(0).abort(Error::Unauthorized); let ended = drive(&mut machine, &[(peer, Peer::Fail(Verdict::Retry))], &mut run, |run| { @@ -402,7 +478,8 @@ async fn a_goaway_before_healthy_establishes_the_policy() { config.restart = restart; let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); - drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + // Withheld, so only the GOAWAY can establish the policy before the relay exits. + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { !run.accepts.is_empty() }) .await; @@ -411,7 +488,7 @@ async fn a_goaway_before_healthy_establishes_the_policy() { // `Redial` follows it to a replacement on the draining relay; `Handoff` serves on, // since the session is not yet healthy. if restart == Restart::Redial { - drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { run.accepts.len() == 2 }) .await; @@ -1103,7 +1180,16 @@ async fn a_fallback_peer_redirect_establishes_the_policy() { let mut config = Config::default(); // Longer than the probe takes, so the fallback session is never healthy. config.backoff.healthy = Duration::from_secs(3600); - let (mut machine, fallback_server) = on_fallback(config).await; + // Nor accepted: its peer never answers the credential. + let peer = url("https://peer:1/"); + let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer, Peer::Withhold)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + machine.fallback(kio::clock::now()); + let fallback_server = run.servers.remove(0); probe(&mut machine).await; assert!(machine.probed(kio::clock::now(), Ok(()))); assert!(matches!(next(&mut machine).await, Action::Migrating)); diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 800bef31e6e..aec01812b9f 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -472,6 +472,13 @@ impl Session { self.credential.as_ref() } + /// Ready once the peer accepted the connection credential: with its SETUP, or on + /// moq-lite-07-wip the AUTH_OK to the connection's own credential. Ready at once when the + /// handshake already carried that answer, and never on moq-lite 03/04, which have none. + pub(crate) fn poll_accepted(&self, waiter: &kio::Waiter) -> Poll<()> { + self.link.poll_accepted(waiter) + } + /// This session's side of its [`auth`](Self::auth) handle. pub(crate) fn link(&self) -> &auth::Link { &self.link From 392cdb6d0240e83ebfc4111518ee9cc6cbb6ff09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:18:18 -0800 Subject: [PATCH 03/19] feat(js/net): Reload is established by the peer accepting its credential Co-Authored-By: Claude Opus 5.5 --- js/net/src/auth_session.ts | 35 ++++++++++++++--- js/net/src/connection/reload.test.ts | 59 +++++++++++++++++++++++----- js/net/src/connection/reload.ts | 44 +++++++++++++-------- js/net/src/lite/connection.ts | 5 ++- 4 files changed, 110 insertions(+), 33 deletions(-) diff --git a/js/net/src/auth_session.ts b/js/net/src/auth_session.ts index d4efcb17ada..927b4288736 100644 --- a/js/net/src/auth_session.ts +++ b/js/net/src/auth_session.ts @@ -620,6 +620,12 @@ export interface AuthSessionProps { * session without AUTH that is live before its peer has checked the credential. */ awaiting?: boolean; + /** + * Whether the version never answers the connection credential (a moq-lite-03/04 client), + * so the peer refuses it only by closing the session. Its credential still counts as + * accepted, but {@link AuthSession.accepted} never says so. + */ + silent?: boolean; } /** @@ -657,6 +663,8 @@ export class AuthSession { // The peer's verdict on the connection credential this session presented: null once accepted, // the refusal, or undefined while it has none. #credential = new Signal(undefined); + // Whether the peer said it accepts the connection credential, which a silent version never does. + #accepted = new Signal(false); #closed = false; // Whoever answers the peer's tokens is decided once, after the task that established @@ -665,7 +673,7 @@ export class AuthSession { #decided = yieldNow(); /** Throws {@link Duplicate} while `handle` backs another active session, or the handle's end. */ - constructor({ wire, requests, peerGrant, handle, draining, awaiting }: AuthSessionProps) { + constructor({ wire, requests, peerGrant, handle, draining, awaiting, silent }: AuthSessionProps) { this.handle = handle ?? ownHandle(); this.#state = stateOf(this.handle); this.#wire = wire; @@ -674,7 +682,10 @@ export class AuthSession { this.#draining = draining ?? (() => false); this.#state.attach(this); // A session without AUTH has the peer's acceptance with its handshake, unless it awaits its SETUP. - if (!wire && !awaiting) this.#credential.set(null); + if (!wire && !awaiting) { + this.#credential.set(null); + if (!silent) this.#accepted.set(true); + } // Present the connection's own credential right away, so both sides learn their // grant without waiting on the app. @@ -696,9 +707,20 @@ export class AuthSession { return this.#credential; } + /** + * Whether the peer said it accepts the connection credential: with its handshake, its SETUP, + * or the AUTH_OK to the connection's own credential (moq-lite-07). Never on a silent version. + * The session ending leaves it as it was. + */ + get accepted(): Getter { + return this.#accepted; + } + /** The peer's SETUP arrived, so it accepted the connection credential a session without AUTH presented. */ - accepted(): void { - if (!this.#wire) this.#credential.set(null); + accept(): void { + if (this.#wire) return; + this.#credential.set(null); + this.#accepted.set(true); } /** The union of this session's grants, undefined until its peer first answers. */ @@ -959,7 +981,10 @@ export class AuthSession { answered = granted = replied = true; delay = undefined; this.#replied = true; - if (presented.setup) this.#credential.set(null); + if (presented.setup) { + this.#credential.set(null); + this.#accepted.set(true); + } this.#answered(presented); this.#recompute(); continue; diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 15f9dec3782..a499d0587b2 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -442,7 +442,8 @@ for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, Se /** * Stub WebTransport for lite-06, which carries the credential in the URL: refuses `refused` - * once `release` resolves, and accepts any other credential, recording each dialed jwt. + * once `release` resolves, never having sent the SETUP that would accept it, and accepts any + * other credential, recording each dialed jwt. */ function refusing(refused: string, release: Promise) { const dialed = new Signal([]); @@ -450,17 +451,19 @@ function refusing(refused: string, release: Promise) { const jwt = new URL(target).searchParams.get("jwt"); dialed.update((jwts) => [...jwts, jwt]); const pair = createMockTransportPair(Lite.ALPN_06); - void accept(pair.server, new URL(target)).then(async () => { - if (jwt !== refused) return; - await release; - pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }); - }); + if (jwt === refused) { + void release.then(() => + pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }), + ); + } else { + void accept(pair.server, new URL(target)); + } return pair.client; }; return { dialed, stub: stub as unknown as typeof WebTransport }; } -test("a first session refused before it is healthy stops the URL, even for replaced tokens", async () => { +test("a first session refused before its credential is accepted stops the URL, even for replaced tokens", async () => { using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); @@ -479,8 +482,8 @@ test("a first session refused before it is healthy stops the URL, even for repla await clock.drive(waitFor(reload.established, (established) => established !== undefined)); tokens.set(["new"]); resolve(); - // The refusal arrives after the session was accepted, as a relay rejects a token on - // moq-lite 03 or later, yet it stops the URL like a failed first attempt. + // The refusal arrives once the session is live, before the server's SETUP, as a relay + // rejects a token on moq-lite 03 or later, and stops the URL like a failed first attempt. const err = await clock.drive(waitFor(reload.error, (error) => error !== undefined)); expect(err).toBeInstanceOf(SessionError); expect((err as SessionError).code).toBe(SessionCode.Unauthorized); @@ -493,6 +496,41 @@ test("a first session refused before it is healthy stops the URL, even for repla } }); +test("a first session dropped right after the peer accepted its credential redials", async () => { + using clock = manual(); + const original = globalThis.WebTransport; + const dials = new Signal(0); + let first: ReturnType | undefined; + const accepted = Promise.withResolvers(); + const stub = function StubWebTransport(target: string | URL) { + dials.update((n) => n + 1); + const pair = createMockTransportPair(Lite.ALPN_06); + first ??= pair; + void accept(pair.server, new URL(target)).then(() => accepted.resolve()); + return pair.client; + }; + globalThis.WebTransport = stub as unknown as typeof WebTransport; + + const reload = new Reload({ + url: new URL("https://example.com/"), + versions: ["moq-lite-06"], + websocket: { enabled: false }, + backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, + }); + try { + const established = await clock.drive(waitFor(reload.established, (session) => session !== undefined)); + await clock.drive(accepted.promise); + // The server's SETUP accepted the credential, long before the healthy bar. + await clock.drive(waitFor(wireOf(established as Session).auth.accepted, (yes) => yes)); + first?.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }); + await clock.drive(waitFor(dials, (n) => n >= 2)); + expect(reload.error.peek()).toBeUndefined(); + } finally { + reload.close(); + globalThis.WebTransport = original; + } +}); + test("a refused credential keeps redialing until the tokens change", async () => { using clock = manual(); const original = globalThis.WebTransport; @@ -509,7 +547,8 @@ test("a refused credential keeps redialing until the tokens change", async () => backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); try { - // Refused only once a session was healthy: until then, a refusal stops the URL. + // Refused only once a session was healthy, though never accepted: until then, a refusal + // stops the URL. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); await clock.advance(Duration.millis(1000)); resolve(); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 324af557372..38250056d87 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -40,10 +40,13 @@ const PROBE_MAX = Duration.secs(300); * Exponential backoff settings for the reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Nothing retries until a session to the current URL has been established, which - * takes staying up past {@link Backoff.healthy} or being drained by a GOAWAY, which no - * refusal sends: an attempt failing, or a session closing sooner, stops that URL with its error. That covers a relay refusing a session right after - * accepting it, as it does a rejected token on moq-lite 03 or later. After that, every failure is retried, + * lockstep. Nothing retries until a session to the current URL has been established: the peer + * accepts the connection credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the + * connection's own credential), drains the session with a GOAWAY, which no refusal sends, or + * the session stays up past {@link Backoff.healthy}, the only bar on moq-lite 03/04, which + * answer the credential with nothing. An attempt failing, or a session closing sooner, stops that + * URL with its error. That covers a relay refusing a live session, as it does a rejected token + * on moq-lite 03 or later. After that, every failure is retried, * including an attempt that outlives {@link ConnectProps.timeout}, until * {@link Backoff.timeout} runs out, whatever the error, an auth rejection included; only a * refused GOAWAY redirect stops the current URL sooner. A new URL or a disable/re-enable starts another sequence. @@ -70,6 +73,9 @@ export type Backoff = { /** * How long a session must stay up to count as healthy (default: 1000ms; must be positive). * + * The first healthy session establishes the URL, though only a peer that never answers the + * connection credential, as on moq-lite 03/04, waits for this: an accepting peer + * establishes it sooner. * A healthy session's end resets the delay and the timeout. One that ends sooner is a * failed attempt, so a peer that accepts and then drops every session still escalates the * delay and exhausts the timeout. @@ -191,8 +197,8 @@ export type ReloadStatus = "connecting" | "connected" | "disconnected"; * The reconnect loop behind a Connection handle: connects, waits for the session to end, then * redials with exponential backoff. * - * The first attempt for a URL fails fast instead: until a session to it has stayed up past - * {@link Backoff.healthy} or been drained by a GOAWAY, a failed attempt or a closed session sets {@link Reload.error} + * The first attempt for a URL fails fast instead: until a session to it was established (see + * {@link Backoff}), a failed attempt or a closed session sets {@link Reload.error} * rather than retrying, so a setup mistake such as an * untrusted certificate or no version in common surfaces at once. A page opened before its * relay is reachable therefore stays disconnected until a new URL or a disable/re-enable. @@ -373,8 +379,8 @@ export class Reload { // loop is disabled. #givenUpHref: string | undefined; - // The configured href a session stayed up past the healthy bar for, or the peer drained with a - // GOAWAY. Until it matches, a failed + // The configured href a session's peer accepted the credential of, a session stayed up past + // the healthy bar for, or the peer drained with a GOAWAY. Until it matches, a failed // attempt or a closed session gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which // resume a URL that worked. #establishedHref: string | undefined; @@ -559,7 +565,7 @@ export class Reload { effect.cleanup(() => { if (current) { // A session torn down while live, as on a page hide, may already have earned it. - if (connected !== undefined) this.#settle(href, connected); + if (connected !== undefined) this.#settle(href, current, connected); this.#end(current); if (this.established.peek() === current) this.#serve(undefined); current = undefined; @@ -655,10 +661,10 @@ export class Reload { reached = raced; } const ended = raced instanceof Reached ? { uri: "" } : raced; - this.#settle(href, connected); + this.#settle(href, connection, connected); if (ended === null || ended instanceof Error) { - // Until a session has stayed up past the healthy bar, a close stops the URL as a - // failed attempt does: it is how a relay refuses a session it already accepted. + // Until a session was established, a close stops the URL as a failed attempt + // does: it is how a relay refuses a live session before accepting its credential. // No error kind is read. if (this.#establishedHref !== href) { console.warn(`connection closed, not retrying: ${ended ? reason(ended) : "no error"}`); @@ -747,9 +753,8 @@ export class Reload { return; } - // Attempts fail fast until a session was established: a URL that never held one past - // the healthy bar is more likely misconfigured than restarting, and only the app can - // fix that. + // Attempts fail fast until a session was established: a URL whose peer never accepted + // one is more likely misconfigured than restarting, and only the app can fix that. if (this.#establishedHref !== href) { console.warn("connection failed, not retrying:", err); this.#giveUp(error(err)); @@ -924,8 +929,13 @@ export class Reload { } // Mark `href` established once the session connected at `connected` stayed up past the healthy bar. - #settle(href: string, connected: Instant): void { - if (now().since(connected).ge(this.#pacing().healthy)) this.#establishedHref = href; + // A session establishes its URL once the peer accepted its credential, which a closed + // session still reports, or once it stayed up past the healthy bar, the only bar on a + // version that never answers the credential (moq-lite-03/04). + #settle(href: string, session: Session, connected: Instant): void { + if (wireOf(session).auth.accepted.peek() || now().since(connected).ge(this.#pacing().healthy)) { + this.#establishedHref = href; + } } // Resolved per use rather than at construction, so an edit to `backoff` (including one that diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index f442e612353..512b4f8b5d2 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -231,6 +231,9 @@ export class Connection implements Session { draining: () => this.#goaway.peek() !== undefined || this.#phase.kind !== "open", // Without AUTH, the server checks a client's credential before its SETUP, which the client reads after going live. awaiting: client && hasSetupStream(version) && !hasAuth(version), + // Lite-03/04 have neither, so the peer refuses the credential only by closing the + // session. Lite-01/02 answered in the negotiation SETUP already read. + silent: client && (version === Version.DRAFT_03 || version === Version.DRAFT_04), }); // Left before anything observes the close, so a successor on the same handle never // finds this session still active. @@ -605,7 +608,7 @@ export class Connection implements Session { if (!this.#client && setup.path !== undefined) throw new ProtocolViolation("Path on a URI-carrying binding"); this.#peerSetup.set(setup); - this.#auth.accepted(); + this.#auth.accept(); this.#peerRole.set(setup.role); this.#peerImplementation.resolve(setup.implementation); } else { From c01962c45ed85d496a520091e2c77ba566c24eb7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:23:05 -0800 Subject: [PATCH 04/19] docs: a session is established once the peer accepts its credential Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 8 ++--- dart/moq/lib/src/aliases.dart | 4 +-- dart/moq/lib/src/client.dart | 4 +-- doc/bin/gstreamer.md | 2 +- doc/lib/cpp/index.md | 2 +- doc/lib/dart/index.md | 2 +- doc/lib/go/index.md | 2 +- doc/lib/index.md | 2 +- doc/lib/kt/index.md | 2 +- doc/lib/py/index.md | 2 +- doc/lib/swift/index.md | 2 +- go/wrapper/client.go | 6 ++-- .../kotlin/dev/moq/Aliases.kt | 4 +-- py/moq-rs/README.md | 2 +- py/moq-rs/moq/_records.py | 2 +- rs/libmoq/src/api.rs | 2 +- rs/moq-ffi/src/session.rs | 6 ++-- rs/moq-gst/src/sink/session.rs | 4 +-- rs/moq-net/verify/reconnect/Handover.tla | 32 +++++++++++-------- rs/moq-tokio/src/connection.rs | 13 +++++--- swift/Sources/Moq/Aliases.swift | 4 +-- swift/Sources/Moq/Client.swift | 2 +- .../reconnect-stops-after-session-loss.patch | 4 +-- test/interop/README.md | 4 +-- test/interop/clients/js/src/close.ts | 4 +-- 25 files changed, 65 insertions(+), 56 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 5ac64091797..a038608f2a2 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -18,7 +18,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | -| `FirstWalkFailsFast` | Until a session has stayed up past the healthy bar or got a GOAWAY, which no refusal sends, no failed dial or closed session backs off: either ends the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | +| `FirstWalkFailsFast` | Until a session's peer accepted its credential, or one stayed up past the healthy bar or got a GOAWAY, which no refusal sends, no failed dial or closed session backs off: either ends the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run @@ -28,7 +28,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 1.5M distinct states, under a minute. The second checks the same rules under `Restart::Handoff` over three sessions: about 330k states. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 5.2M distinct states, under a minute. The second checks the same rules under `Restart::Handoff` over three sessions: about 1.2M states. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -40,12 +40,12 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `27b46ef35bdf284ba61158521723768436cd8460`. Run `git log 27b46ef35b.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `392cdb6d0240e83ebfc4111518ee9cc6cbb6ff09`. Run `git log 392cdb6d02.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | | `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | -| `rs/moq-net/src/reconnect.rs` `established`, set by `end` and by a GOAWAY in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | +| `rs/moq-net/src/reconnect.rs` `established`, set by `end`, by a GOAWAY or `Session::poll_accepted` in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "handoff"` (`RestartHandoff.cfg`) | diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 3a488d7368d..1b300c9f469 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -299,8 +299,8 @@ typedef ConnectionState = MoqConnectionState; /// Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. /// A null field keeps its default, zero means zero (retry at once, or give up -/// at the first failure), and `maxUnlimited` lifts the ceiling. Until a session -/// has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed +/// at the first failure), and `maxUnlimited` lifts the ceiling. Until the peer +/// has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed /// session, such as a refused credential, ends the connection with its error /// rather than retrying. typedef Backoff = MoqBackoff; diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 22d787ab48e..acf65ed4027 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -60,8 +60,8 @@ final class ConnectOptions { /// The WebSocket fallback raced against QUIC for `http(s)` URLs. final WebSocketConfig? websocket; - /// Set true for a one-shot dial. By default a session that stayed up past 1s - /// or was drained by a GOAWAY redials with backoff whenever the transport + /// Set true for a one-shot dial. By default a session whose credential the peer accepted, + /// or that the peer drained with a GOAWAY, redials with backoff whenever the transport /// drops; before then, a failed dial or a closed session, such as a refused /// credential, ends it with that error instead. final bool once; diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 4fca7f8465f..b715d2ab516 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -91,7 +91,7 @@ codec config, a pad requested again, or a stream resumed after EOS publishes as `url`, `broadcast`, `linger`, `connect-version`, `connect-websocket-enabled`, `tls-disable-verify`, `quic-idle-timeout`, `quic-keylog`, `quic-keep-alive`, and read-only `status`, `connected`, `moq-version`, and `estimated-send-rate`, `estimated-recv-rate`, `connection-stats`, and -`sessions`. Once a session has stayed up past a second, the sink reconnects for as long as +`sessions`. Once the relay has accepted a session, the sink reconnects for as long as the pipeline runs, whatever ends each session. Before then it reports `failed` and posts an error instead of retrying: a first connect that reaches no relay, or a session the relay refuses, such as for a rejected token. diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index b52a06f4c8a..d0c26c41c9d 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -257,7 +257,7 @@ Set `ClientConfig::tokens` instead of putting the credential in the URL. The fir ## Stats -`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. +`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. ## Shutdown diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 7f8b38ec8f4..637b6633b40 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -160,7 +160,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` drains for up to one second and reports its result. `session.shutdown()` requests the drain synchronously; `await session.closed()` waits for its result. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements and grants also release their consumer; server listeners leave their shared server usable. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` or disposing its `MoqRoute` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `client.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps: ...)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps: ...)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `dispose()` it once the broadcast stops being watched. `setTarget(targetBps: ...)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `OutOfRangeMoqException`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. - **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential: credential)` or `claim.setAuth(credential: credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index cb19909cacb..0f923a095f5 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -133,7 +133,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown()` requests that drain immediately; `session.Closed(ctx)` waits for its result, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `Finish()` on its `media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.Close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `Session().Viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `Watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `Rate()` reads its slice in bits per second, nil while there is no estimate, so hold the current selection. `Update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `Close()` it once the broadcast stops being watched, from the goroutine that last used it. `SetTarget(&bps)` sends another target, zero sends none, and nil follows the viewers again. `Watch`, `Update`, and `SetTarget` return `moq.ErrOutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. Client viewers survive reconnects and read nil while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Cancel()` it when done. - **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `Session().Credential(token)` returns a `Credential` that `Route.SetAuth` or `Claim.SetAuth` names (nil for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `Answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `Answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `Answered(ctx)` returns the verdict, and `Cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/index.md b/doc/lib/index.md index 7b29ed3811d..32b9a53a4d5 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -67,7 +67,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. Until a session has stayed up past 1s or been drained by a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). +- **Reconnect** automatically with backoff when the transport drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 8f95eedee1e..0faec1c42e1 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -111,7 +111,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `moq.shutdown()` requests a drain immediately; suspending `moq.session.closed()` waits up to one second and reports its result. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.end()` takes it offline. How long to keep it up is yours to decide. A track still open at the end ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `moq.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `close()` it, or leave `use { }`, once the broadcast stops being watched. `setTarget(bps)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `MoqException.OutOfRange`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. - **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential)` or `claim.setAuth(credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. Suspending `credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 87f965ad3f3..a9887b9b113 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -125,7 +125,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `await session.closed()` waits up to one second and reports its result. `cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `moq.media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(max_bps)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `None` while there is no estimate (keep the current rendition), and `update(max_bps)` changes the ceiling. Holding a viewer requests the estimate like `recv_bandwidth()`, and dropping it releases the claim. The session sends the summed ceilings as its PROBE target; `set_target(bps)` overrides that, `0` sends none, and `None` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, raise `moq.Error.OutOfRange`. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. - **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session.credential(token)` returns a `Credential` that `route.set_auth(credential)` or `claim.set_auth(credential)` names; setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` raises the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 6cbc0cdf4a0..606af2df8ff 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -108,7 +108,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `try await session.closed()` waits up to one second and reports its result. `session.cancel(code:)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try finish()` on its `Media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()`, or releasing its route, takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(maxBps:)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `nil` while there is no estimate (keep the current rendition), and `update(maxBps:)` changes the ceiling. Holding a viewer requests the estimate like `recvBandwidth()`, and releasing it releases the claim. The session sends the summed ceilings as its PROBE target; `setTarget(targetBps:)` overrides that, `0` sends none, and `nil` follows the viewers again. `watch`, `update`, and `setTarget` throw `MoqError.OutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. - **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `try session.credential(token)` returns a `Credential` that `try route.setAuth(credential)` or `try claim.setAuth(credential)` names (`nil` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `try await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/go/wrapper/client.go b/go/wrapper/client.go index 87f69548a89..4dec04c6d63 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -204,8 +204,8 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // Backoff is the retry pacing for automatic reconnects: the delay starts at // Initial, multiplies by Multiplier after each failed attempt, and caps at Max. -// After Timeout of consecutive failures the connection gives up for good. Until a -// session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a +// After Timeout of consecutive failures the connection gives up for good. Until the +// peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a // closed session, such as a refused credential, ends the connection with its // error rather than retrying. // @@ -251,7 +251,7 @@ type Client struct { } // Dial starts a MoQ connection and returns without waiting for its first native dial to succeed. Cancel ctx -// to abort an in-flight connect. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed +// to abort an in-flight connect. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed // dial or a closed session, such as a refused credential, closes the session with its error (see Session.Closed) // rather than retrying. func Dial(ctx context.Context, url string, config ClientConfig) (*Client, error) { diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index c2e33504612..2a05b28ac43 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -390,8 +390,8 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState /** * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. * A null field keeps its default, zero means zero (retry at once, or give up at the first failure), - * and `maxUnlimited` lifts the ceiling. Until a session has stayed up past 1s or been drained by a - * GOAWAY, a failed dial or a closed session, such as a refused credential, ends the connection with + * and `maxUnlimited` lifts the ceiling. Until the peer has accepted a session's credential or drained a session with + * a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the connection with * its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index ca281c2ac8a..5a6cb6c9278 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -160,7 +160,7 @@ client = moq.Client( - `.cancel(code)`, `session.shutdown()`. Cancel immediately, or request a drain now and await `session.closed()` for its result, including a delivery timeout after one second. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - `.publish() → OriginProducer`, `.consume() → OriginConsumer`. The wired origin sides. - `.stats() → ConnectionStats`. Snapshot RTT, bandwidth estimates, and byte/packet counters. - - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends it with that error. + - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends it with that error. - `.allocator() → Allocator`. Divide the send estimate between encoders and app-owned tracks. - `.viewers() → Viewers`. Divide the receive estimate between watched broadcasts: `.watch(max_bps) → Viewer` claims up to a ceiling, `Viewer.rate()` reads its slice (`None` while there is no estimate), and dropping the viewer releases it. The summed ceilings are the PROBE target; `.set_target(bps)` overrides it (`0` sends none, `None` follows the viewers again). - `.grants() → GrantConsumer`. On `moq-lite-07-wip`, `.grant()` reads the peer's `Grant` and `async for grant in session.grants()` follows it. diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index 36702a24c8f..9840185a868 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -345,7 +345,7 @@ def _track_info(info: TrackInfo | None) -> MoqTrackInfo | None: class Backoff: """Retry pacing for the automatic reconnect. - Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or + Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the connection with its error rather than retrying. diff --git a/rs/libmoq/src/api.rs b/rs/libmoq/src/api.rs index 407baf96288..ccd76d51f8d 100644 --- a/rs/libmoq/src/api.rs +++ b/rs/libmoq/src/api.rs @@ -1790,7 +1790,7 @@ unsafe fn connect_session( /// You should call [moq_session_close], even on error, to free up resources. /// /// Once connected, the session reconnects automatically with exponential backoff if the -/// connection drops. Until a session has stayed up past 1s or been drained by a GOAWAY, a +/// connection drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a /// failed dial or a closed session, such as a refused credential, ends it instead, with that error as the terminal /// snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on /// each reconnect, since the origins outlive the underlying connection. diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 91df5e9e37a..4348ae6680c 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -202,7 +202,7 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// -/// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a +/// Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a /// closed session, such as a refused credential, ends the connection with its error rather /// than retrying. The delay starts at `initial`, multiplies by `multiplier` after each /// failed attempt, and caps at `max`. @@ -615,8 +615,8 @@ impl MoqClient { /// /// Once a session has been established, it automatically reconnects with backoff when /// the transport drops (unless [`MoqClientConfig::once`] is set), and broadcasts - /// consumed through it ride out the gap. Until a session has stayed up past 1s or been - /// drained by a GOAWAY it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the + /// consumed through it ride out the gap. Until the peer has accepted a session's credential or drained + /// a session with a GOAWAY it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the /// one it accepted, as a relay refusing a credential does, [`MoqSession::closed`] resolves /// with that error, so an app started before its relay is reachable must start the session /// again. diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index d8a8a432b47..33b73953aa0 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -293,8 +293,8 @@ impl Session { // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC // outages of any length, which is the trade this element wants: a pipeline nobody is watching - // should still be publishing when the relay comes back. The loop still ends when no session - // stays up past the healthy bar, such as a relay refusing the token, posting the bus error + // should still be publishing when the relay comes back. The loop still ends when the relay + // never accepted a session, such as when it refuses the token, posting the bus error // below. During an outage the pad threads keep writing (bounded // by moq-net's per-group eviction) and the relay catches up from a group boundary on // reconnect. A bounded policy is available via `ClientConfig::backoff`. diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 9f5d29eac7a..d541512369c 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -20,8 +20,9 @@ (* one it refuses, no retry window, no tokens, not connecting once, and no *) (* redirect-loop scoring. A session's age stands in for its tenure: 1 is *) (* past `Backoff::healthy`. Until a session has been established, which *) -(* takes one past that bar or a GOAWAY, a failed dial or a closed session *) -(* ends the policy (`Reconnect::established`). *) +(* takes the peer accepting its credential, one past that bar, or a *) +(* GOAWAY, a failed dial or a closed session ends the policy *) +(* (`Reconnect::established`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -59,7 +60,7 @@ VARIABLES started, \* `Presence::sessions_started` ended, \* `Presence::sessions_ended` misdials, \* 1 once a dial went elsewhere than an unheeded redirect named - established \* `Reconnect::established`: a session stayed up past the healthy bar or got a GOAWAY + established \* `Reconnect::established`: a session's peer accepted its credential, or one stayed up past the healthy bar or got a GOAWAY policyVars == <> @@ -71,11 +72,13 @@ vars == <> \* "abort" for `Session::abort`, "drain" for a `start_close` that finished), \* whether a `start_close` began, the GOAWAY it got, its age, whether it \* carries no subscription or fetch, whether the policy acted on its GOAWAY, -\* whether we sent our own, and whether the policy watches its GOAWAY as the -\* predecessor (`Predecessor::redirect`, set when a probe moved us off it). +\* whether we sent our own, whether the policy watches its GOAWAY as the +\* predecessor (`Predecessor::redirect`, set when a probe moved us off it), and +\* whether the peer accepted its credential (`Session::poll_accepted`), which a +\* close leaves as it was. Fresh(e) == [ep |-> e, open |-> TRUE, by |-> "", closing |-> FALSE, goaway |-> NoGoaway, age |-> 0, idle |-> FALSE, heeded |-> FALSE, - sent |-> FALSE, watched |-> FALSE] + sent |-> FALSE, watched |-> FALSE, accepted |-> FALSE] Unused == [Fresh(Start) EXCEPT !.open = FALSE] Timers == {"dial", "backoff", "probe", "handover"} @@ -330,12 +333,12 @@ EndGoaway(s, uri) == /\ UNCHANGED watch \* `Reconnect::closed`. Until a session has been established, a close ends the -\* policy as a failed dial does: it is how a peer refuses a session it already -\* accepted. After that, every close retries, whatever its error. A predecessor +\* policy as a failed dial does: it is how a peer refuses a live session before +\* accepting its credential. `poll` reads the acceptance before the close. After that, every close retries, whatever its error. A predecessor \* that is not closing gracefully serves again, so the policy migrates back \* onto it; the `Retrying` that follows changes no status, so it is left out. EndClosed == - /\ established' = (established \/ sess[live].age >= 1) + /\ established' = (established \/ sess[live].age >= 1 \/ sess[live].accepted) /\ probe' = "none" /\ IF established' THEN /\ live' = None @@ -499,8 +502,8 @@ ProbeGetsThrough == /\ dial' = "handshake" /\ dialTo' = sess[live].ep /\ Emit(Append(queue, "Migrating"), live) - \* The fallback served all along, so a long one establishes the policy. - /\ established' = (established \/ sess[live].age >= 1) + \* The fallback served all along, so a long or accepted one establishes the policy. + /\ established' = (established \/ sess[live].age >= 1 \/ sess[live].accepted) /\ UNCHANGED <> Caller == Connected \/ Failed \/ ProbeFails \/ ProbeGetsThrough @@ -522,11 +525,14 @@ PeerClose == /\ sess' = [sess EXCEPT ![s].open = FALSE, ![s].by = IF sess[s].closing THEN "drain" ELSE "peer"] -\* The live session passes the healthy bar, or the predecessor stops carrying -\* a subscription or fetch. Only the policy reads either, and only there. +\* The live session passes the healthy bar, its peer accepts its credential, or +\* the predecessor stops carrying a subscription or fetch. Only the policy reads +\* any of them, and only there. Ages == \/ /\ Open(live) /\ sess[live].age = 0 /\ sess' = [sess EXCEPT ![live].age = 1] + \/ /\ Open(live) /\ ~sess[live].accepted + /\ sess' = [sess EXCEPT ![live].accepted = TRUE] \/ /\ Open(pred) /\ ~sess[pred].idle /\ sess' = [sess EXCEPT ![pred].idle = TRUE] diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 32a19a3f2af..f28de695067 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -44,8 +44,10 @@ fn verdict(client: &Client, url: &url::Url, err: &Error) -> Verdict { /// Three knobs, each with one job: `initial` and `multiplier` set the wait before /// each retry, `max` caps it, and `healthy` decides which sessions count as failed /// attempts. See [`moq_net::reconnect::Backoff`], which this resolves to, for how -/// the policy applies them. Until a session has stayed up past `healthy` or been drained by -/// a GOAWAY, a failed dial or a closed session ends the connection instead of retrying. +/// the policy applies them. Until the peer has accepted a session's credential or drained a +/// session with a GOAWAY (or, on moq-lite 03/04, which never answer the credential, a session +/// stayed up past `healthy`), a failed dial or a closed session ends the connection instead +/// of retrying. #[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -89,7 +91,8 @@ pub struct Backoff { /// How long a session must stay up to count as healthy. Defaults to 1s; zero is refused. /// - /// A healthy session's end resets the delay and the timeout. One that ends sooner is a + /// A healthy session establishes the connection even when its peer never answered the + /// credential, as on moq-lite 03/04. A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. Set by the embedding binary, not configured. #[usage(skip)] @@ -896,8 +899,8 @@ pub struct Snapshot { /// /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) -/// redials with exponential backoff. Until a session has stayed up past the healthy bar or -/// been drained by a GOAWAY, nothing retries: a failed dial, or a session the peer closes, as a relay refusing a +/// redials with exponential backoff. Until the peer has accepted a session's credential or +/// drained a session with a GOAWAY, nothing retries: a failed dial, or a session the peer closes, as a relay refusing a /// credential does, stops the loop with its error, so a setup mistake surfaces at once. The read surface /// mirrors [`moq_net::Session`] so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 0d2f24aa334..5412e383583 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -91,8 +91,8 @@ public typealias ConnectionStats = MoqFFI.MoqConnectionStats /// Retry pacing for the automatic reconnect; see `Client(backoff:)`. A `nil` /// field keeps its default, zero means zero (retry at once, or give up at the -/// first failure), and `maxUnlimited` lifts the ceiling. Until a session has -/// stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed +/// first failure), and `maxUnlimited` lifts the ceiling. Until the peer has +/// accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed /// session, such as a refused credential, ends the connection with its error /// rather than retrying. public typealias Backoff = MoqFFI.MoqBackoff diff --git a/swift/Sources/Moq/Client.swift b/swift/Sources/Moq/Client.swift index 1be3e289c78..e4769c838c4 100644 --- a/swift/Sources/Moq/Client.swift +++ b/swift/Sources/Moq/Client.swift @@ -65,7 +65,7 @@ public final class Client: Sendable { } /// Start a connection and return its session without waiting for the native first dial to succeed. - /// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed + /// Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed /// session, such as a refused credential, closes the session with its error rather than retrying. public func connect(to url: String) async throws -> Session { Session(try await ffi.connect(url: url)) diff --git a/test/drill/mutations/reconnect-stops-after-session-loss.patch b/test/drill/mutations/reconnect-stops-after-session-loss.patch index 09322a59de6..87d79be4499 100644 --- a/test/drill/mutations/reconnect-stops-after-session-loss.patch +++ b/test/drill/mutations/reconnect-stops-after-session-loss.patch @@ -8,7 +8,7 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs -@@ -1088,7 +1088,8 @@ impl + From> Reco +@@ -1104,7 +1104,8 @@ impl + From> Reco fn closed(&mut self, now: Instant, err: Error, healthy: bool, url: &Url) { let err = E::from(err); // Connecting once, the session ending ends the policy with its close reason. @@ -17,4 +17,4 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs + if self.config.once || healthy { return self.done(Err(err)); } - // Until a session has stayed up past the healthy bar, a close ends the policy as a + // Until a session was established, a close ends the policy as a failed walk does. It diff --git a/test/interop/README.md b/test/interop/README.md index d3d72dc2ce8..4cf9715aaf1 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -124,8 +124,8 @@ released relays and clients, skip it. Whenever the browser subscriber is in the run, the matrix ends with a close-code case: Chromium dials the relay with a token its public rules refuse, and `WebTransport.closed` must carry the relay's code and reason. A WebSocket-only -`Connection` counts the refused session as connected, but it never stayed up -past the healthy bar, so the handle must stop at it without redialing and +`Connection` counts the refused session as connected, but the relay never +accepted its credential, so the handle must stop at it without redialing and publish the same session error. Chromium treats a server's HTTP/3 control stream ending as fatal, so a server that ends it under the close capsule loses both; no Rust peer is that strict. diff --git a/test/interop/clients/js/src/close.ts b/test/interop/clients/js/src/close.ts index 7385af7292b..df2a6fa2ddc 100644 --- a/test/interop/clients/js/src/close.ts +++ b/test/interop/clients/js/src/close.ts @@ -45,8 +45,8 @@ export async function refused(relay: string): Promise { /** * A WebSocket refusal reaches the reconnecting handle as its session error. * - * The relay closes a session that ALPN already established. It never stayed up past the healthy - * bar, so the handle stops at once with that close, as after a refused dial, instead of redialing. + * The relay closes a session that ALPN already established, never having accepted its credential, + * so the handle stops at once with that close, as after a refused dial, instead of redialing. */ export async function refusedWebSocket(relay: string): Promise { const url = new URL(relay); From 6aebca0ed366828b2cd713e23b0fea474f04cc2c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:28:34 -0800 Subject: [PATCH 05/19] chore: format, and name acceptance in the coverage table Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- js/net/src/connection/reload.test.ts | 4 +--- .../jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt | 6 +++--- rs/moq-net/src/reconnect/tests.rs | 15 ++++++++------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index a038608f2a2..e9ef0a6ec73 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -45,7 +45,7 @@ Sync commit: `392cdb6d0240e83ebfc4111518ee9cc6cbb6ff09`. Run `git log 392cdb6d02 | Code | Model | | --- | --- | | `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | -| `rs/moq-net/src/reconnect.rs` `established`, set by `end`, by a GOAWAY or `Session::poll_accepted` in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | +| `rs/moq-net/src/reconnect.rs` `established`, set by `end`, by a GOAWAY or the session's acceptance in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "handoff"` (`RestartHandoff.cfg`) | diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index a499d0587b2..120c5161f91 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -452,9 +452,7 @@ function refusing(refused: string, release: Promise) { dialed.update((jwts) => [...jwts, jwt]); const pair = createMockTransportPair(Lite.ALPN_06); if (jwt === refused) { - void release.then(() => - pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }), - ); + void release.then(() => pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" })); } else { void accept(pair.server, new URL(target)); } diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 2a05b28ac43..4d534fc08b9 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -390,9 +390,9 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState /** * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. * A null field keeps its default, zero means zero (retry at once, or give up at the first failure), - * and `maxUnlimited` lifts the ceiling. Until the peer has accepted a session's credential or drained a session with - * a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the connection with - * its error rather than retrying. + * and `maxUnlimited` lifts the ceiling. Until the peer has accepted a session's credential or + * drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, + * ends the connection with its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index a87d73bcbf8..5857ab38adc 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -420,12 +420,9 @@ async fn an_accepted_session_establishes_the_policy() { async fn a_silent_version_is_established_by_uptime() { let lite04: crate::Version = "moq-lite-04".parse().unwrap(); for healthy in [false, true] { - let mut machine = Reconnect::::new( - config(INITIAL, None), - vec![url("tcp://peer:1/")], - kio::clock::now(), - ) - .unwrap(); + let mut machine = + Reconnect::::new(config(INITIAL, None), vec![url("tcp://peer:1/")], kio::clock::now()) + .unwrap(); assert!(matches!(next(&mut machine).await, Action::Dial(_))); let (client, server, _) = handshake_with(lite04, false).await; machine.connected(kio::clock::now(), client); @@ -433,7 +430,11 @@ async fn a_silent_version_is_established_by_uptime() { let healthy_bar = Config::default().backoff.healthy; kio::clock::sleep(if healthy { healthy_bar } else { healthy_bar / 2 }).await; assert!( - machine.session().expect("live").poll_accepted(&kio::Waiter::noop()).is_pending(), + machine + .session() + .expect("live") + .poll_accepted(&kio::Waiter::noop()) + .is_pending(), "moq-lite-04 has no answer to accept with" ); server.abort(Error::Unauthorized); From 10a12e842f28be8810bd994478a4da62377c04c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:35:03 -0800 Subject: [PATCH 06/19] test(ffi): the first-dial state test no longer assumes a refusal Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/test.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index d4ac78e218e..dc1e1643956 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7705,11 +7705,9 @@ async fn session_state_names_a_failing_first_dial() { )); assert_eq!(accepted.state().revision, terminal.revision); assert!(accepted.state_changed(terminal.revision).await.is_err()); - // The server ended the client's first session before it was healthy, which ends the - // client's session too rather than redialing. - let closed = kio::clock::timeout(TIMEOUT, session.closed()).await.unwrap(); - assert!(closed.is_err(), "the refused first session closed cleanly"); - assert!(matches!(session.state().phase, MoqConnectionPhase::Closed { .. })); + // Whether the client redials depends on whether the server's AUTH_OK reached it before the + // close, which this test does not order, so it only ends the client's session. + session.cancel(0); accepted.cancel(0); server.cancel(); } From fa9fe3652662f997f0414b8161094ea6daaa0247 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 17:46:13 -0800 Subject: [PATCH 07/19] quest: rewrite credential-established to the binding tests it leaves Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/credential-established.md | 67 ++++++++++------------ 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/quest/m0/nightly/credential-established.md b/quest/m0/nightly/credential-established.md index 33eff2b9515..a8e80ddc771 100644 --- a/quest/m0/nightly/credential-established.md +++ b/quest/m0/nightly/credential-established.md @@ -1,48 +1,41 @@ -# [M] A reconnecting client is established by the credential's answer +# [S] Binding tests drop a session once the client saw it accepted ## Goal -A reconnecting client (Rust `Reconnect` and `moq_tokio::Connection`, JS -`Reload`, and every binding through `moq-ffi`) counts a session as established -once the peer accepts the connection credential, instead of after it stays up -`Backoff::healthy` (1 s). A close before that answer is a refusal and ends the -policy, as today. Versions with no acceptance message keep the uptime bar. No -wire change. - The tests [moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817) gave -real 1 s waits (Go, Python, moq-c), and the OBS tests in draft +real 1 s waits (Go `go/wrapper/reconnect_test.go`, Python +`py/moq-rs/tests/test_server.py`, `rs/moq-ffi/src/test.rs`, `rs/libmoq/src/test.rs`), +and the OBS tests in draft [moq-dev/moq#9016](https://github.com/moq-dev/moq/pull/9016), drop the session -right after it is accepted instead. +as soon as the client has seen the peer accept its credential, instead of +waiting out `Backoff::healthy`. No wire change. ## Plan -Found by the triage of Nightly run -[44127032469](https://github.com/moq-dev/moq/actions/runs/44127032469) -(2026-11-17): the OBS output and source tests drop a session the moment it goes -live, which #8817 now treats as a refusal. #9016 fixes them with real 1 s -waits; the maintainer chose this quest instead, and #9016 waits on it. - -Where acceptance already shows (researched 2026-11-17): - -- lite-05/06 and moq-transport: the server sends SETUP only from `ok()`, and - `moq-relay` judges the token before `ok()` - (`rs/moq-relay/src/connection.rs`), so a refused session never sends it. -- lite-07: also the AUTH_OK answering each side's empty-token Auth Stream, - which carries the grant for the connection credential. -- lite-01/02: the SERVER_SETUP reply. -- lite-03/04: nothing. They keep `Backoff::healthy` as the bar. -- `Tokens::credential` (`rs/moq-net/src/auth/tokens.rs`) already computes this - verdict across versions; `js/net/src/connection/tokens.ts`'s `Verdict` is its - counterpart. Reuse them rather than reading SETUP again. - -Decided 2026-11-17 (maintainer, /quest-plan): - -- Signal: ✅ the existing credential verdict (server SETUP, or AUTH_OK on - lite-07) (rejected: a new SETUP_OK message, which duplicates AUTH_OK; 1 s of - uptime for every version). -- ✅ Versions with no acceptance message keep a timeout, `Backoff::healthy`. -- Update `Handover.tla` and the verify-reconnect skill where `established` is - set, and the `Backoff` docs in every binding. +[moq-dev/moq#9038](https://github.com/moq-dev/moq/pull/9038) made the +reconnect policy (Rust `Reconnect`, JS `Reload`, every binding through +`moq-ffi`) count a session as established once the peer accepts the +connection credential: its SETUP, or on lite-07 the AUTH_OK to the +connection's own credential. moq-lite 03/04 keep the 1 s uptime bar. + +What is left: a test can drop a session the moment the *server* accepted it, +but the client counts it only once it has *read* that answer, and a server +close can overtake it. No binding reports that moment: `Connected` fires when +the session goes live, before the answer. So the waits stay until the client +can observe acceptance. + +## Waiting + +- Maintainer: how should a binding observe that the peer accepted the session? + (a) Recommended: moq-tokio's `Connection` reports Connected, and + `established()` resolves, only once the peer accepted the credential (or + the session passed `healthy` on moq-lite 03/04), and every binding's + `Connected` phase follows. A refused session then never reads as + connected, and `established().await` returns the refusal instead of `Ok`. + It's a behavior change in every binding, but no API change. (b) Add an + `accepted` flag to the state snapshot in every binding, which is new API that only + tests need today. (c) Keep the 1 s waits and delete this quest. Check: the + answer is recorded in this Plan. ## Related From 59eca3523142559df3ec624ecbfe0a706525db91 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:06:25 -0800 Subject: [PATCH 08/19] feat(tokio): a connection reports Connected once its peer accepts the session Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect.rs | 32 +++++++- rs/moq-net/src/reconnect/tests.rs | 88 ++++++++++++++++----- rs/moq-tokio/src/connection.rs | 28 +++++-- rs/moq-tokio/tests/integration/backend.rs | 7 +- rs/moq-tokio/tests/integration/broadcast.rs | 34 +++++--- 5 files changed, 144 insertions(+), 45 deletions(-) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 2aab59a6d44..8ff882fd044 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -6,7 +6,8 @@ //! [`Reconnect::failed`], and otherwise polls it with the current time. The machine //! watches each live session for its close and GOAWAY itself, keeps a drained //! predecessor serving through its handover, and drops neither on the caller's behalf -//! until it reports [`Action::Done`]. +//! until it reports [`Action::Done`]. A connected session serves only once the machine +//! reports it [`Action::Established`], so a caller never shows a refusal as connected. //! //! The policy fails fast until a session is established: the peer accepts the connection //! credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the connection's own @@ -340,6 +341,10 @@ pub enum Action { /// [`Reconnect::failed`], polling meanwhile. Build each session's client with /// [`Attempt::client`]. Dial(Attempt), + /// The live session is established: its peer accepted the connection credential, + /// drained it with a GOAWAY, or it stayed up past [`Backoff::healthy`]. Until then a + /// connected session may still be a refusal, so report it as serving only from here. + Established, /// Drop the dial or probe in flight: it ran out its deadline, which already counted as /// its failure, or the predecessor's peer redirected while it headed for that peer. Abandon, @@ -417,6 +422,8 @@ struct Live { session: Session, since: Instant, index: usize, + /// Whether [`Action::Established`] was reported for this session. + established: bool, /// Dropped once a restart is waited out: it is a latch, so it would keep firing. goaway: Option, /// Set while the session runs on a fallback transport. @@ -560,9 +567,11 @@ impl + From> Reco } let live = matches!(self.phase, Phase::Live(_)); + // A replacement serves only once established: until then it may be a refusal. + let serving = matches!(&self.phase, Phase::Live(live) if live.established); if let Some(predecessor) = self.predecessor.as_mut() { // The replacement serves, so a predecessor carrying nothing closes now. - if live { + if serving { predecessor.poll_idle(waiter); } if predecessor.poll(now, waiter) { @@ -570,7 +579,7 @@ impl + From> Reco self.queue.push_back(Action::Retired); // Nothing is serving once it retires between sessions, so a caller must // stop handing out a session that is already closed. - if !live { + if !serving { self.queue.push_back(Action::Offline(None)); } continue; @@ -600,8 +609,11 @@ impl + From> Reco Phase::Live(live) => { // Checked before the close, which may follow the peer's acceptance in one wake: // a session the peer accepted and then dropped is an outage, not a refusal. - if !self.established && live.session.poll_accepted(waiter).is_ready() { + let healthy = live.since.saturating_add(self.config.backoff.healthy); + if !live.established && (now >= healthy || live.session.poll_accepted(waiter).is_ready()) { + live.established = true; self.established = true; + return Action::Established; } let mut ended = None; // Checked before the close: a GOAWAY means the session is still up, and @@ -612,6 +624,9 @@ impl + From> Reco // client started while its relay drains rides out the restart, whether it // redials now or serves until the peer closes the session. self.established = true; + if !std::mem::replace(&mut live.established, true) { + self.queue.push_back(Action::Established); + } // Under `Handoff`, a session drained before it was healthy is a redial that // reached the draining process rather than a replacement, so another would // only reach it again. @@ -636,6 +651,14 @@ impl + From> Reco self.end(now, ended); continue; } + if !self.queue.is_empty() { + continue; + } + // An unestablished session is established by uptime at the latest. + let handover = match live.established { + true => handover, + false => earliest(Some(healthy), handover), + }; match live.probe.as_mut().map(|probe| probe.poll(now)) { None => return Action::Wait(handover), Some(ProbePoll::Wait(at)) => return Action::Wait(earliest(at, handover)), @@ -689,6 +712,7 @@ impl + From> Reco session, since: now, index: dialing.index, + established: false, probe: None, })); } diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 5857ab38adc..0e4ed143fc7 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -93,9 +93,22 @@ async fn handshake_with(version: crate::Version, withhold: bool) -> (Session, Se (client, server, requests) } -/// The next action that is not a wait, sleeping on the simulated clock meanwhile. +/// The next action that is neither a wait nor [`Action::Established`], sleeping on the +/// simulated clock meanwhile. async fn next + From>( machine: &mut Reconnect, +) -> Action { + loop { + match next_any(machine).await { + Action::Established => continue, + action => return action, + } + } +} + +/// The next action that is not a wait, sleeping on the simulated clock meanwhile. +async fn next_any + From>( + machine: &mut Reconnect, ) -> Action { let mut sleep: Option = None; kio::wait(|waiter| { @@ -141,9 +154,13 @@ async fn drive( mut stop: impl FnMut(&Run) -> bool, ) -> Option> { while !stop(run) { - let attempt = match next(machine).await { + let attempt = match next_any(machine).await { Action::Dial(attempt) => attempt, Action::Done(res) => return Some(res), + Action::Established => { + run.events.push("established"); + continue; + } Action::Abandon => { run.events.push("abandon"); continue; @@ -165,7 +182,7 @@ async fn drive( continue; } Action::Probe(_) => unreachable!("no test driven here reports a fallback"), - Action::Wait(_) => unreachable!("next never returns a wait"), + Action::Wait(_) => unreachable!("next_any never returns a wait"), }; let url = attempt.target().url().clone(); run.dials.push(url.clone()); @@ -411,7 +428,7 @@ async fn an_accepted_session_establishes_the_policy() { }) .await; assert!(ended.is_none(), "an established policy gave up"); - assert_eq!(run.events, ["offline", "retrying"]); + assert_eq!(run.events, ["established", "offline", "retrying"]); } /// A version that never answers the connection credential (moq-lite-04) keeps the uptime @@ -437,8 +454,11 @@ async fn a_silent_version_is_established_by_uptime() { .is_pending(), "moq-lite-04 has no answer to accept with" ); + if healthy { + assert!(matches!(next_any(&mut machine).await, Action::Established)); + } server.abort(Error::Unauthorized); - match next(&mut machine).await { + match next_any(&mut machine).await { Action::Offline(_) => assert!(healthy, "a refusal before the bar was retried"), Action::Done(res) => assert!(!healthy && res.is_err(), "a healthy session gave up: {res:?}"), _ => panic!("unexpected action"), @@ -465,7 +485,7 @@ async fn a_healthy_session_establishes_the_policy() { }) .await; assert!(ended.is_none(), "an established policy gave up"); - assert_eq!(run.events, ["offline", "retrying"]); + assert_eq!(run.events, ["established", "offline", "retrying"]); } /// A GOAWAY establishes the policy however soon it comes, since a refusal never drains a @@ -837,7 +857,7 @@ async fn a_healthy_goaway_hands_over_to_the_redirect() { assert_eq!(run.accepts[1], (b.clone(), migrated), "migrated without a backoff"); assert_eq!( run.events, - ["migrating", "retired"], + ["established", "migrating", "established", "retired"], "a replacement serves, so nothing went offline" ); assert!(machine.predecessor().is_none()); @@ -857,6 +877,38 @@ async fn a_healthy_goaway_hands_over_to_the_redirect() { assert_eq!(machine.targets(), [a]); } +/// A replacement the peer has not yet accepted does not serve: a predecessor that retires +/// meanwhile leaves the connection offline, and the replacement reports +/// [`Action::Established`] only once it passes the healthy bar unanswered. +#[kio::test] +async fn an_unanswered_replacement_does_not_serve() { + const HANDOVER: Duration = Duration::from_millis(500); + let peer = url("tcp://peer:1/"); + let mut config = Config::default(); + config.goaway.handover = HANDOVER; + let healthy = config.backoff.healthy; + let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + kio::clock::sleep(Duration::from_secs(2)).await; + run.servers[0].goaway(goaway::Goaway::new()).unwrap(); + drive(&mut machine, &[(peer, Peer::Withhold)], &mut run, |run| { + run.accepts.len() == 2 + }) + .await; + let connected = run.accepts[1].1; + assert_eq!(run.events, ["established", "migrating"]); + assert!(matches!(next_any(&mut machine).await, Action::Retired)); + assert!(matches!(next_any(&mut machine).await, Action::Offline(None))); + assert_eq!(kio::clock::now() - connected, HANDOVER, "the idle predecessor closed early"); + assert!(matches!(next_any(&mut machine).await, Action::Established)); + assert_eq!(kio::clock::now() - connected, healthy); +} + /// Our handover window caps the peer's GOAWAY deadline, and a predecessor retiring while /// no replacement serves leaves the connection offline. #[kio::test] @@ -881,7 +933,7 @@ async fn the_handover_takes_the_earlier_deadline() { run.events.contains(&"offline") }) .await; - assert_eq!(run.events, ["migrating", "retired", "offline"]); + assert_eq!(run.events, ["established", "migrating", "retired", "offline"]); assert_eq!( kio::clock::now() - drained, HANDOVER, @@ -922,10 +974,9 @@ async fn a_connect_cancels_the_offline_a_retirement_queued() { let (client, _server) = handshake().await; machine.connected(kio::clock::now(), client); assert!( - matches!( - machine.poll(kio::clock::now(), &kio::Waiter::noop()), - Action::Wait(None) - ), + kio::clock::timeout(Duration::from_secs(60), next(&mut machine)) + .await + .is_err(), "a stale offline followed the connect" ); assert!(machine.session().is_some()); @@ -988,7 +1039,7 @@ async fn a_handoff_restart_moves_to_a_replacement() { }) .await; assert_eq!(run.accepts[1].1, goaway, "the redial waited"); - assert_eq!(run.events, ["migrating"]); + assert_eq!(run.events, ["established", "migrating"]); assert_eq!(settle(&mut machine).await, ["retired"]); assert!(machine.session().is_some(), "the replacement serves"); assert_eq!(run.dials.len(), 2); @@ -1018,7 +1069,7 @@ async fn a_handoff_restart_redials_once_until_the_session_closes() { |run| run.accepts.len() == 2, ) .await; - assert_eq!(run.events, ["migrating"]); + assert_eq!(run.events, ["established", "migrating"]); assert_eq!(settle(&mut machine).await, ["retired"], "the policy left the session"); assert_eq!(run.dials.len(), 2, "redialed the draining process again"); @@ -1352,6 +1403,8 @@ async fn a_dropped_replacement_goes_offline_over_a_closing_predecessor() { assert!(matches!(next(&mut machine).await, Action::Dial(_))); let (replacement, replacement_server) = handshake().await; machine.connected(kio::clock::now(), replacement.clone()); + // Only an established replacement serves, which lets the idle predecessor close. + assert!(matches!(next_any(&mut machine).await, Action::Established)); // The replacement is already closed when the next poll starts the idle predecessor's // graceful close, so its close is seen over a predecessor that is closing, not retired. drop(replacement_server); @@ -1436,10 +1489,9 @@ async fn connecting_once_never_probes() { }; let (mut machine, _fallback) = on_fallback(config).await; kio::clock::sleep(PROBE_MAX * 2).await; - assert!(matches!( - machine.poll(kio::clock::now(), &kio::Waiter::noop()), - Action::Wait(None) - )); + let waiter = kio::Waiter::noop(); + assert!(matches!(machine.poll(kio::clock::now(), &waiter), Action::Established)); + assert!(matches!(machine.poll(kio::clock::now(), &waiter), Action::Wait(None))); } /// The policy must not log or report a credential riding in a dial URL. diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index f28de695067..f550bf6a586 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -256,7 +256,10 @@ impl Backoff { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Status { - /// A session connected (the first connect, or a reconnect after a drop). + /// A session connected and its peer accepted the connection credential (the first + /// connect, or a reconnect after a drop). On moq-lite 03/04, which never answer the + /// credential, the session must stay up past [`Backoff::healthy`] instead. A session the + /// peer refuses never reports it. Connected, /// An established session dropped; a reconnect attempt follows. Disconnected, @@ -1122,6 +1125,9 @@ impl Connection { let (mut probe, mut fallback): (Option, Option) = (None, None); // Mirrors the live session's estimates while it serves. let mut forward: Option = None; + // The transport of the connected session the policy has yet to establish, published + // only once it does, so a refused session never shows as connected. + let mut pending: Option = None; let mut timer: Option = None; // Consecutive walks that failed since the last session, and whether a dial failed // since then, which tells a failed walk from a session that ended too soon. @@ -1138,8 +1144,7 @@ impl Connection { Ok(crate::client::Dialed { session, transport }) => { tracing::info!(peer = %Endpoint(&url), %transport, "connected"); (failed_walks, dial_failed) = (0, false); - shared.connected(&session, transport); - forward = Some(Forward::new(&session)); + pending = Some(transport); policy.connected(kio::clock::now(), session); if client.on_fallback(&url, transport) { policy.fallback(kio::clock::now()); @@ -1198,6 +1203,12 @@ impl Connection { dial = Some((url, Box::pin(async move { client.dial(attempt).await }))); } // The policy already counted the timeout; dropping the dial cancels it. + Action::Established => { + if let (Some(transport), Some(session)) = (pending.take(), policy.session()) { + shared.connected(session, transport); + forward = Some(Forward::new(session)); + } + } Action::Abandon => { if dial.is_some() { dial_failed = true; @@ -1274,9 +1285,9 @@ impl Connection { /// Wait until a session is established, handing the connection back. /// - /// Returns as soon as a session is live, so the first call waits out the initial dial - /// (surfacing its error if the loop gives up, e.g. on an auth failure or in one-shot - /// mode). Useful when a caller wants dial errors up front rather than through + /// Returns once a session reports [`Status::Connected`], so the first call waits out the + /// initial dial and the peer's answer to its credential, surfacing the error if the loop + /// gives up instead, e.g. when the peer refuses the credential or in one-shot mode. Useful when a caller wants dial errors up front rather than through /// [`closed`](Self::closed). /// /// It consumes and returns the connection so the reconnecting chain, @@ -2044,8 +2055,9 @@ mod tests { let (accepted, mut sessions) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(async move { while let Ok(request) = server.accept().await { - let _ = accepted.send(()); - let _ = request.ok().await; + // Held open: a session the server drops before the client reads its answer + // is a refusal. + let _ = accepted.send(request.ok().await); } }); diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index d316d5e3fa6..36e0677de17 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -277,10 +277,13 @@ async fn sni_test() { let fingerprints = server.certificates().fingerprints(); assert_eq!(fingerprints.len(), 2, "expected both certificates to be served"); - // Keep accepting so the client handshakes complete, then drop the server. + // Keep accepting and holding each session, so the client sees it accepted. let server_handle = tokio::spawn(async move { let mut server = server; - while server.accept().await.is_ok() {} + let mut sessions = Vec::new(); + while let Ok(request) = server.accept().await { + sessions.push(request.ok().await); + } }); let dial = |host_name: &str, fingerprint: &str| { diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index 2176e227d96..d8647d0ebc6 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -2671,10 +2671,14 @@ async fn one_shot_connect_surfaces_the_session_close() { let accepts = Arc::new(AtomicUsize::new(0)); let server_accepts = accepts.clone(); let pub_origin = moq_tokio::origin::spawn(); + // Each session is aborted once the client is established on it: one aborted sooner is a + // refusal, which `established` reports instead. + let (established, mut wait_established) = tokio::sync::mpsc::unbounded_channel::<()>(); let server_handle = tokio::spawn(async move { while let Ok(request) = server.accept().await { server_accepts.fetch_add(1, Ordering::SeqCst); let session = request.with_publisher(&pub_origin).ok().await?; + wait_established.recv().await; session.abort(moq_net::Error::Cancel); } Ok::<_, anyhow::Error>(()) @@ -2693,6 +2697,7 @@ async fn one_shot_connect_surfaces_the_session_close() { .await .expect("connect timed out") .expect("connect failed"); + established.send(()).unwrap(); // The server aborts the session; one-shot mode turns that into the terminal error. kio::clock::timeout(TIMEOUT, connection.closed()) @@ -3346,9 +3351,12 @@ async fn goaway_timeout_force_close_moq_transport_22_quic() { let client = client_config.init(Default::default()).expect("failed to init client"); let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); + // Sent once the client is established, so the one-shot loop is not over before it is. + let (established, wait_established) = tokio::sync::oneshot::channel::<()>(); let server_handle = tokio::spawn(async move { let request = server.accept().await.expect("no incoming connection"); let session = request.with_publisher(&pub_origin).ok().await?; + let _ = wait_established.await; session .goaway(moq_net::goaway::Goaway::new().with_timeout(Some(Duration::from_millis(200)))) @@ -3366,6 +3374,7 @@ async fn goaway_timeout_force_close_moq_transport_22_quic() { // Observe the GOAWAY but do NOT leave. let draining = connection.draining().expect("connected"); + established.send(()).unwrap(); let goaway = kio::clock::timeout(TIMEOUT, kio::wait(|waiter| draining.poll(waiter))) .await .expect("goaway timed out") @@ -3444,10 +3453,10 @@ async fn zero_initial_backoff_still_gives_up_on_a_flapping_peer() { } /// A rejection at the MoQ layer lands after the transport handshake, so the dial -/// itself succeeds and the rejection arrives as the session's close. That close -/// rides the specified UNAUTHORIZED session code, so it decodes back into a typed -/// error rather than an opaque transport string, even in one-shot mode where the -/// session ending is what ends the connection. +/// itself succeeds and the rejection arrives as the session's close, before the client +/// ever reports it connected. That close rides the specified UNAUTHORIZED session code, so +/// it decodes back into a typed error rather than an opaque transport string, which +/// `established` returns. #[tokio::test] async fn session_close_surfaces_a_rejection_code() { let (mut server, addr) = test_server().await; @@ -3460,14 +3469,11 @@ async fn session_close_surfaces_a_rejection_code() { Ok::<_, anyhow::Error>(()) }); - let (_client, connection) = kio::clock::timeout(TIMEOUT, connect_once(test_client(), url)) - .await - .expect("connect timed out") - .expect("connect failed"); - let err = kio::clock::timeout(TIMEOUT, connection.closed()) - .await - .expect("close timed out") - .expect_err("a rejected session must surface as an error"); + let err = expect_connect_err( + kio::clock::timeout(TIMEOUT, connect_once(test_client(), url)) + .await + .expect("connect timed out"), + ); assert_connect_error(&err, moq_tokio::ConnectError::Unauthorized); server_handle.abort(); @@ -3475,7 +3481,8 @@ async fn session_close_surfaces_a_rejection_code() { /// The same rejection of a first session under the reconnect loop ends it with the code, /// as a refused first dial does: on these versions the refusal arrives after ALPN, so the -/// session connected first, yet redialing would only present the same credential again. +/// transport connected first, yet redialing would only present the same credential again. +/// The refused session never reports as connected. #[tokio::test] async fn reconnect_ends_on_a_first_session_rejection() { for code in [ @@ -3501,6 +3508,7 @@ async fn reconnect_ends_on_a_first_session_rejection() { .await .unwrap_or_else(|_| panic!("{code:?} kept the loop redialing")) .expect_err("a rejected first session must surface as an error"); + assert_eq!(connection.connects(), 0, "{code:?} showed the refused session as connected"); server_handle.abort(); let mut count = 0; while accepts.try_recv().is_ok() { From 3f6a5d7678e68064c243e995e064144c958a8987 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:12:06 -0800 Subject: [PATCH 09/19] test: binding reconnect tests drop a session once it reports connected Co-Authored-By: Claude Opus 5.5 --- go/wrapper/reconnect_test.go | 7 ++++--- py/moq-rs/tests/test_server.py | 5 ++--- rs/libmoq/src/test.rs | 9 ++------- rs/moq-ffi/src/test.rs | 20 +++++++++++++------- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/go/wrapper/reconnect_test.go b/go/wrapper/reconnect_test.go index 574678961d2..36cae8ca0e0 100644 --- a/go/wrapper/reconnect_test.go +++ b/go/wrapper/reconnect_test.go @@ -229,6 +229,10 @@ func TestReconnectAcrossRelayRestart(t *testing.T) { if frame, err := trackConsumer.ReadFrame(ctx); err != nil || frame == nil { t.Fatalf("frame before restart: frame=%+v err=%v", frame, err) } + // Connected once the relay accepted each session, so the drop below redials rather than + // ending them as refusals. + waitConnects(t, ctx, publisher.Session(), 1) + waitConnects(t, ctx, subscriber.Session(), 1) if state := subscriber.Session().State(); !isConnected(state) { t.Fatalf("not serving before any drop: %+v", state) } @@ -236,9 +240,6 @@ func TestReconnectAcrossRelayRestart(t *testing.T) { t.Fatal("a live session must report its negotiated version") } - // Past the 1s healthy bar, so the drop redials rather than ending the sessions. - time.Sleep(time.Second) - // Drop every session and the listener, then bind a replacement on the same // port: a deploy, with an origin the replacement has never seen. current = current.restart(t, ctx) diff --git a/py/moq-rs/tests/test_server.py b/py/moq-rs/tests/test_server.py index 1c79515a072..f670e65360a 100644 --- a/py/moq-rs/tests/test_server.py +++ b/py/moq-rs/tests/test_server.py @@ -139,10 +139,9 @@ async def accept_loop() -> None: assert state.connects == 1 assert session.stats().version is not None - # Kill the transport under the client, like a relay restart, once past - # the 1s healthy bar so the drop redials rather than ending the session. + # Kill the transport under the client, like a relay restart. CONNECTED + # means the peer accepted the session, so the drop redials. await asyncio.wait_for(accepted.wait(), timeout=10) - await asyncio.sleep(1) sessions[0].cancel(0) # Nothing accepts the redial until the gate opens, so DISCONNECTED diff --git a/rs/libmoq/src/test.rs b/rs/libmoq/src/test.rs index efc93bc8954..76dc5bb4863 100644 --- a/rs/libmoq/src/test.rs +++ b/rs/libmoq/src/test.rs @@ -9795,13 +9795,8 @@ fn a_credential_rides_every_reconnect() { let _first = first.accept(moq_net::auth::Grant::all()); assert_eq!(answer.recv().0, 0); - // The server ends the session once it was healthy; the client redials and presents the - // token again. - #[expect( - clippy::disallowed_methods, - reason = "the test thread has no runtime; the C API's session ages on the real clock" - )] - std::thread::sleep(Duration::from_secs(1)); + // The server ends the session the client saw connected, so accepted; the client redials + // and presents the token again. server.admit(); let again = server.token(); assert_eq!(again.token().as_ref(), b"kept", "the reconnect presents it again"); diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index dc1e1643956..e8949a2060e 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7705,8 +7705,16 @@ async fn session_state_names_a_failing_first_dial() { )); assert_eq!(accepted.state().revision, terminal.revision); assert!(accepted.state_changed(terminal.revision).await.is_err()); - // Whether the client redials depends on whether the server's AUTH_OK reached it before the - // close, which this test does not order, so it only ends the client's session. + // The client reported the session connected, so the peer had accepted it, and its end is + // an outage the client redials rather than a refusal. + let dropped = kio::clock::timeout( + TIMEOUT, + wait_phase(&session, |phase| matches!(phase, MoqConnectionPhase::Disconnected { .. })), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(dropped.connects, 1); session.cancel(0); accepted.cancel(0); server.cancel(); @@ -7796,8 +7804,7 @@ async fn client_reconnects_and_resumes_announcements() { .await .expect("first session timed out") .expect("accept task gone"); - // Past the 1s healthy bar, so the drop redials rather than ending the session. - kio::clock::sleep(Duration::from_secs(1)).await; + // Connected means the peer accepted the session, so the drop redials rather than ending it. first.cancel(0); let status = kio::clock::timeout( @@ -8048,9 +8055,8 @@ async fn cancelled_state_wait_does_not_swallow_the_next_transition() { Ok(status) => panic!("no transition was pending, got {status:?}"), } - // The transition the abandoned call would have eaten, once past the 1s healthy bar so - // the session redials rather than ends. - kio::clock::sleep(Duration::from_secs(1)).await; + // The transition the abandoned call would have eaten: a disconnect, since the session was + // accepted and so redials rather than ends. server_session.cancel(0); let status = kio::clock::timeout(TIMEOUT, cs.state_changed(status.revision)) From 9d7c4fd76f967417ad2296bb020fa98c134931e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:13:35 -0800 Subject: [PATCH 10/19] feat(net): Reload reports connected once the peer accepts the session Co-Authored-By: Claude Opus 5.5 --- js/net/src/connection/reload.test.ts | 8 +++++++- js/net/src/connection/reload.ts | 26 +++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 120c5161f91..975739f9e62 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -476,8 +476,11 @@ test("a first session refused before its credential is accepted stops the URL, e tokens, backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); + const statuses: string[] = []; + const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); try { - await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + await clock.drive(waitFor(dialed, (dialed) => dialed.length > 0)); + await clock.advance(Duration.millis(10)); tokens.set(["new"]); resolve(); // The refusal arrives once the session is live, before the server's SETUP, as a relay @@ -488,7 +491,10 @@ test("a first session refused before its credential is accepted stops the URL, e await clock.advance(Duration.millis(100)); expect(dialed.peek()).toEqual(["old"]); expect(reload.established.peek()).toBeUndefined(); + // The refused session never read as connected. + expect(statuses).not.toContain("connected"); } finally { + unsubscribe(); reload.close(); globalThis.WebTransport = original; } diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 38250056d87..3e7788adc7a 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -629,11 +629,31 @@ export class Reload { } current = connection; this.#verdict.connected(connection); + connected = now(); - // The replacement serves now; a predecessor keeps draining its groups in flight. - this.#serve(connection); + // Serve only once the session is established: the peer accepted its credential, + // drained it with a GOAWAY, or it stayed up past the healthy bar, the only sign + // on a version that never answers (moq-lite-03/04). Until then it may still be a + // refusal, which never reads as connected. A close first is handled below. + const bar = this.#pacing().healthy; + const acceptance = wireOf(connection).auth.accepted; + if (!acceptance.peek()) { + const waiting = new AbortController(); + const uptime = sleep(bar, { signal: waiting.signal }).catch(() => {}); + await effect + .race(Signal.race(acceptance), uptime, connection.closed, connection.draining) + .finally(() => waiting.abort()); + if (signal.aborted) return; + } + if ( + acceptance.peek() || + connection.draining.peek() !== undefined || + now().since(connected).ge(bar) + ) { + // The replacement serves now; a predecessor keeps draining its groups in flight. + this.#serve(connection); + } this.#setDialing(false); - connected = now(); // A cancelled effect resolves undefined, so the sentinel tells the session // closing (null for clean, an Error otherwise) apart from this run being From e91df3340ad646a08f3d36d35d7f2c509ba9f037 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:22:43 -0800 Subject: [PATCH 11/19] fix(net): a predecessor's retirement mid-dial stays offline until the replacement is established Handover.tla models the per-session Established step and checks that a connection reads as connected only once its session is established. Co-Authored-By: Claude Opus 5.5 --- doc/lib/index.md | 2 +- go/wrapper/session.go | 4 +- js/net/src/connection/reload.ts | 6 +- rs/libmoq/src/api.rs | 3 +- rs/moq-ffi/src/session.rs | 3 +- rs/moq-net/src/reconnect.rs | 8 +- rs/moq-net/src/reconnect/tests.rs | 16 ++- rs/moq-net/verify/reconnect/Handover.cfg | 1 + rs/moq-net/verify/reconnect/Handover.tla | 105 +++++++++++++----- .../verify/reconnect/RestartHandoff.cfg | 1 + 10 files changed, 101 insertions(+), 48 deletions(-) diff --git a/doc/lib/index.md b/doc/lib/index.md index 32b9a53a4d5..2426b15e63e 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -67,7 +67,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). +- **Reconnect** automatically with backoff when the transport drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. A session reads as connected only once the peer accepts its credential (or, on moq-lite 03/04, which never answer it, once it stays up 1s), so a refused one never does. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/go/wrapper/session.go b/go/wrapper/session.go index a70218a474d..d4c8031a8bc 100644 --- a/go/wrapper/session.go +++ b/go/wrapper/session.go @@ -46,7 +46,9 @@ type ConnectionPhaseConnecting struct { Failure error } -// ConnectionPhaseConnected means a session is serving. +// ConnectionPhaseConnected means a session is serving: its peer accepted the +// credential, or on moq-lite 03/04, which never answer it, the session stayed up 1s. +// A refused session never reaches this phase. type ConnectionPhaseConnected struct { // Version names the live session's negotiated protocol, like "moq-lite-03". Version string diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 3e7788adc7a..be142b2d762 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -219,7 +219,11 @@ export class Reload { */ tokens: Signal; - /** Current connection status. */ + /** + * Current connection status. A session reads as connected only once its peer accepts the + * credential (on moq-lite-03/04, which never answer it, once it stays up past the healthy + * bar), so a refused one never does. + */ status = new Signal("disconnected"); /** diff --git a/rs/libmoq/src/api.rs b/rs/libmoq/src/api.rs index ccd76d51f8d..245da762a50 100644 --- a/rs/libmoq/src/api.rs +++ b/rs/libmoq/src/api.rs @@ -1187,7 +1187,8 @@ pub struct moq_announce_event { pub enum moq_connection_phase { /// No connection has been established yet. MOQ_CONNECTION_PHASE_CONNECTING = 0, - /// A connection is serving. + /// A connection is serving: its peer accepted the credential, or on moq-lite 03/04, which + /// never answer it, the session stayed up 1s. A refused session never reaches this phase. MOQ_CONNECTION_PHASE_CONNECTED = 1, /// No connection is serving while the session redials. MOQ_CONNECTION_PHASE_DISCONNECTED = 2, diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 4348ae6680c..f72e2b370c7 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -737,7 +737,8 @@ pub enum MoqConnectionPhase { /// The latest failed dial, or none before the first one fails. failure: Option, }, - /// A session is serving. + /// A session is serving: its peer accepted the credential, or on moq-lite 03/04, which + /// never answer it, the session stayed up 1s. A refused session never reaches this phase. Connected { /// The live session's negotiated version, spelled like `moq-lite-03`. version: String, diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 8ff882fd044..31f038b845d 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -659,7 +659,10 @@ impl + From> Reco true => handover, false => earliest(Some(healthy), handover), }; - match live.probe.as_mut().map(|probe| probe.poll(now)) { + // Only an established session is left for the preferred transport: one that may + // still be a refusal is no session to hand over. + let probe = live.probe.as_mut().filter(|_| live.established); + match probe.map(|probe| probe.poll(now)) { None => return Action::Wait(handover), Some(ProbePoll::Wait(at)) => return Action::Wait(earliest(at, handover)), Some(ProbePoll::Abandon) => return Action::Abandon, @@ -703,9 +706,6 @@ impl + From> Reco if let Some(tokens) = &self.tokens { tokens.connected(&session); } - // A predecessor that retired mid-dial queued an offline this session now contradicts. - // Its `Retired` stays, so the caller still drops the predecessor. - self.queue.retain(|action| !matches!(action, Action::Offline(_))); self.walk = Walk::default(); self.phase = Phase::Live(Box::new(Live { goaway: Some(session.draining()), diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 0e4ed143fc7..822ec910014 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -949,10 +949,10 @@ async fn the_handover_takes_the_earlier_deadline() { ); } -/// A replacement that connects after the predecessor retired mid-dial serves, so the -/// offline the retirement queued no longer holds and is never reported. +/// A replacement that connects after the predecessor retired mid-dial does not serve until +/// it is established, so the offline the retirement queued is still reported first. #[kio::test] -async fn a_connect_cancels_the_offline_a_retirement_queued() { +async fn a_retirement_mid_dial_is_offline_until_the_replacement_is_established() { const HANDOVER: Duration = Duration::from_millis(500); let peer = url("tcp://peer:1/"); let mut config = Config::default(); @@ -973,12 +973,10 @@ async fn a_connect_cancels_the_offline_a_retirement_queued() { let (client, _server) = handshake().await; machine.connected(kio::clock::now(), client); - assert!( - kio::clock::timeout(Duration::from_secs(60), next(&mut machine)) - .await - .is_err(), - "a stale offline followed the connect" - ); + let action = next_any(&mut machine).await; + assert!(matches!(action, Action::Offline(None)), "{action:?}"); + let action = next_any(&mut machine).await; + assert!(matches!(action, Action::Established), "{action:?}"); assert!(machine.session().is_some()); } diff --git a/rs/moq-net/verify/reconnect/Handover.cfg b/rs/moq-net/verify/reconnect/Handover.cfg index ea4c0c89aa3..0bdc4e68c55 100644 --- a/rs/moq-net/verify/reconnect/Handover.cfg +++ b/rs/moq-net/verify/reconnect/Handover.cfg @@ -12,6 +12,7 @@ VIEW View INVARIANTS NoStaleSession StatusTruthful + ConnectedMeansEstablished OfflineMeansNothingServes NoOrphan PresenceBalanced diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index d541512369c..4f5c60fd19a 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -73,12 +73,13 @@ vars == <> \* whether a `start_close` began, the GOAWAY it got, its age, whether it \* carries no subscription or fetch, whether the policy acted on its GOAWAY, \* whether we sent our own, whether the policy watches its GOAWAY as the -\* predecessor (`Predecessor::redirect`, set when a probe moved us off it), and +\* predecessor (`Predecessor::redirect`, set when a probe moved us off it), \* whether the peer accepted its credential (`Session::poll_accepted`), which a -\* close leaves as it was. +\* close leaves as it was, and whether the policy reported it established +\* (`Live::established`), the only point a caller serves it from. Fresh(e) == [ep |-> e, open |-> TRUE, by |-> "", closing |-> FALSE, goaway |-> NoGoaway, age |-> 0, idle |-> FALSE, heeded |-> FALSE, - sent |-> FALSE, watched |-> FALSE, accepted |-> FALSE] + sent |-> FALSE, watched |-> FALSE, accepted |-> FALSE, served |-> FALSE] Unused == [Fresh(Start) EXCEPT !.open = FALSE] Timers == {"dial", "backoff", "probe", "handover"} @@ -116,6 +117,16 @@ Abort(sx, s) == IF s # None /\ sx[s].open (* predecessor once the pass ends, which `Reconnect::predecessor` reads. *) (***************************************************************************) +\* `Action::Established`: `Shared::connected` with the live session `s`. +OnEstablished(s) == + /\ ended' = IF lc = "live" THEN ended + 1 ELSE ended + /\ spred' = IF lc \in {"live", "offline"} THEN spred ELSE None + /\ started' = started + 1 + /\ lc' = "live" + /\ lcSess' = s + /\ migrating' = FALSE + /\ UNCHANGED probing + \* `Shared::predecessor`. Mirror(p) == spred' = IF lc \in {"live", "offline"} THEN p ELSE spred @@ -158,8 +169,10 @@ OnDone == /\ probing' = FALSE /\ UNCHANGED started +\* `p` is the predecessor, or for `Established` the session it serves. Apply(a, p) == - CASE a = "Migrating" -> OnMigrating(p) + CASE a = "Established" -> OnEstablished(p) + [] a = "Migrating" -> OnMigrating(p) [] a = "Retired" -> OnRetired [] a = "Offline" -> OnOffline [] a = "Done" -> OnDone @@ -189,6 +202,13 @@ Pop == dial, dialTo, misdials>> /\ UNCHANGED lifeVars +\* A replacement serves only once established: until then it may be a refusal. +Serving == phase = "live" /\ sess[live].served + +\* The live session is due its `Action::Established`, which `poll` returns +\* before reading its GOAWAY or close. +EstablishDue == phase = "live" /\ ~sess[live].served /\ (sess[live].accepted \/ sess[live].age >= 1) + \* The predecessor closed or overstayed its handover: `Predecessor::poll`. PredDue == pred # None /\ (~sess[pred].open \/ timer["handover"]) @@ -197,7 +217,7 @@ Retire == /\ PredDue /\ sess' = Abort(sess, pred) /\ pred' = None - /\ Emit(IF phase = "live" THEN <<"Retired">> ELSE <<"Retired", "Offline">>, None) + /\ Emit(IF Serving THEN <<"Retired">> ELSE <<"Retired", "Offline">>, None) /\ UNCHANGED <> /\ UNCHANGED lifeVars @@ -208,7 +228,7 @@ Retire == IdleClose == /\ queue = <<>> /\ ~PredDue - /\ phase = "live" + /\ Serving /\ pred # None /\ sess[pred].idle /\ ~sess[pred].closing @@ -312,24 +332,27 @@ BackoffEnds == \* `Reconnect::end` for a GOAWAY naming `uri` off the live session `s`. The \* GOAWAY establishes the policy however soon it came, as `Reconnect::poll` does \* on its receipt: the peer accepted the session, which a refusal never does. +\* A session not yet reported established is, first. EndGoaway(s, uri) == + LET first == IF sess[s].served THEN <<>> ELSE <<"Established">> IN IF uri = Refused THEN \* A refused redirect is terminal. /\ Done - /\ Emit(<<"Done">>, None) + /\ Emit(first \o <<"Done">>, s) /\ established' = TRUE - /\ UNCHANGED <> + /\ sess' = [sess EXCEPT ![s].served = TRUE] + /\ UNCHANGED <> ELSE /\ established' = TRUE /\ target' = IF uri = "" THEN target ELSE uri \* An older predecessor is retired outright, with no `Retired`. - /\ sess' = [Abort(sess, pred) EXCEPT ![s].heeded = TRUE] + /\ sess' = [Abort(sess, pred) EXCEPT ![s].heeded = TRUE, ![s].served = TRUE] /\ pred' = s /\ live' = None \* A healthy session hands over at once; one that drained too soon backs off. /\ phase' = IF sess[s].age >= 1 THEN "start" ELSE "backoff" /\ timer' = [timer EXCEPT !["handover"] = FALSE, !["backoff"] = FALSE] /\ probe' = "none" - /\ Emit(<<"Migrating">>, s) + /\ Emit(first \o <<"Migrating">>, s) /\ UNCHANGED watch \* `Reconnect::closed`. Until a session has been established, a close ends the @@ -361,17 +384,33 @@ WaitOut(s) == /\ Restart = "handoff" /\ sess[s].age = 0 +\* `Action::Established`, from the peer's acceptance or the healthy bar. +Establish == + /\ queue = <<>> + /\ ~PredDue + /\ EstablishDue + /\ sess' = [sess EXCEPT ![live].served = TRUE] + /\ established' = TRUE + /\ Emit(<<"Established">>, live) + /\ UNCHANGED <> + LiveEnds == /\ queue = <<>> /\ ~PredDue /\ phase = "live" + /\ ~EstablishDue /\ \/ /\ Goaway(live) /\ WaitOut(live) \* Drop the latch and serve until the peer closes. The GOAWAY establishes \* the policy here too, so that close retries. /\ watch' = FALSE - /\ UNCHANGED <> - /\ CallerSame + /\ UNCHANGED <> + /\ IF sess[live].served + THEN /\ UNCHANGED <> + /\ CallerSame + ELSE /\ sess' = [sess EXCEPT ![live].served = TRUE] + /\ Emit(<<"Established">>, live) /\ established' = TRUE \/ /\ Goaway(live) /\ ~WaitOut(live) @@ -381,11 +420,12 @@ LiveEnds == /\ EndClosed /\ UNCHANGED <> -\* `Reconnect::probe`: ask for a probe of the preferred transport. +\* `Reconnect::probe`: ask for a probe of the preferred transport, only off an +\* established session. ProbeAsk == /\ queue = <<>> /\ ~PredDue - /\ phase = "live" + /\ Serving /\ ~Goaway(live) /\ sess[live].open /\ probe = "due" @@ -413,14 +453,15 @@ ProbeAbandon == /\ UNCHANGED lifeVars Policy == Pop \/ Retire \/ IdleClose \/ FollowPred \/ Dial \/ Abandon \/ BackoffEnds - \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon + \/ Establish \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon (***************************************************************************) (* What `run` reports back to the policy. *) (***************************************************************************) -\* A dial connected: `Shared::connected`, then `Reconnect::connected`, and -\* `Reconnect::fallback` when it landed on the fallback transport. +\* A dial connected: `Reconnect::connected`, and `Reconnect::fallback` when it +\* landed on the fallback transport. Nothing is published until the policy +\* reports the session established. Connected == /\ dial # "none" /\ used < MaxSessions @@ -428,20 +469,15 @@ Connected == /\ used' = s /\ dial' = "none" /\ sess' = [sess EXCEPT ![s] = Fresh(dialTo)] - \* `Shared::connected` - /\ ended' = IF lc = "live" THEN ended + 1 ELSE ended - /\ spred' = IF lc \in {"live", "offline"} THEN spred ELSE None - /\ started' = started + 1 - /\ lc' = "live" - /\ lcSess' = s - /\ migrating' = FALSE - \* `Reconnect::connected`, ignored unless a dial is in flight. It drops - \* the `Offline` a predecessor that retired mid-dial left queued. + /\ UNCHANGED <> + \* `Reconnect::connected`, ignored unless a dial is in flight. An `Offline` + \* a predecessor that retired mid-dial left queued stands: nothing serves + \* until the session is established. /\ IF phase = "dialing" THEN /\ phase' = "live" /\ live' = s /\ watch' = TRUE - /\ queue' = SelectSeq(queue, LAMBDA a : a # "Offline") + /\ UNCHANGED queue /\ \E fallback \in BOOLEAN : probe' = IF fallback THEN "wait" ELSE "none" ELSE UNCHANGED <> @@ -585,7 +621,8 @@ Spec == Init /\ [][Next]_vars \* field after it closes: `LiveEnds` still reads its GOAWAY and age. Seen(s) == IF sess[s].open \/ s = live THEN sess[s] - ELSE [Unused EXCEPT !.by = sess[s].by, !.closing = sess[s].closing] + ELSE [Unused EXCEPT !.by = sess[s].by, !.closing = sess[s].closing, + !.served = sess[s].served] Due(t) == CASE t = "dial" -> timer[t] /\ phase = "dialing" [] t = "backoff" -> timer[t] /\ phase = "backoff" @@ -616,11 +653,19 @@ NoStaleSession == StatusTruthful == Parked => /\ lc = "live" /\ ~migrating => live = lcSess - /\ lc = "live" /\ migrating => pred = lcSess /\ phase # "live" - /\ phase = "live" => lc = "live" /\ lcSess = live /\ ~migrating + /\ lc = "live" /\ migrating => pred = lcSess /\ ~Serving + /\ Serving => lc = "live" /\ lcSess = live /\ ~migrating /\ (lc = "closed") = (phase = "done") /\ lc # "closed" => spred = pred +\* A session reads as connected only once established: the peer accepted its +\* credential, it stayed up past the healthy bar, or the peer drained it with +\* a GOAWAY. A refusal never does. +ConnectedMeansEstablished == + /\ Parked /\ lc = "live" => sess[lcSess].served + /\ \A s \in 1..used : + sess[s].served => sess[s].accepted \/ sess[s].age >= 1 \/ sess[s].goaway # NoGoaway + \* `Action::Offline`: nothing serves anymore. A predecessor closing gracefully \* already carries nothing and takes no new work. OfflineMeansNothingServes == diff --git a/rs/moq-net/verify/reconnect/RestartHandoff.cfg b/rs/moq-net/verify/reconnect/RestartHandoff.cfg index 75377d9e5a3..39214d15354 100644 --- a/rs/moq-net/verify/reconnect/RestartHandoff.cfg +++ b/rs/moq-net/verify/reconnect/RestartHandoff.cfg @@ -14,6 +14,7 @@ VIEW View INVARIANTS NoStaleSession StatusTruthful + ConnectedMeansEstablished OfflineMeansNothingServes NoOrphan PresenceBalanced From 91a60fb9b70a22ef1702c87c375b4f9e11090676 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:23:09 -0800 Subject: [PATCH 12/19] quest: finish credential-established Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 9 ++--- quest/m0/nightly/credential-established.md | 42 ---------------------- 2 files changed, 5 insertions(+), 46 deletions(-) delete mode 100644 quest/m0/nightly/credential-established.md diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index e9ef0a6ec73..52c0cd2f01f 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -15,6 +15,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | --- | --- | --- | | `NoStaleSession` | The session a `Connection` reports as live is never one the policy aborted or started closing. | An app opens subscriptions on a session that is going away. | | `StatusTruthful` | Connected is on the policy's live session, Migrating on its predecessor while nothing else is live, Closed exactly when the policy is done, and the predecessor `Connection::close` drains is the policy's. | Status or `close` acts on the wrong session, or a closed loop looks alive. | +| `ConnectedMeansEstablished` | A `Connection` reports a session only once the policy reported it established (`Action::Established`): its peer accepted the credential, it stayed up past the healthy bar, or the peer drained it with a GOAWAY. | A refused session reads as connected, and `established()` returns `Ok` for it. | | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | @@ -28,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 5.2M distinct states, under a minute. The second checks the same rules under `Restart::Handoff` over three sessions: about 1.2M states. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 6.5M distinct states, about a minute. The second checks the same rules under `Restart::Handoff` over three sessions: about 1.1M states. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -40,17 +41,17 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `392cdb6d0240e83ebfc4111518ee9cc6cbb6ff09`. Run `git log 392cdb6d02.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `e91df3340ad646a08f3d36d35d7f2c509ba9f037`. Run `git log e91df3340a.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | -| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | +| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `Establish`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | | `rs/moq-net/src/reconnect.rs` `established`, set by `end`, by a GOAWAY or the session's acceptance in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "handoff"` (`RestartHandoff.cfg`) | | `rs/moq-net/src/reconnect/redirect.rs` `Redirect::target` | `Refused` and `Endpoints` | -| `rs/moq-tokio/src/connection.rs` `run`, `Shared::connected`, `migrating`, `retired`, `offline`, `end` | `Connected`, `Apply`, `OnMigrating`, `OnRetired`, `OnOffline`, `OnDone` | +| `rs/moq-tokio/src/connection.rs` `run`, `Shared::connected`, `migrating`, `retired`, `offline`, `end` | `Connected`, `Apply`, `OnEstablished`, `OnMigrating`, `OnRetired`, `OnOffline`, `OnDone` | ## Not covered diff --git a/quest/m0/nightly/credential-established.md b/quest/m0/nightly/credential-established.md deleted file mode 100644 index a8e80ddc771..00000000000 --- a/quest/m0/nightly/credential-established.md +++ /dev/null @@ -1,42 +0,0 @@ -# [S] Binding tests drop a session once the client saw it accepted - -## Goal - -The tests [moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817) gave -real 1 s waits (Go `go/wrapper/reconnect_test.go`, Python -`py/moq-rs/tests/test_server.py`, `rs/moq-ffi/src/test.rs`, `rs/libmoq/src/test.rs`), -and the OBS tests in draft -[moq-dev/moq#9016](https://github.com/moq-dev/moq/pull/9016), drop the session -as soon as the client has seen the peer accept its credential, instead of -waiting out `Backoff::healthy`. No wire change. - -## Plan - -[moq-dev/moq#9038](https://github.com/moq-dev/moq/pull/9038) made the -reconnect policy (Rust `Reconnect`, JS `Reload`, every binding through -`moq-ffi`) count a session as established once the peer accepts the -connection credential: its SETUP, or on lite-07 the AUTH_OK to the -connection's own credential. moq-lite 03/04 keep the 1 s uptime bar. - -What is left: a test can drop a session the moment the *server* accepted it, -but the client counts it only once it has *read* that answer, and a server -close can overtake it. No binding reports that moment: `Connected` fires when -the session goes live, before the answer. So the waits stay until the client -can observe acceptance. - -## Waiting - -- Maintainer: how should a binding observe that the peer accepted the session? - (a) Recommended: moq-tokio's `Connection` reports Connected, and - `established()` resolves, only once the peer accepted the credential (or - the session passed `healthy` on moq-lite 03/04), and every binding's - `Connected` phase follows. A refused session then never reads as - connected, and `established().await` returns the refusal instead of `Ok`. - It's a behavior change in every binding, but no API change. (b) Add an - `accepted` flag to the state snapshot in every binding, which is new API that only - tests need today. (c) Keep the 1 s waits and delete this quest. Check: the - answer is recorded in this Plan. - -## Related - -- [Connect policy state](/quest/m4/connect-policy-state.md) - moves the same policy into one place per language From d70adc3ddcacf3695919f35f816ede7eb0cae42e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 18:53:45 -0800 Subject: [PATCH 13/19] feat(tokio): established resolves once any session was, and relay drain tests read an arrival's GOAWAY deterministically Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/test.rs | 4 +++- rs/moq-net/src/reconnect/tests.rs | 6 +++++- rs/moq-relay/src/shutdown.rs | 9 ++++---- .../tests/integration/runtime_uring.rs | 15 ++++++++----- .../tests/integration/shutdown_signal.rs | 4 +++- rs/moq-tokio/src/connection.rs | 21 ++++++++++++++----- rs/moq-tokio/tests/integration/broadcast.rs | 6 +++++- 7 files changed, 46 insertions(+), 19 deletions(-) diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index e8949a2060e..4fb2bfdf1a1 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7709,7 +7709,9 @@ async fn session_state_names_a_failing_first_dial() { // an outage the client redials rather than a refusal. let dropped = kio::clock::timeout( TIMEOUT, - wait_phase(&session, |phase| matches!(phase, MoqConnectionPhase::Disconnected { .. })), + wait_phase(&session, |phase| { + matches!(phase, MoqConnectionPhase::Disconnected { .. }) + }), ) .await .unwrap() diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 822ec910014..a1159c6b6be 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -904,7 +904,11 @@ async fn an_unanswered_replacement_does_not_serve() { assert_eq!(run.events, ["established", "migrating"]); assert!(matches!(next_any(&mut machine).await, Action::Retired)); assert!(matches!(next_any(&mut machine).await, Action::Offline(None))); - assert_eq!(kio::clock::now() - connected, HANDOVER, "the idle predecessor closed early"); + assert_eq!( + kio::clock::now() - connected, + HANDOVER, + "the idle predecessor closed early" + ); assert!(matches!(next_any(&mut machine).await, Action::Established)); assert_eq!(kio::clock::now() - connected, healthy); } diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 72fb6bd95a3..41a53d70956 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -653,8 +653,8 @@ mod tests { .await .expect("the admission outlived the drain deadline"); running.await.unwrap().unwrap(); - // The client may already count the session established (SETUP precedes - // auth); either way the refusal ends it. + // The paused clock may carry the session past the healthy bar before the admission + // ends, which establishes it; either way the refusal ends it. if let Ok(connection) = connecting.await.unwrap() { assert!(connection.closed().await.is_err(), "an abandoned admission is refused"); } @@ -683,10 +683,9 @@ mod tests { let all = moq_auth::Pattern::all; let (_lease, consumer) = moq_auth::lease::Producer::new(moq_auth::Grant::new(all(), all())); admission.grant(consumer); + // The relay accepts the session and drains it at once, so the one-shot loop may already + // be over: only a GOAWAY ends it cleanly, where a refusal or a close is an error. let connection = connecting.await.unwrap().unwrap(); - let draining = connection.draining().unwrap(); - let goaway = kio::wait(|waiter| draining.poll(waiter)).await.unwrap(); - assert_eq!(goaway.uri(), ""); connection.closed().await.unwrap(); running.await.unwrap().unwrap(); } diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index 7088f0fae1d..15a477cbdc0 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -447,7 +447,14 @@ async fn drain_on_the_trigger(version: Option) { // Hold the next real connection at authentication. Its admission keeps the // drain open even after the first session leaves, without an unrelated peer. - let arriving = tokio::spawn(connect(client, url)); + // Reconnecting, so the GOAWAY that lands with the relay's acceptance leaves the drained + // session readable as the predecessor, where a one-shot loop ends on it. + let arriving = tokio::spawn(async move { + kio::clock::timeout(TIMEOUT, client.with_reconnect(true).connect(url).established()) + .await + .expect("connect timeout") + .expect("connect failed") + }); let pending = admissions.next().await.expect("pending admission"); // The relay fixes its deadline inside `start`, so these bracket it: what is // left of each bounds what the relay tells the arrival from below and above. @@ -506,10 +513,8 @@ async fn drain_on_the_trigger(version: Option) { "{label}: an arrival mid-drain got {timeout:?}, short of the {least:?} left of the window" ); } - kio::clock::timeout(TIMEOUT, arrival.closed()) - .await - .unwrap_or_else(|_| panic!("{label}: the arrival never closed")) - .expect("a one-shot session leaves cleanly on GOAWAY"); + // A reconnecting arrival would redial the draining relay, so leave instead. + drop(arrival); // `run` joins the worker threads before returning, so returning at all is // the clean join; the rebind proves every worker let go of the port. diff --git a/rs/moq-relay/tests/integration/shutdown_signal.rs b/rs/moq-relay/tests/integration/shutdown_signal.rs index cf95cae703e..a80b6a84e35 100644 --- a/rs/moq-relay/tests/integration/shutdown_signal.rs +++ b/rs/moq-relay/tests/integration/shutdown_signal.rs @@ -349,7 +349,9 @@ async fn a_session_arriving_mid_drain_gets_what_is_left_inner() { // then told to leave at once. kio::clock::sleep(DRAIN_TIMEOUT / 2).await; let dialed = kio::clock::now(); - let arrival = connect(&client, port).await; + // Reconnecting, so the GOAWAY that lands with the relay's acceptance leaves the drained + // session readable as the predecessor, where a one-shot loop ends on it. + let arrival = connect(&client.clone().with_reconnect(true), port).await; let draining = arrival.draining().expect("connected"); let goaway = kio::clock::timeout(Duration::from_secs(1), kio::wait(|waiter| draining.poll(waiter))) .await diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index f550bf6a586..6e87ea83ce5 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -1270,24 +1270,27 @@ impl Connection { /// Poll until a session is established (the first connect, or the current one). /// /// `Ready(Ok(session))` while a session is live (during a GOAWAY handover this is the - /// old session, which still serves), `Ready(Err)` once the loop has stopped, `Pending` + /// old session, which still serves), or once the loop has stopped after one was, since + /// a peer can accept a one-shot session and end it before this is polled. `Ready(Err)` + /// once the loop has stopped without one, as on a refused credential, `Pending` /// otherwise. pub fn poll_established(&self, waiter: &kio::Waiter) -> Poll> { match ready!(self.state.poll(waiter, |state| match &state.lifecycle { Lifecycle::Live(_) => Poll::Ready(Ok(())), - Lifecycle::Closed(_) => Poll::Ready(Err(terminal(state))), + Lifecycle::Closed(_) => Poll::Ready(stopped(state)), Lifecycle::Connecting { .. } | Lifecycle::Offline { .. } => Poll::Pending, })) { Ok(res) => Poll::Ready(res), - Err(state) => Poll::Ready(Err(terminal(&state))), + Err(state) => Poll::Ready(stopped(&state)), } } /// Wait until a session is established, handing the connection back. /// /// Returns once a session reports [`Status::Connected`], so the first call waits out the - /// initial dial and the peer's answer to its credential, surfacing the error if the loop - /// gives up instead, e.g. when the peer refuses the credential or in one-shot mode. Useful when a caller wants dial errors up front rather than through + /// initial dial and the peer's answer to its credential. If the loop gives up before any + /// session did, as when the peer refuses the credential, it returns that error instead. + /// Useful when a caller wants dial errors up front rather than through /// [`closed`](Self::closed). /// /// It consumes and returns the connection so the reconnecting chain, @@ -1580,6 +1583,14 @@ fn poll_forward(bw: &mut Option, out: &BandwidthProducer, wai /// this reports [`Error::Stopped`]: [`Connection::closed`] treats that as the `Ok` /// it is, and a watcher that only has an error to go on can still tell it apart /// from a connection that failed. +/// What [`Connection::poll_established`] reports for a stopped loop: whether any session was. +fn stopped(state: &State) -> crate::Result<()> { + match state.connects { + 0 => Err(terminal(state)), + _ => Ok(()), + } +} + fn terminal(state: &State) -> Error { match &state.lifecycle { Lifecycle::Closed(Some(err)) => err.clone(), diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index d8647d0ebc6..3a7be1066d8 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -3508,7 +3508,11 @@ async fn reconnect_ends_on_a_first_session_rejection() { .await .unwrap_or_else(|_| panic!("{code:?} kept the loop redialing")) .expect_err("a rejected first session must surface as an error"); - assert_eq!(connection.connects(), 0, "{code:?} showed the refused session as connected"); + assert_eq!( + connection.connects(), + 0, + "{code:?} showed the refused session as connected" + ); server_handle.abort(); let mut count = 0; while accepts.try_recv().is_ok() { From 8bd99441195aaed56acded99ad66c30f1d13eced Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 19:32:03 -0800 Subject: [PATCH 14/19] test(interop): a refused WebSocket session never reads as connected Co-Authored-By: Claude Opus 5.5 --- test/interop/README.md | 4 ++-- test/interop/clients/js/src/close.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/interop/README.md b/test/interop/README.md index 4cf9715aaf1..b00e364cb51 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -124,8 +124,8 @@ released relays and clients, skip it. Whenever the browser subscriber is in the run, the matrix ends with a close-code case: Chromium dials the relay with a token its public rules refuse, and `WebTransport.closed` must carry the relay's code and reason. A WebSocket-only -`Connection` counts the refused session as connected, but the relay never -accepted its credential, so the handle must stop at it without redialing and +`Connection` never reports the refused session as connected, since the relay +never accepted its credential, and must stop at it without redialing and publish the same session error. Chromium treats a server's HTTP/3 control stream ending as fatal, so a server that ends it under the close capsule loses both; no Rust peer is that strict. diff --git a/test/interop/clients/js/src/close.ts b/test/interop/clients/js/src/close.ts index df2a6fa2ddc..d26af09522b 100644 --- a/test/interop/clients/js/src/close.ts +++ b/test/interop/clients/js/src/close.ts @@ -46,7 +46,8 @@ export async function refused(relay: string): Promise { * A WebSocket refusal reaches the reconnecting handle as its session error. * * The relay closes a session that ALPN already established, never having accepted its credential, - * so the handle stops at once with that close, as after a refused dial, instead of redialing. + * so the handle stops at once with that close, as after a refused dial, instead of redialing, and + * never reports it connected. */ export async function refusedWebSocket(relay: string): Promise { const url = new URL(relay); @@ -58,7 +59,7 @@ export async function refusedWebSocket(relay: string): Promise { share: false, backoff: { initial: Duration.millis(10), max: Duration.millis(10), timeout: null }, }); - // The refused session counted as connected first; a second one means the handle redialed. + // The refused session never reads as connected, so any connect means the relay admitted one. let connected = 0; const dispose = connection.status.subscribe((status) => { if (status === "connected") connected++; @@ -66,8 +67,7 @@ export async function refusedWebSocket(relay: string): Promise { try { let error = connection.error.peek(); while (!error) error = await connection.error.changed(); - if (connected !== 1) - return { error: `stopped after ${connected} session(s) instead of the refused one: ${error}` }; + if (connected !== 0) return { error: `the refused session read as connected ${connected} time(s): ${error}` }; if (!(error instanceof MoqError.Session)) return { error: String(error) }; return { closeCode: error.code, reason: error.message }; } finally { From 00d4cb19a705e8eb5bf4d11d86d9652ea7c5a9a3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 20:58:46 -0800 Subject: [PATCH 15/19] fix(net): Action::Established carries its session, which an early GOAWAY moves out of the live slot Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect.rs | 7 +++-- rs/moq-net/src/reconnect/tests.rs | 38 +++++++++++++++++++----- rs/moq-net/src/session.rs | 8 +++++ rs/moq-net/verify/reconnect/Handover.tla | 3 +- rs/moq-tokio/src/connection.rs | 8 ++--- 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 31f038b845d..960dc174699 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -344,7 +344,8 @@ pub enum Action { /// The live session is established: its peer accepted the connection credential, /// drained it with a GOAWAY, or it stayed up past [`Backoff::healthy`]. Until then a /// connected session may still be a refusal, so report it as serving only from here. - Established, + /// Carries the session, which a GOAWAY may already have moved to the predecessor. + Established(Session), /// Drop the dial or probe in flight: it ran out its deadline, which already counted as /// its failure, or the predecessor's peer redirected while it headed for that peer. Abandon, @@ -613,7 +614,7 @@ impl + From> Reco if !live.established && (now >= healthy || live.session.poll_accepted(waiter).is_ready()) { live.established = true; self.established = true; - return Action::Established; + return Action::Established(live.session.clone()); } let mut ended = None; // Checked before the close: a GOAWAY means the session is still up, and @@ -625,7 +626,7 @@ impl + From> Reco // redials now or serves until the peer closes the session. self.established = true; if !std::mem::replace(&mut live.established, true) { - self.queue.push_back(Action::Established); + self.queue.push_back(Action::Established(live.session.clone())); } // Under `Handoff`, a session drained before it was healthy is a redial that // reached the draining process rather than a replacement, so another would diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index a1159c6b6be..6c8acead65d 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -100,7 +100,7 @@ async fn next + From Action { loop { match next_any(machine).await { - Action::Established => continue, + Action::Established(_) => continue, action => return action, } } @@ -157,7 +157,7 @@ async fn drive( let attempt = match next_any(machine).await { Action::Dial(attempt) => attempt, Action::Done(res) => return Some(res), - Action::Established => { + Action::Established(_) => { run.events.push("established"); continue; } @@ -455,7 +455,7 @@ async fn a_silent_version_is_established_by_uptime() { "moq-lite-04 has no answer to accept with" ); if healthy { - assert!(matches!(next_any(&mut machine).await, Action::Established)); + assert!(matches!(next_any(&mut machine).await, Action::Established(_))); } server.abort(Error::Unauthorized); match next_any(&mut machine).await { @@ -466,6 +466,27 @@ async fn a_silent_version_is_established_by_uptime() { } } +/// A GOAWAY before any answer (moq-lite-04 never sends one) establishes the session, and the +/// action carries it, since the same pass already moved it out of the live slot. +#[kio::test] +async fn an_early_goaway_establishes_the_session_it_drains() { + let lite04: crate::Version = "moq-lite-04".parse().unwrap(); + let mut machine = + Reconnect::::new(config(INITIAL, None), vec![url("tcp://peer:1/")], kio::clock::now()).unwrap(); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + let (client, server, _) = handshake_with(lite04, false).await; + machine.connected(kio::clock::now(), client.clone()); + server.goaway(goaway::Goaway::new()).unwrap(); + + let Action::Established(established) = next_any(&mut machine).await else { + panic!("the drained session was never established"); + }; + assert!(established == client, "established another session"); + assert!(matches!(next_any(&mut machine).await, Action::Migrating)); + assert!(machine.session().is_none(), "the drained session left the live slot"); + drop(server); +} + /// A session that stayed up past the healthy bar establishes the policy even when the peer /// never answered the credential: its close, and the failed walk after it, retry. #[kio::test] @@ -909,7 +930,7 @@ async fn an_unanswered_replacement_does_not_serve() { HANDOVER, "the idle predecessor closed early" ); - assert!(matches!(next_any(&mut machine).await, Action::Established)); + assert!(matches!(next_any(&mut machine).await, Action::Established(_))); assert_eq!(kio::clock::now() - connected, healthy); } @@ -980,7 +1001,7 @@ async fn a_retirement_mid_dial_is_offline_until_the_replacement_is_established() let action = next_any(&mut machine).await; assert!(matches!(action, Action::Offline(None)), "{action:?}"); let action = next_any(&mut machine).await; - assert!(matches!(action, Action::Established), "{action:?}"); + assert!(matches!(action, Action::Established(_)), "{action:?}"); assert!(machine.session().is_some()); } @@ -1406,7 +1427,7 @@ async fn a_dropped_replacement_goes_offline_over_a_closing_predecessor() { let (replacement, replacement_server) = handshake().await; machine.connected(kio::clock::now(), replacement.clone()); // Only an established replacement serves, which lets the idle predecessor close. - assert!(matches!(next_any(&mut machine).await, Action::Established)); + assert!(matches!(next_any(&mut machine).await, Action::Established(_))); // The replacement is already closed when the next poll starts the idle predecessor's // graceful close, so its close is seen over a predecessor that is closing, not retired. drop(replacement_server); @@ -1492,7 +1513,10 @@ async fn connecting_once_never_probes() { let (mut machine, _fallback) = on_fallback(config).await; kio::clock::sleep(PROBE_MAX * 2).await; let waiter = kio::Waiter::noop(); - assert!(matches!(machine.poll(kio::clock::now(), &waiter), Action::Established)); + assert!(matches!( + machine.poll(kio::clock::now(), &waiter), + Action::Established(_) + )); assert!(matches!(machine.poll(kio::clock::now(), &waiter), Action::Wait(None))); } diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index aec01812b9f..c3e175f7ad2 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -226,6 +226,14 @@ impl PartialEq for Session { impl Eq for Session {} +impl std::fmt::Debug for Session { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Session") + .field("version", &self.version()) + .finish_non_exhaustive() + } +} + impl Session { /// Returns the negotiated protocol version. pub fn version(&self) -> Version { diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 4f5c60fd19a..3ee7ab0fe8c 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -169,7 +169,8 @@ OnDone == /\ probing' = FALSE /\ UNCHANGED started -\* `p` is the predecessor, or for `Established` the session it serves. +\* `p` is the predecessor, or for `Established` the session the action carries, +\* which a GOAWAY in the same pass already moved out of the live slot. Apply(a, p) == CASE a = "Established" -> OnEstablished(p) [] a = "Migrating" -> OnMigrating(p) diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 6e87ea83ce5..b38beb0a61c 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -1203,10 +1203,10 @@ impl Connection { dial = Some((url, Box::pin(async move { client.dial(attempt).await }))); } // The policy already counted the timeout; dropping the dial cancels it. - Action::Established => { - if let (Some(transport), Some(session)) = (pending.take(), policy.session()) { - shared.connected(session, transport); - forward = Some(Forward::new(session)); + Action::Established(session) => { + if let Some(transport) = pending.take() { + shared.connected(&session, transport); + forward = Some(Forward::new(&session)); } } Action::Abandon => { From 50a604c5c31026fedbc48b5fcaf2b4ea38da5e33 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 21:17:33 -0800 Subject: [PATCH 16/19] fix(net): box the established session, which clippy flags as a large variant Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect.rs | 6 +++--- rs/moq-net/src/reconnect/tests.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 960dc174699..18c3a2782f6 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -345,7 +345,7 @@ pub enum Action { /// drained it with a GOAWAY, or it stayed up past [`Backoff::healthy`]. Until then a /// connected session may still be a refusal, so report it as serving only from here. /// Carries the session, which a GOAWAY may already have moved to the predecessor. - Established(Session), + Established(Box), /// Drop the dial or probe in flight: it ran out its deadline, which already counted as /// its failure, or the predecessor's peer redirected while it headed for that peer. Abandon, @@ -614,7 +614,7 @@ impl + From> Reco if !live.established && (now >= healthy || live.session.poll_accepted(waiter).is_ready()) { live.established = true; self.established = true; - return Action::Established(live.session.clone()); + return Action::Established(Box::new(live.session.clone())); } let mut ended = None; // Checked before the close: a GOAWAY means the session is still up, and @@ -626,7 +626,7 @@ impl + From> Reco // redials now or serves until the peer closes the session. self.established = true; if !std::mem::replace(&mut live.established, true) { - self.queue.push_back(Action::Established(live.session.clone())); + self.queue.push_back(Action::Established(Box::new(live.session.clone()))); } // Under `Handoff`, a session drained before it was healthy is a redial that // reached the draining process rather than a replacement, so another would diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 6c8acead65d..10ef4d0dbcf 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -481,7 +481,7 @@ async fn an_early_goaway_establishes_the_session_it_drains() { let Action::Established(established) = next_any(&mut machine).await else { panic!("the drained session was never established"); }; - assert!(established == client, "established another session"); + assert!(*established == client, "established another session"); assert!(matches!(next_any(&mut machine).await, Action::Migrating)); assert!(machine.session().is_none(), "the drained session left the live slot"); drop(server); From 1d2ab40f89bd4be4ad106aec97ec07fc0f2d48ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 21:23:54 -0800 Subject: [PATCH 17/19] style: rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/reconnect.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 18c3a2782f6..ef58bb03813 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -626,7 +626,8 @@ impl + From> Reco // redials now or serves until the peer closes the session. self.established = true; if !std::mem::replace(&mut live.established, true) { - self.queue.push_back(Action::Established(Box::new(live.session.clone()))); + self.queue + .push_back(Action::Established(Box::new(live.session.clone()))); } // Under `Handoff`, a session drained before it was healthy is a redial that // reached the draining process rather than a replacement, so another would From 51e8538a84adacd16f91ef9ef6d54acfab7289db Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 23:44:58 -0800 Subject: [PATCH 18/19] feat(net)!: on a version that answers the credential, uptime never establishes a session A lite-07 session reads as connected only once the peer sends the AUTH_OK to the connection's own credential (or a SETUP on lite-05/06 and moq-transport) or drains it with a GOAWAY. The 1 s healthy bar establishes a session only on moq-lite 03/04, which never answer the credential. Applied to the Rust policy (`Reconnect`), JS `Reload`, and `Handover.tla`, which now picks per session whether its version answers. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 8 +- dart/moq/lib/src/aliases.dart | 3 +- dart/moq/lib/src/client.dart | 3 +- doc/bin/gstreamer.md | 7 +- doc/lib/cpp/index.md | 2 +- doc/lib/dart/index.md | 2 +- doc/lib/go/index.md | 2 +- doc/lib/index.md | 2 +- doc/lib/kt/index.md | 2 +- doc/lib/py/index.md | 2 +- doc/lib/swift/index.md | 2 +- go/wrapper/client.go | 6 +- js/net/src/auth_session.ts | 10 +++ js/net/src/connection/answered.test.ts | 51 +++++++++++-- js/net/src/connection/reload.test.ts | 25 +++++-- js/net/src/connection/reload.ts | 46 ++++++------ .../kotlin/dev/moq/Aliases.kt | 5 +- py/moq-rs/README.md | 2 +- py/moq-rs/moq/_records.py | 3 +- rs/libmoq/src/api.rs | 3 +- rs/moq-ffi/src/session.rs | 6 +- rs/moq-net/src/auth.rs | 6 ++ rs/moq-net/src/reconnect.rs | 54 +++++++------- rs/moq-net/src/reconnect/tests.rs | 74 +++++++------------ rs/moq-net/src/session.rs | 6 ++ rs/moq-net/verify/reconnect/Handover.tla | 53 +++++++------ rs/moq-tokio/src/connection.rs | 11 ++- swift/Sources/Moq/Aliases.swift | 3 +- swift/Sources/Moq/Client.swift | 3 +- 29 files changed, 241 insertions(+), 161 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 52c0cd2f01f..3d53f5c3e3f 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -15,11 +15,11 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | --- | --- | --- | | `NoStaleSession` | The session a `Connection` reports as live is never one the policy aborted or started closing. | An app opens subscriptions on a session that is going away. | | `StatusTruthful` | Connected is on the policy's live session, Migrating on its predecessor while nothing else is live, Closed exactly when the policy is done, and the predecessor `Connection::close` drains is the policy's. | Status or `close` acts on the wrong session, or a closed loop looks alive. | -| `ConnectedMeansEstablished` | A `Connection` reports a session only once the policy reported it established (`Action::Established`): its peer accepted the credential, it stayed up past the healthy bar, or the peer drained it with a GOAWAY. | A refused session reads as connected, and `established()` returns `Ok` for it. | +| `ConnectedMeansEstablished` | A `Connection` reports a session only once the policy reported it established (`Action::Established`): its peer accepted the credential or drained it with a GOAWAY, or, on a version that never answers the credential (moq-lite 03/04), it stayed up past the healthy bar. Uptime alone never establishes a session on a version that answers. | A refused session reads as connected, and `established()` returns `Ok` for it, or a lite-07 session the peer never accepted reads as connected once it is a second old. | | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | -| `FirstWalkFailsFast` | Until a session's peer accepted its credential, or one stayed up past the healthy bar or got a GOAWAY, which no refusal sends, no failed dial or closed session backs off: either ends the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | +| `FirstWalkFailsFast` | Until a session was reported established (see `ConnectedMeansEstablished`; a GOAWAY counts, since no refusal sends one), no failed dial or closed session backs off: either ends the policy. Only a session reported established establishes the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 6.5M distinct states, about a minute. The second checks the same rules under `Restart::Handoff` over three sessions: about 1.1M states. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 19M distinct states, about four minutes. The second checks the same rules under `Restart::Handoff` over three sessions: about 3.4M states, under a minute. Each session picks whether its version answers the credential. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -46,7 +46,7 @@ Sync commit: `e91df3340ad646a08f3d36d35d7f2c509ba9f037`. Run `git log e91df3340a | Code | Model | | --- | --- | | `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `Establish`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | -| `rs/moq-net/src/reconnect.rs` `established`, set by `end`, by a GOAWAY or the session's acceptance in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | +| `rs/moq-net/src/reconnect.rs` `established`, set in `poll` by a GOAWAY, the session's acceptance, or on a silent version the healthy bar, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, the session's `silent` (`Session::is_silent`), `Healthy`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "handoff"` (`RestartHandoff.cfg`) | diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 1b300c9f469..eb356e452c7 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -300,7 +300,8 @@ typedef ConnectionState = MoqConnectionState; /// Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. /// A null field keeps its default, zero means zero (retry at once, or give up /// at the first failure), and `maxUnlimited` lifts the ceiling. Until the peer -/// has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed +/// has accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed /// session, such as a refused credential, ends the connection with its error /// rather than retrying. typedef Backoff = MoqBackoff; diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index acf65ed4027..36ee3f65a15 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -61,7 +61,8 @@ final class ConnectOptions { final WebSocketConfig? websocket; /// Set true for a one-shot dial. By default a session whose credential the peer accepted, - /// or that the peer drained with a GOAWAY, redials with backoff whenever the transport + /// or that the peer drained with a GOAWAY (or, on moq-lite 03/04, which never answer it, + /// that stayed up 1s), redials with backoff whenever the transport /// drops; before then, a failed dial or a closed session, such as a refused /// credential, ends it with that error instead. final bool once; diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index b715d2ab516..65b4ddf1173 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -91,7 +91,8 @@ codec config, a pad requested again, or a stream resumed after EOS publishes as `url`, `broadcast`, `linger`, `connect-version`, `connect-websocket-enabled`, `tls-disable-verify`, `quic-idle-timeout`, `quic-keylog`, `quic-keep-alive`, and read-only `status`, `connected`, `moq-version`, and `estimated-send-rate`, `estimated-recv-rate`, `connection-stats`, and -`sessions`. Once the relay has accepted a session, the sink reconnects for as long as +`sessions`. Once the relay has accepted a session (on moq-lite 03/04, which never answer, once +one has stayed up for a second), the sink reconnects for as long as the pipeline runs, whatever ends each session. Before then it reports `failed` and posts an error instead of retrying: a first connect that reaches no relay, or a session the relay refuses, such as for a rejected token. @@ -172,8 +173,8 @@ its pads without EOS until it is announced again, so the pipeline does not end on its own. Losing the relay connection also holds the pads: `moqsrc` redials without a retry deadline and resumes when the broadcast returns. A first connect that reaches no relay posts an error rather than retrying, as does a first -connection the relay refuses, or a session it closes before the connection has -stayed up for a second. Once playback has begun, a session the relay closes is +connection the relay refuses, or a session it closes before accepting it (on +moq-lite 03/04, which never answer, before it has stayed up for a second). Once playback has begun, a session the relay closes is redialed, whatever its code, and a refused redial is retried like any other failure. A refused path or catalog, malformed input or catalog, or a catalog its publisher ended with an application code posts an error. A media track its diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index d0c26c41c9d..3613c972ca5 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -257,7 +257,7 @@ Set `ClientConfig::tokens` instead of putting the credential in the URL. The fir ## Stats -`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. +`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. ## Shutdown diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 637b6633b40..62e32b5dc12 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -160,7 +160,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` drains for up to one second and reports its result. `session.shutdown()` requests the drain synchronously; `await session.closed()` waits for its result. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements and grants also release their consumer; server listeners leave their shared server usable. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` or disposing its `MoqRoute` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `client.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps: ...)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps: ...)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `dispose()` it once the broadcast stops being watched. `setTarget(targetBps: ...)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `OutOfRangeMoqException`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. - **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential: credential)` or `claim.setAuth(credential: credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 0f923a095f5..6b50cb5a11e 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -133,7 +133,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown()` requests that drain immediately; `session.Closed(ctx)` waits for its result, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `Finish()` on its `media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.Close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `Session().Viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `Watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `Rate()` reads its slice in bits per second, nil while there is no estimate, so hold the current selection. `Update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `Close()` it once the broadcast stops being watched, from the goroutine that last used it. `SetTarget(&bps)` sends another target, zero sends none, and nil follows the viewers again. `Watch`, `Update`, and `SetTarget` return `moq.ErrOutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. Client viewers survive reconnects and read nil while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Cancel()` it when done. - **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `Session().Credential(token)` returns a `Credential` that `Route.SetAuth` or `Claim.SetAuth` names (nil for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `Answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `Answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `Answered(ctx)` returns the verdict, and `Cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/index.md b/doc/lib/index.md index 2426b15e63e..78fb2806cfe 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -67,7 +67,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. A session reads as connected only once the peer accepts its credential (or, on moq-lite 03/04, which never answer it, once it stays up 1s), so a refused one never does. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). +- **Reconnect** automatically with backoff when the transport drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s) it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. A session reads as connected only once the peer accepts its credential (or, on moq-lite 03/04, which never answer it, once it stays up 1s), so a refused one never does. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 0faec1c42e1..7cf42983efc 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -111,7 +111,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `moq.shutdown()` requests a drain immediately; suspending `moq.session.closed()` waits up to one second and reports its result. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.end()` takes it offline. How long to keep it up is yours to decide. A track still open at the end ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `moq.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `close()` it, or leave `use { }`, once the broadcast stops being watched. `setTarget(bps)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `MoqException.OutOfRange`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. - **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential)` or `claim.setAuth(credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. Suspending `credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index a9887b9b113..ce6f03b5829 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -125,7 +125,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `await session.closed()` waits up to one second and reports its result. `cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `moq.media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(max_bps)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `None` while there is no estimate (keep the current rendition), and `update(max_bps)` changes the ceiling. Holding a viewer requests the estimate like `recv_bandwidth()`, and dropping it releases the claim. The session sends the summed ceilings as its PROBE target; `set_target(bps)` overrides that, `0` sends none, and `None` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, raise `moq.Error.OutOfRange`. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. - **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session.credential(token)` returns a `Credential` that `route.set_auth(credential)` or `claim.set_auth(credential)` names; setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` raises the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 606af2df8ff..f469603c2e3 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -108,7 +108,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `try await session.closed()` waits up to one second and reports its result. `session.cancel(code:)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try finish()` on its `Media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()`, or releasing its route, takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(maxBps:)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `nil` while there is no estimate (keep the current rendition), and `update(maxBps:)` changes the ceiling. Holding a viewer requests the estimate like `recvBandwidth()`, and releasing it releases the claim. The session sends the summed ceilings as its PROBE target; `setTarget(targetBps:)` overrides that, `0` sends none, and `nil` follows the viewers again. `watch`, `update`, and `setTarget` throw `MoqError.OutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. - **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `try session.credential(token)` returns a `Credential` that `try route.setAuth(credential)` or `try claim.setAuth(credential)` names (`nil` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `try await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/go/wrapper/client.go b/go/wrapper/client.go index 4dec04c6d63..90d984c598c 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -205,7 +205,8 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // Backoff is the retry pacing for automatic reconnects: the delay starts at // Initial, multiplies by Multiplier after each failed attempt, and caps at Max. // After Timeout of consecutive failures the connection gives up for good. Until the -// peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a +// peer has accepted a session's credential or drained a session with a GOAWAY +// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a // closed session, such as a refused credential, ends the connection with its // error rather than retrying. // @@ -251,7 +252,8 @@ type Client struct { } // Dial starts a MoQ connection and returns without waiting for its first native dial to succeed. Cancel ctx -// to abort an in-flight connect. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed +// to abort an in-flight connect. Until the peer has accepted a session's credential or drained a session with a GOAWAY +// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed // dial or a closed session, such as a refused credential, closes the session with its error (see Session.Closed) // rather than retrying. func Dial(ctx context.Context, url string, config ClientConfig) (*Client, error) { diff --git a/js/net/src/auth_session.ts b/js/net/src/auth_session.ts index 927b4288736..69f3888519d 100644 --- a/js/net/src/auth_session.ts +++ b/js/net/src/auth_session.ts @@ -665,6 +665,7 @@ export class AuthSession { #credential = new Signal(undefined); // Whether the peer said it accepts the connection credential, which a silent version never does. #accepted = new Signal(false); + #silent: boolean; #closed = false; // Whoever answers the peer's tokens is decided once, after the task that established @@ -680,6 +681,7 @@ export class AuthSession { this.#requests = requests ?? false; this.#peerGrant = peerGrant; this.#draining = draining ?? (() => false); + this.#silent = silent ?? false; this.#state.attach(this); // A session without AUTH has the peer's acceptance with its handshake, unless it awaits its SETUP. if (!wire && !awaiting) { @@ -716,6 +718,14 @@ export class AuthSession { return this.#accepted; } + /** + * Whether the version never answers the connection credential (a moq-lite-03/04 client), so + * only uptime suggests the peer took it. + */ + get silent(): boolean { + return this.#silent; + } + /** The peer's SETUP arrived, so it accepted the connection credential a session without AUTH presented. */ accept(): void { if (this.#wire) return; diff --git a/js/net/src/connection/answered.test.ts b/js/net/src/connection/answered.test.ts index 61b36d9d3a1..ef91a22a3d1 100644 --- a/js/net/src/connection/answered.test.ts +++ b/js/net/src/connection/answered.test.ts @@ -4,7 +4,7 @@ import { Signal } from "@moq/signals"; import type { Request } from "../auth.ts"; import { SessionCode, SessionError } from "../error.ts"; import * as Lite from "../lite/index.ts"; -import { createMockTransportPair, manual, waitFor } from "../mock.ts"; +import { createMockTransportPair, type MockTransport, manual, waitFor } from "../mock.ts"; import * as Path from "../path.ts"; import { wireOf } from "../wire.ts"; import { accept as acceptSession } from "./index.ts"; @@ -44,20 +44,25 @@ function relay(decide: (token: string, request: Request) => void) { /** * A lite-06 relay, which has no AUTH: it accepts any credential but `refused`, whose session it - * closes once `release` resolves, before its SETUP. + * closes once `release` resolves, before its SETUP. `dials` counts the dials, and `served` holds + * the server side of each session it accepted. */ function closing(refused: string, release: Promise) { + const served: MockTransport[] = []; + const dials = new Signal(0); const stub = function StubWebTransport(target: string | URL) { + dials.update((n) => n + 1); const pair = createMockTransportPair(Lite.ALPN_06); const url = new URL(target); if (url.searchParams.get("jwt") === refused) { void release.then(() => pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" })); } else { + served.push(pair.server); void acceptSession({ transport: pair.server, url }); } return pair.client; }; - return stub as unknown as typeof WebTransport; + return { dials, served, stub: stub as unknown as typeof WebTransport }; } function loop(versions: string[], tokens: Signal) { @@ -110,13 +115,20 @@ test("a close that refuses the connection credential answers a session without A using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); - globalThis.WebTransport = closing("old", release); - const tokens = new Signal(["old"]); + const { dials, served, stub } = closing("old", release); + globalThis.WebTransport = stub; + const tokens = new Signal(["ok"]); const reload = loop(["moq-lite-06"], tokens); try { + // The server's SETUP accepts the first credential, so the refusing session's close is + // retried rather than ending the URL. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); - // A session that stayed up past the healthy bar, so its close is retried rather than ending the URL. - await clock.advance(Duration.millis(1500)); + // Swapped while no session is live, so no session takes it in band. + served[0]?.close({ closeCode: 0, reason: "restart" }); + await clock.drive(waitFor(reload.established, (established) => established === undefined)); + tokens.set(["old"]); + await clock.drive(waitFor(dials, (n) => n >= 2)); + await clock.advance(Duration.millis(10)); const waiting = refusal(reload.answered()); resolve(); const err = await clock.drive(waiting); @@ -131,6 +143,31 @@ test("a close that refuses the connection credential answers a session without A } }); +test("a lite-07 session up past the healthy bar without an AUTH_OK never reads as connected", async () => { + using clock = manual(); + const original = globalThis.WebTransport; + const held = new Signal([]); + globalThis.WebTransport = relay((_token, request) => held.update((requests) => [...requests, request])); + const reload = loop(["moq-lite-07-wip"], new Signal([])); + const statuses: string[] = []; + const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); + try { + const [credential] = await clock.drive(waitFor(held, (requests) => requests.length > 0)); + // Uptime stands in for an answer only on moq-lite 03/04, which have none. + await clock.advance(Duration.millis(5000)); + expect(statuses).not.toContain("connected"); + expect(reload.established.peek()).toBeUndefined(); + + credential?.accept(GRANT); + await clock.drive(waitFor(reload.status, (status) => status === "connected")); + expect(reload.established.peek()).toBeDefined(); + } finally { + unsubscribe(); + reload.close(); + globalThis.WebTransport = original; + } +}); + test("a kept credential counts only once the peer refuses it", async () => { using clock = manual(); const original = globalThis.WebTransport; diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 975739f9e62..17bc24114c6 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -447,6 +447,8 @@ for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, Se */ function refusing(refused: string, release: Promise) { const dialed = new Signal([]); + // The server sides of the sessions it accepted. + const served: MockTransport[] = []; const stub = function StubWebTransport(target: string | URL) { const jwt = new URL(target).searchParams.get("jwt"); dialed.update((jwts) => [...jwts, jwt]); @@ -454,11 +456,12 @@ function refusing(refused: string, release: Promise) { if (jwt === refused) { void release.then(() => pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" })); } else { + served.push(pair.server); void accept(pair.server, new URL(target)); } return pair.client; }; - return { dialed, stub: stub as unknown as typeof WebTransport }; + return { dialed, served, stub: stub as unknown as typeof WebTransport }; } test("a first session refused before its credential is accepted stops the URL, even for replaced tokens", async () => { @@ -539,10 +542,10 @@ test("a refused credential keeps redialing until the tokens change", async () => using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); - const { dialed, stub } = refusing("old", release); + const { dialed, served, stub } = refusing("old", release); globalThis.WebTransport = stub; - const tokens = new Signal(["old"]); + const tokens = new Signal(["ok"]); const reload = new Reload({ url: new URL("https://example.com/"), versions: ["moq-lite-06"], @@ -551,13 +554,19 @@ test("a refused credential keeps redialing until the tokens change", async () => backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); try { - // Refused only once a session was healthy, though never accepted: until then, a refusal - // stops the URL. + // The server's SETUP accepts the first credential, which establishes the URL: until + // then, a refusal stops it. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); - await clock.advance(Duration.millis(1000)); + tokens.set(["old"]); + served[0]?.close({ closeCode: 0, reason: "restart" }); resolve(); - await clock.drive(waitFor(dialed, (jwts) => jwts.length >= 3)); - expect(dialed.peek().every((jwt) => jwt === "old")).toBe(true); + await clock.drive(waitFor(dialed, (jwts) => jwts.length >= 4)); + expect( + dialed + .peek() + .slice(1) + .every((jwt) => jwt === "old"), + ).toBe(true); expect(reload.error.peek()).toBeUndefined(); tokens.set(["new"]); await clock.drive(waitFor(dialed, (jwts) => jwts.includes("new"))); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index be142b2d762..749e812e8a0 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -42,9 +42,10 @@ const PROBE_MAX = Duration.secs(300); * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in * lockstep. Nothing retries until a session to the current URL has been established: the peer * accepts the connection credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the - * connection's own credential), drains the session with a GOAWAY, which no refusal sends, or - * the session stays up past {@link Backoff.healthy}, the only bar on moq-lite 03/04, which - * answer the credential with nothing. An attempt failing, or a session closing sooner, stops that + * connection's own credential) or drains the session with a GOAWAY, which no refusal sends. On + * moq-lite 03/04, which answer the credential with nothing, staying up past + * {@link Backoff.healthy} establishes it too; a version with an answer waits for it however long + * the session stays up. An attempt failing, or a session closing sooner, stops that * URL with its error. That covers a relay refusing a live session, as it does a rejected token * on moq-lite 03 or later. After that, every failure is retried, * including an attempt that outlives {@link ConnectProps.timeout}, until @@ -73,9 +74,9 @@ export type Backoff = { /** * How long a session must stay up to count as healthy (default: 1000ms; must be positive). * - * The first healthy session establishes the URL, though only a peer that never answers the - * connection credential, as on moq-lite 03/04, waits for this: an accepting peer - * establishes it sooner. + * Where the peer never answers the connection credential, as on moq-lite 03/04, the first + * healthy session establishes the URL. Elsewhere only the peer's answer or a GOAWAY does, + * however long a session stays up. * A healthy session's end resets the delay and the timeout. One that ends sooner is a * failed attempt, so a peer that accepts and then drops every session still escalates the * delay and exhausts the timeout. @@ -383,8 +384,8 @@ export class Reload { // loop is disabled. #givenUpHref: string | undefined; - // The configured href a session's peer accepted the credential of, a session stayed up past - // the healthy bar for, or the peer drained with a GOAWAY. Until it matches, a failed + // The configured href a session's peer accepted the credential of or drained with a GOAWAY, + // or a moq-lite-03/04 session stayed up past the healthy bar for. Until it matches, a failed // attempt or a closed session gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which // resume a URL that worked. #establishedHref: string | undefined; @@ -635,24 +636,25 @@ export class Reload { this.#verdict.connected(connection); connected = now(); - // Serve only once the session is established: the peer accepted its credential, - // drained it with a GOAWAY, or it stayed up past the healthy bar, the only sign - // on a version that never answers (moq-lite-03/04). Until then it may still be a - // refusal, which never reads as connected. A close first is handled below. + // Serve only once the session is established: the peer accepted its credential or + // drained it with a GOAWAY. On a version that never answers (moq-lite-03/04), + // staying up past the healthy bar is the only sign, so it counts there alone. + // Until then it may still be a refusal, which never reads as connected. A close + // first is handled below. + const auth = wireOf(connection).auth; const bar = this.#pacing().healthy; - const acceptance = wireOf(connection).auth.accepted; + const acceptance = auth.accepted; if (!acceptance.peek()) { const waiting = new AbortController(); - const uptime = sleep(bar, { signal: waiting.signal }).catch(() => {}); - await effect - .race(Signal.race(acceptance), uptime, connection.closed, connection.draining) - .finally(() => waiting.abort()); + const waits: unknown[] = [Signal.race(acceptance), connection.closed, connection.draining]; + if (auth.silent) waits.push(sleep(bar, { signal: waiting.signal }).catch(() => {})); + await effect.race(...waits).finally(() => waiting.abort()); if (signal.aborted) return; } if ( acceptance.peek() || connection.draining.peek() !== undefined || - now().since(connected).ge(bar) + (auth.silent && now().since(connected).ge(bar)) ) { // The replacement serves now; a predecessor keeps draining its groups in flight. this.#serve(connection); @@ -952,12 +954,12 @@ export class Reload { if (draining) this.#end(draining); } - // Mark `href` established once the session connected at `connected` stayed up past the healthy bar. // A session establishes its URL once the peer accepted its credential, which a closed - // session still reports, or once it stayed up past the healthy bar, the only bar on a - // version that never answers the credential (moq-lite-03/04). + // session still reports, or, on a version that never answers the credential + // (moq-lite-03/04), once it stayed up past the healthy bar. #settle(href: string, session: Session, connected: Instant): void { - if (wireOf(session).auth.accepted.peek() || now().since(connected).ge(this.#pacing().healthy)) { + const auth = wireOf(session).auth; + if (auth.accepted.peek() || (auth.silent && now().since(connected).ge(this.#pacing().healthy))) { this.#establishedHref = href; } } diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 4d534fc08b9..1442a09c64e 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -391,8 +391,9 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. * A null field keeps its default, zero means zero (retry at once, or give up at the first failure), * and `maxUnlimited` lifts the ceiling. Until the peer has accepted a session's credential or - * drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, - * ends the connection with its error rather than retrying. + * drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed + * up 1s), a failed dial or a closed session, such as a refused credential, ends the connection with + * its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 5a6cb6c9278..82516d69c7d 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -160,7 +160,7 @@ client = moq.Client( - `.cancel(code)`, `session.shutdown()`. Cancel immediately, or request a drain now and await `session.closed()` for its result, including a delivery timeout after one second. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - `.publish() → OriginProducer`, `.consume() → OriginConsumer`. The wired origin sides. - `.stats() → ConnectionStats`. Snapshot RTT, bandwidth estimates, and byte/packet counters. - - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed session, such as a refused credential, ends it with that error. + - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends it with that error. - `.allocator() → Allocator`. Divide the send estimate between encoders and app-owned tracks. - `.viewers() → Viewers`. Divide the receive estimate between watched broadcasts: `.watch(max_bps) → Viewer` claims up to a ceiling, `Viewer.rate()` reads its slice (`None` while there is no estimate), and dropping the viewer releases it. The summed ceilings are the PROBE target; `.set_target(bps)` overrides it (`0` sends none, `None` follows the viewers again). - `.grants() → GrantConsumer`. On `moq-lite-07-wip`, `.grant()` reads the peer's `Grant` and `async for grant in session.grants()` follows it. diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index 9840185a868..850c3746b66 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -345,7 +345,8 @@ def _track_info(info: TrackInfo | None) -> MoqTrackInfo | None: class Backoff: """Retry pacing for the automatic reconnect. - Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or + Until the peer has accepted a session's credential or drained a session with a GOAWAY + (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the connection with its error rather than retrying. diff --git a/rs/libmoq/src/api.rs b/rs/libmoq/src/api.rs index 245da762a50..6eb52a56f38 100644 --- a/rs/libmoq/src/api.rs +++ b/rs/libmoq/src/api.rs @@ -1791,7 +1791,8 @@ unsafe fn connect_session( /// You should call [moq_session_close], even on error, to free up resources. /// /// Once connected, the session reconnects automatically with exponential backoff if the -/// connection drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY, a +/// connection drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a /// failed dial or a closed session, such as a refused credential, ends it instead, with that error as the terminal /// snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on /// each reconnect, since the origins outlive the underlying connection. diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index f72e2b370c7..73686182c2d 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -202,7 +202,8 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// -/// Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a +/// Until the peer has accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a /// closed session, such as a refused credential, ends the connection with its error rather /// than retrying. The delay starts at `initial`, multiplies by `multiplier` after each /// failed attempt, and caps at `max`. @@ -616,7 +617,8 @@ impl MoqClient { /// Once a session has been established, it automatically reconnects with backoff when /// the transport drops (unless [`MoqClientConfig::once`] is set), and broadcasts /// consumed through it ride out the gap. Until the peer has accepted a session's credential or drained - /// a session with a GOAWAY it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the + /// a session with a GOAWAY + /// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s) it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the /// one it accepted, as a relay refusing a credential does, [`MoqSession::closed`] resolves /// with that error, so an app started before its relay is reachable must start the session /// again. diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index ec3aa00804f..95748af9107 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -1440,6 +1440,12 @@ impl Link { self.with(|link| link.acceptance = Acceptance::Silent); } + /// Whether the version never answers the connection credential, so only uptime can + /// suggest the peer took it. + pub(crate) fn is_silent(&self) -> bool { + self.read(|link, _| link.acceptance == Acceptance::Silent) + } + /// The peer's SETUP arrived, so it accepted the connection credential. pub(crate) fn accepted(&self) { self.with_state(|state, link| { diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index ef58bb03813..f30b492bf72 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -11,9 +11,10 @@ //! //! The policy fails fast until a session is established: the peer accepts the connection //! credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the connection's own -//! credential), drains the session with a GOAWAY, or the session stays up past -//! [`Backoff::healthy`], the only bar on moq-lite 03/04, which answer the credential with -//! nothing. A failed walk, or a session closed before then, ends the policy +//! credential) or drains the session with a GOAWAY. On moq-lite 03/04, which answer the +//! credential with nothing, staying up past [`Backoff::healthy`] establishes it too; a +//! version with an answer waits for it however long the session stays up. A failed walk, +//! or a session closed before then, ends the policy //! with [`Action::Done`] carrying that error rather than retrying, so a setup mistake //! surfaces at once instead of looping. That covers a peer that refuses the session right //! after connecting, as a relay rejecting a token on moq-lite 03 or later does. A client @@ -60,9 +61,9 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// When to redial after a failure, and when to give up, once a session has been established. /// -/// A session is established once the peer accepts the connection credential, drains the -/// session with a GOAWAY, or the session stays up past `healthy`. Before then nothing -/// retries: a failed walk, or a session closed sooner, ends the policy. +/// A session is established once the peer accepts the connection credential or drains the +/// session with a GOAWAY, or, on a version with no answer to the credential (moq-lite +/// 03/04), once it stays up past `healthy`. Before then nothing retries: a failed walk, or a session closed sooner, ends the policy. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter, @@ -90,9 +91,9 @@ pub struct Backoff { pub max: Option, /// How long a session must stay up to count as healthy. Must be nonzero. Defaults to 1 s. /// - /// The first healthy session establishes the policy, so failures retry from then on. That - /// bar matters only where the peer never answers the connection credential, as on moq-lite - /// 03/04: an accepting peer establishes it sooner. + /// Where the peer never answers the connection credential, as on moq-lite 03/04, the first + /// healthy session establishes the policy, so failures retry from then on. Elsewhere only + /// the peer's answer or a GOAWAY establishes it, however long a session stays up. /// A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. @@ -341,8 +342,9 @@ pub enum Action { /// [`Reconnect::failed`], polling meanwhile. Build each session's client with /// [`Attempt::client`]. Dial(Attempt), - /// The live session is established: its peer accepted the connection credential, - /// drained it with a GOAWAY, or it stayed up past [`Backoff::healthy`]. Until then a + /// The live session is established: its peer accepted the connection credential or + /// drained it with a GOAWAY, or, on a version with no answer (moq-lite 03/04), it stayed + /// up past [`Backoff::healthy`]. Until then a /// connected session may still be a refusal, so report it as serving only from here. /// Carries the session, which a GOAWAY may already have moved to the predecessor. Established(Box), @@ -387,8 +389,8 @@ pub struct Reconnect { phase: Phase, /// The failures of the walk under way. walk: Walk, - /// Whether any session's peer accepted its credential, or one stayed up past the healthy - /// bar or ended in a GOAWAY. Until then, a failed walk or a closed session ends the policy. + /// Whether any session was established: [`Action::Established`] was reported for it. Until + /// then, a failed walk or a closed session ends the policy. established: bool, /// An old session kept serving after a GOAWAY so its in-flight groups finish. predecessor: Option, @@ -610,8 +612,14 @@ impl + From> Reco Phase::Live(live) => { // Checked before the close, which may follow the peer's acceptance in one wake: // a session the peer accepted and then dropped is an outage, not a refusal. - let healthy = live.since.saturating_add(self.config.backoff.healthy); - if !live.established && (now >= healthy || live.session.poll_accepted(waiter).is_ready()) { + // Uptime stands in for an answer only on a version that has none: a peer that + // can answer and has not may still refuse. + let healthy = live + .session + .is_silent() + .then(|| live.since.saturating_add(self.config.backoff.healthy)); + let uptime = healthy.is_some_and(|healthy| now >= healthy); + if !live.established && (uptime || live.session.poll_accepted(waiter).is_ready()) { live.established = true; self.established = true; return Action::Established(Box::new(live.session.clone())); @@ -656,10 +664,10 @@ impl + From> Reco if !self.queue.is_empty() { continue; } - // An unestablished session is established by uptime at the latest. + // An unestablished silent session is established by uptime at the latest. let handover = match live.established { true => handover, - false => earliest(Some(healthy), handover), + false => earliest(healthy, handover), }; // Only an established session is left for the preferred transport: one that may // still be a refusal is no session to hand over. @@ -785,7 +793,8 @@ impl + From> Reco tracing::debug!("the fallback session was already draining"); } // The fallback session served all along, so a long one still earns the healthy reset. - self.established |= self.retry.settle( + // Only an established session is probed, so the policy already is. + self.retry.settle( now, now.saturating_duration_since(old.since), &Pacing::new(&self.config.backoff), @@ -969,12 +978,8 @@ impl + From> Reco let Poll::Ready(msg) = redirect.goaway.poll(waiter) else { return None; }; + // No need to establish the policy here: only an established session is probed. let redirect = predecessor.redirect.take()?; - // The peer accepted the fallback session and asked us to move, so its GOAWAY - // establishes the policy as one off the live session does. - if msg.is_ok() { - self.established = true; - } let target = match msg.map(|msg| { self.config .goaway @@ -1017,9 +1022,6 @@ impl + From> Reco let tenure = now.saturating_duration_since(live.since); let url = self.targets[live.index].url().clone(); let pacing = Pacing::new(&self.config.backoff); - // Judged on tenure alone, so a long session that ends in a redirect loop still counts. - self.established |= tenure >= pacing.healthy; - // The connected target owns the policy, including when connecting once. A refused // redirect is terminal: the peer is leaving and named somewhere we won't go, so // redialing the old address or a fallback would ignore it. diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 10ef4d0dbcf..3c1e34cdaed 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -487,10 +487,11 @@ async fn an_early_goaway_establishes_the_session_it_drains() { drop(server); } -/// A session that stayed up past the healthy bar establishes the policy even when the peer -/// never answered the credential: its close, and the failed walk after it, retry. +/// On a version that answers the connection credential (moq-lite-07-wip), uptime is no +/// answer: a session up well past the healthy bar without its AUTH_OK is never established, +/// and its close ends the policy as a refusal. #[kio::test] -async fn a_healthy_session_establishes_the_policy() { +async fn an_unanswered_session_is_not_established_by_uptime() { let peer = url("tcp://peer:1/"); let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); @@ -499,14 +500,17 @@ async fn a_healthy_session_establishes_the_policy() { }) .await; - kio::clock::sleep(Config::default().backoff.healthy).await; + kio::clock::sleep(Config::default().backoff.healthy * 10).await; + let action = machine.poll(kio::clock::now(), &kio::Waiter::noop()); + assert!(matches!(action, Action::Wait(None)), "{action:?}"); run.servers.remove(0).abort(Error::Unauthorized); - let ended = drive(&mut machine, &[(peer, Peer::Fail(Verdict::Retry))], &mut run, |run| { - run.dials.len() == 3 - }) - .await; - assert!(ended.is_none(), "an established policy gave up"); - assert_eq!(run.events, ["established", "offline", "retrying"]); + let ended = drive(&mut machine, &[(peer, Peer::Serve)], &mut run, |_| false).await; + assert!( + matches!(ended, Some(Err(TestError::Net(_)))), + "the unanswered session was retried: {ended:?}" + ); + assert_eq!(run.dials.len(), 1, "redialed after the refusal"); + assert!(run.events.is_empty(), "established or retried: {:?}", run.events); } /// A GOAWAY establishes the policy however soon it comes, since a refusal never drains a @@ -900,7 +904,7 @@ async fn a_healthy_goaway_hands_over_to_the_redirect() { /// A replacement the peer has not yet accepted does not serve: a predecessor that retires /// meanwhile leaves the connection offline, and the replacement reports -/// [`Action::Established`] only once it passes the healthy bar unanswered. +/// [`Action::Established`] only once the peer answers, however long that takes. #[kio::test] async fn an_unanswered_replacement_does_not_serve() { const HANDOVER: Duration = Duration::from_millis(500); @@ -930,8 +934,19 @@ async fn an_unanswered_replacement_does_not_serve() { HANDOVER, "the idle predecessor closed early" ); + kio::clock::sleep(healthy * 10).await; + let action = machine.poll(kio::clock::now(), &kio::Waiter::noop()); + assert!( + matches!(action, Action::Wait(None)), + "established by uptime: {action:?}" + ); + + let mut requests = run.withheld.remove(0); + let request = kio::wait(|waiter| requests.poll_next(waiter)) + .await + .expect("the credential"); + let _issued = request.accept(auth::Grant::all()); assert!(matches!(next_any(&mut machine).await, Action::Established(_))); - assert_eq!(kio::clock::now() - connected, healthy); } /// Our handover window caps the peer's GOAWAY deadline, and a predecessor retiring while @@ -1247,41 +1262,6 @@ async fn a_redirect_after_a_probe_got_through_is_followed() { assert!(machine.predecessor().is_some(), "the redirected session still drains"); } -/// The draining fallback's peer accepted that session and asked us to move, which no refusal -/// does, so its redirect establishes the policy however young the session was: the redirected -/// dial failing retries rather than ending it. -#[kio::test] -async fn a_fallback_peer_redirect_establishes_the_policy() { - let mut config = Config::default(); - // Longer than the probe takes, so the fallback session is never healthy. - config.backoff.healthy = Duration::from_secs(3600); - // Nor accepted: its peer never answers the credential. - let peer = url("https://peer:1/"); - let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); - let mut run = Run::default(); - drive(&mut machine, &[(peer, Peer::Withhold)], &mut run, |run| { - !run.accepts.is_empty() - }) - .await; - machine.fallback(kio::clock::now()); - let fallback_server = run.servers.remove(0); - probe(&mut machine).await; - assert!(machine.probed(kio::clock::now(), Ok(()))); - assert!(matches!(next(&mut machine).await, Action::Migrating)); - - fallback_server - .goaway(goaway::Goaway::redirect(url("https://peer:2/").to_string())) - .expect("the fallback peer sends a GOAWAY"); - assert!(matches!(next(&mut machine).await, Action::Abandon)); - assert!(matches!(next(&mut machine).await, Action::Dial(_))); - machine.failed(kio::clock::now(), TestError::Dial, Verdict::Retry); - let action = next(&mut machine).await; - assert!( - !matches!(action, Action::Done(_)), - "the redirected dial's failure ended the policy" - ); -} - /// A redirect from the draining fallback's peer during the backoff after a failed /// handshake keeps the wait, and the redial goes where the peer named. #[kio::test] diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index c3e175f7ad2..d7cdf506f08 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -487,6 +487,12 @@ impl Session { self.link.poll_accepted(waiter) } + /// Whether the peer never answers the connection credential, as on moq-lite 03/04, so + /// only uptime suggests it took the credential. + pub(crate) fn is_silent(&self) -> bool { + self.link.is_silent() + } + /// This session's side of its [`auth`](Self::auth) handle. pub(crate) fn link(&self) -> &auth::Link { &self.link diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 3ee7ab0fe8c..ab8e351b476 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -20,9 +20,9 @@ (* one it refuses, no retry window, no tokens, not connecting once, and no *) (* redirect-loop scoring. A session's age stands in for its tenure: 1 is *) (* past `Backoff::healthy`. Until a session has been established, which *) -(* takes the peer accepting its credential, one past that bar, or a *) -(* GOAWAY, a failed dial or a closed session ends the policy *) -(* (`Reconnect::established`). *) +(* takes the peer accepting its credential or a GOAWAY, or on a silent *) +(* version (moq-lite 03/04, no answer) one past that bar, a failed dial or *) +(* a closed session ends the policy (`Reconnect::established`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -60,7 +60,7 @@ VARIABLES started, \* `Presence::sessions_started` ended, \* `Presence::sessions_ended` misdials, \* 1 once a dial went elsewhere than an unheeded redirect named - established \* `Reconnect::established`: a session's peer accepted its credential, or one stayed up past the healthy bar or got a GOAWAY + established \* `Reconnect::established`: a session was reported established policyVars == <> @@ -75,11 +75,13 @@ vars == <> \* whether we sent our own, whether the policy watches its GOAWAY as the \* predecessor (`Predecessor::redirect`, set when a probe moved us off it), \* whether the peer accepted its credential (`Session::poll_accepted`), which a -\* close leaves as it was, and whether the policy reported it established +\* close leaves as it was, whether its version never answers the credential +\* (`Session::is_silent`), and whether the policy reported it established \* (`Live::established`), the only point a caller serves it from. Fresh(e) == [ep |-> e, open |-> TRUE, by |-> "", closing |-> FALSE, goaway |-> NoGoaway, age |-> 0, idle |-> FALSE, heeded |-> FALSE, - sent |-> FALSE, watched |-> FALSE, accepted |-> FALSE, served |-> FALSE] + sent |-> FALSE, watched |-> FALSE, accepted |-> FALSE, silent |-> FALSE, + served |-> FALSE] Unused == [Fresh(Start) EXCEPT !.open = FALSE] Timers == {"dial", "backoff", "probe", "handover"} @@ -208,7 +210,9 @@ Serving == phase = "live" /\ sess[live].served \* The live session is due its `Action::Established`, which `poll` returns \* before reading its GOAWAY or close. -EstablishDue == phase = "live" /\ ~sess[live].served /\ (sess[live].accepted \/ sess[live].age >= 1) +\* Uptime stands in for the peer's answer only on a version that has none. +Healthy(s) == sess[s].silent /\ sess[s].age >= 1 +EstablishDue == phase = "live" /\ ~sess[live].served /\ (sess[live].accepted \/ Healthy(live)) \* The predecessor closed or overstayed its handover: `Predecessor::poll`. PredDue == pred # None /\ (~sess[pred].open \/ timer["handover"]) @@ -276,8 +280,8 @@ FollowPred == /\ UNCHANGED <> /\ UNCHANGED <> - \* Its GOAWAY establishes the policy, as one off the live session does. - /\ established' = TRUE + \* Only an established session is probed, so the policy already is. + /\ UNCHANGED established \* `Phase::Next`: ask for a dial. Dial == @@ -358,13 +362,15 @@ EndGoaway(s, uri) == \* `Reconnect::closed`. Until a session has been established, a close ends the \* policy as a failed dial does: it is how a peer refuses a live session before -\* accepting its credential. `poll` reads the acceptance before the close. After that, every close retries, whatever its error. A predecessor +\* accepting its credential. `poll` reads the acceptance and the healthy bar +\* before the close, so a session either establishes would already have. After +\* that, every close retries, whatever its error. A predecessor \* that is not closing gracefully serves again, so the policy migrates back \* onto it; the `Retrying` that follows changes no status, so it is left out. EndClosed == - /\ established' = (established \/ sess[live].age >= 1 \/ sess[live].accepted) + /\ UNCHANGED established /\ probe' = "none" - /\ IF established' + /\ IF established THEN /\ live' = None /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] @@ -385,7 +391,8 @@ WaitOut(s) == /\ Restart = "handoff" /\ sess[s].age = 0 -\* `Action::Established`, from the peer's acceptance or the healthy bar. +\* `Action::Established`, from the peer's acceptance, or the healthy bar on a +\* silent version. Establish == /\ queue = <<>> /\ ~PredDue @@ -469,7 +476,8 @@ Connected == /\ LET s == used + 1 IN /\ used' = s /\ dial' = "none" - /\ sess' = [sess EXCEPT ![s] = Fresh(dialTo)] + \* Any version: one that answers the credential, or a silent one. + /\ \E silent \in BOOLEAN : sess' = [sess EXCEPT ![s] = [Fresh(dialTo) EXCEPT !.silent = silent]] /\ UNCHANGED <> \* `Reconnect::connected`, ignored unless a dial is in flight. An `Offline` \* a predecessor that retired mid-dial left queued stands: nothing serves @@ -539,9 +547,8 @@ ProbeGetsThrough == /\ dial' = "handshake" /\ dialTo' = sess[live].ep /\ Emit(Append(queue, "Migrating"), live) - \* The fallback served all along, so a long or accepted one establishes the policy. - /\ established' = (established \/ sess[live].age >= 1 \/ sess[live].accepted) - /\ UNCHANGED <> + \* Only an established session is probed, so the policy already is. + /\ UNCHANGED <> Caller == Connected \/ Failed \/ ProbeFails \/ ProbeGetsThrough @@ -660,12 +667,13 @@ StatusTruthful == /\ lc # "closed" => spred = pred \* A session reads as connected only once established: the peer accepted its -\* credential, it stayed up past the healthy bar, or the peer drained it with -\* a GOAWAY. A refusal never does. +\* credential or drained it with a GOAWAY, or on a silent version it stayed up +\* past the healthy bar. A refusal never does, nor does uptime alone on a +\* version that answers. ConnectedMeansEstablished == /\ Parked /\ lc = "live" => sess[lcSess].served /\ \A s \in 1..used : - sess[s].served => sess[s].accepted \/ sess[s].age >= 1 \/ sess[s].goaway # NoGoaway + sess[s].served => sess[s].accepted \/ Healthy(s) \/ sess[s].goaway # NoGoaway \* `Action::Offline`: nothing serves anymore. A predecessor closing gracefully \* already carries nothing and takes no new work. @@ -685,7 +693,10 @@ PresenceBalanced == \* The policy fails fast: no failed dial or closed session backs off before a \* session has been established, so a setup mistake or a refusal ends the \* policy instead of looping. A GOAWAY establishes it: no refusal sends one. -FirstWalkFailsFast == phase = "backoff" => established +\* Only a session reported established establishes the policy. +FirstWalkFailsFast == + /\ phase = "backoff" => established + /\ established => \E s \in 1..used : sess[s].served \* A dial never goes elsewhere than where a session the policy still holds \* was redirected and the policy never acted on it: the peer is leaving, and diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index b38beb0a61c..ba6ffa51376 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -903,7 +903,8 @@ pub struct Snapshot { /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) /// redials with exponential backoff. Until the peer has accepted a session's credential or -/// drained a session with a GOAWAY, nothing retries: a failed dial, or a session the peer closes, as a relay refusing a +/// drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), nothing retries: a failed dial, or a session the peer closes, as a relay refusing a /// credential does, stops the loop with its error, so a setup mistake surfaces at once. The read surface /// mirrors [`moq_net::Session`] so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), @@ -2364,15 +2365,19 @@ mod tests { let mut request = kio::clock::timeout(WAIT, listener.accept()).await.unwrap().unwrap(); let mut requests = request.requests(); let session = request.ok().await.unwrap(); - loop { + // Both, in either order: the connection's own credential, whose AUTH_OK establishes + // the session, and the token. + let (mut credential, mut presented) = (false, false); + while !(credential && presented) { let request = kio::clock::timeout(WAIT, requests.next()).await.unwrap().unwrap(); if request.token().is_empty() { issued.push(request.accept(moq_net::auth::Grant::default())); + credential = true; continue; } assert_eq!(request.token().as_ref(), b"token", "round {round}"); issued.push(request.accept(moq_net::auth::Grant::all())); - break; + presented = true; } if let Some(added) = added.take() { token = Some(added.await.unwrap().expect("granted")); diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 5412e383583..444114ebf8b 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -92,7 +92,8 @@ public typealias ConnectionStats = MoqFFI.MoqConnectionStats /// Retry pacing for the automatic reconnect; see `Client(backoff:)`. A `nil` /// field keeps its default, zero means zero (retry at once, or give up at the /// first failure), and `maxUnlimited` lifts the ceiling. Until the peer has -/// accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed +/// accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed /// session, such as a refused credential, ends the connection with its error /// rather than retrying. public typealias Backoff = MoqFFI.MoqBackoff diff --git a/swift/Sources/Moq/Client.swift b/swift/Sources/Moq/Client.swift index e4769c838c4..9136cab1567 100644 --- a/swift/Sources/Moq/Client.swift +++ b/swift/Sources/Moq/Client.swift @@ -65,7 +65,8 @@ public final class Client: Sendable { } /// Start a connection and return its session without waiting for the native first dial to succeed. - /// Until the peer has accepted a session's credential or drained a session with a GOAWAY, a failed dial or a closed + /// Until the peer has accepted a session's credential or drained a session with a GOAWAY + /// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed /// session, such as a refused credential, closes the session with its error rather than retrying. public func connect(to url: String) async throws -> Session { Session(try await ffi.connect(url: url)) From 2bf3599f4fc343005fa9f0bdd8d96c337130d423 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 23:45:02 -0800 Subject: [PATCH 19/19] docs(verify-reconnect): sync the model to the uptime rule Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 3d53f5c3e3f..5d140f02de5 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -41,7 +41,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `e91df3340ad646a08f3d36d35d7f2c509ba9f037`. Run `git log e91df3340a.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `51e8538a84adacd16f91ef9ef6d54acfab7289db`. Run `git log 51e8538a84.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- |