diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 5ac64091797..5d140f02de5 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -15,10 +15,11 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | --- | --- | --- | | `NoStaleSession` | The session a `Connection` reports as live is never one the policy aborted or started closing. | An app opens subscriptions on a session that is going away. | | `StatusTruthful` | Connected is on the policy's live session, Migrating on its predecessor while nothing else is live, Closed exactly when the policy is done, and the predecessor `Connection::close` drains is the policy's. | Status or `close` acts on the wrong session, or a closed loop looks alive. | +| `ConnectedMeansEstablished` | A `Connection` reports a session only once the policy reported it established (`Action::Established`): its peer accepted the credential or drained it with a GOAWAY, or, on a version that never answers the credential (moq-lite 03/04), it stayed up past the healthy bar. Uptime alone never establishes a session on a version that answers. | A refused session reads as connected, and `established()` returns `Ok` for it, or a lite-07 session the peer never accepted reads as connected once it is a second old. | | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | -| `FirstWalkFailsFast` | Until a session has stayed up past the healthy bar or got a GOAWAY, which no refusal sends, no failed dial or closed session backs off: either ends the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | +| `FirstWalkFailsFast` | Until a session was reported established (see `ConnectedMeansEstablished`; a GOAWAY counts, since no refusal sends one), no failed dial or closed session backs off: either ends the policy. Only a session reported established establishes the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run @@ -28,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 1.5M distinct states, under a minute. The second checks the same rules under `Restart::Handoff` over three sessions: about 330k states. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 19M distinct states, about four minutes. The second checks the same rules under `Restart::Handoff` over three sessions: about 3.4M states, under a minute. Each session picks whether its version answers the credential. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -40,17 +41,17 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `27b46ef35bdf284ba61158521723768436cd8460`. Run `git log 27b46ef35b.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `51e8538a84adacd16f91ef9ef6d54acfab7289db`. Run `git log 51e8538a84.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | -| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | -| `rs/moq-net/src/reconnect.rs` `established`, set by `end` and by a GOAWAY in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | +| `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `Establish`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | +| `rs/moq-net/src/reconnect.rs` `established`, set in `poll` by a GOAWAY, the session's acceptance, or on a silent version the healthy bar, and `walk_failed` and `closed` failing fast until it is set | `established`, the session's `accepted`, the session's `silent` (`Session::is_silent`), `Healthy`, `Ages`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "handoff"` (`RestartHandoff.cfg`) | | `rs/moq-net/src/reconnect/redirect.rs` `Redirect::target` | `Refused` and `Endpoints` | -| `rs/moq-tokio/src/connection.rs` `run`, `Shared::connected`, `migrating`, `retired`, `offline`, `end` | `Connected`, `Apply`, `OnMigrating`, `OnRetired`, `OnOffline`, `OnDone` | +| `rs/moq-tokio/src/connection.rs` `run`, `Shared::connected`, `migrating`, `retired`, `offline`, `end` | `Connected`, `Apply`, `OnEstablished`, `OnMigrating`, `OnRetired`, `OnOffline`, `OnDone` | ## Not covered diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 3a488d7368d..eb356e452c7 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -299,8 +299,9 @@ typedef ConnectionState = MoqConnectionState; /// Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. /// A null field keeps its default, zero means zero (retry at once, or give up -/// at the first failure), and `maxUnlimited` lifts the ceiling. Until a session -/// has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed +/// at the first failure), and `maxUnlimited` lifts the ceiling. Until the peer +/// has accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed /// session, such as a refused credential, ends the connection with its error /// rather than retrying. typedef Backoff = MoqBackoff; diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 22d787ab48e..36ee3f65a15 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -60,8 +60,9 @@ final class ConnectOptions { /// The WebSocket fallback raced against QUIC for `http(s)` URLs. final WebSocketConfig? websocket; - /// Set true for a one-shot dial. By default a session that stayed up past 1s - /// or was drained by a GOAWAY redials with backoff whenever the transport + /// Set true for a one-shot dial. By default a session whose credential the peer accepted, + /// or that the peer drained with a GOAWAY (or, on moq-lite 03/04, which never answer it, + /// that stayed up 1s), redials with backoff whenever the transport /// drops; before then, a failed dial or a closed session, such as a refused /// credential, ends it with that error instead. final bool once; diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 4fca7f8465f..65b4ddf1173 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -91,7 +91,8 @@ codec config, a pad requested again, or a stream resumed after EOS publishes as `url`, `broadcast`, `linger`, `connect-version`, `connect-websocket-enabled`, `tls-disable-verify`, `quic-idle-timeout`, `quic-keylog`, `quic-keep-alive`, and read-only `status`, `connected`, `moq-version`, and `estimated-send-rate`, `estimated-recv-rate`, `connection-stats`, and -`sessions`. Once a session has stayed up past a second, the sink reconnects for as long as +`sessions`. Once the relay has accepted a session (on moq-lite 03/04, which never answer, once +one has stayed up for a second), the sink reconnects for as long as the pipeline runs, whatever ends each session. Before then it reports `failed` and posts an error instead of retrying: a first connect that reaches no relay, or a session the relay refuses, such as for a rejected token. @@ -172,8 +173,8 @@ its pads without EOS until it is announced again, so the pipeline does not end on its own. Losing the relay connection also holds the pads: `moqsrc` redials without a retry deadline and resumes when the broadcast returns. A first connect that reaches no relay posts an error rather than retrying, as does a first -connection the relay refuses, or a session it closes before the connection has -stayed up for a second. Once playback has begun, a session the relay closes is +connection the relay refuses, or a session it closes before accepting it (on +moq-lite 03/04, which never answer, before it has stayed up for a second). Once playback has begun, a session the relay closes is redialed, whatever its code, and a refused redial is retried like any other failure. A refused path or catalog, malformed input or catalog, or a catalog its publisher ended with an application code posts an error. A media track its diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index b52a06f4c8a..3613c972ca5 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -257,7 +257,7 @@ Set `ClientConfig::tokens` instead of putting the credential in the URL. The fir ## Stats -`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. +`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. ## Shutdown diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 7f8b38ec8f4..62e32b5dc12 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -160,7 +160,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` drains for up to one second and reports its result. `session.shutdown()` requests the drain synchronously; `await session.closed()` waits for its result. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements and grants also release their consumer; server listeners leave their shared server usable. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` or disposing its `MoqRoute` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `client.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps: ...)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps: ...)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `dispose()` it once the broadcast stops being watched. `setTarget(targetBps: ...)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `OutOfRangeMoqException`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. - **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential: credential)` or `claim.setAuth(credential: credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index cb19909cacb..6b50cb5a11e 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -133,7 +133,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown()` requests that drain immediately; `session.Closed(ctx)` waits for its result, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `Finish()` on its `media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.Close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `Session().Viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `Watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `Rate()` reads its slice in bits per second, nil while there is no estimate, so hold the current selection. `Update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `Close()` it once the broadcast stops being watched, from the goroutine that last used it. `SetTarget(&bps)` sends another target, zero sends none, and nil follows the viewers again. `Watch`, `Update`, and `SetTarget` return `moq.ErrOutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. Client viewers survive reconnects and read nil while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Cancel()` it when done. - **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `Session().Credential(token)` returns a `Credential` that `Route.SetAuth` or `Claim.SetAuth` names (nil for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `Answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `Answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `Answered(ctx)` returns the verdict, and `Cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/index.md b/doc/lib/index.md index 7b29ed3811d..78fb2806cfe 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -67,7 +67,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. Until a session has stayed up past 1s or been drained by a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). +- **Reconnect** automatically with backoff when the transport drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s) it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. A session reads as connected only once the peer accepts its credential (or, on moq-lite 03/04, which never answer it, once it stays up 1s), so a refused one never does. The backoff's `max` and `timeout` read the same everywhere: unset takes the default (5s, and never give up), zero means zero (retry at once, or give up at the first failure), and a separate off spelling lifts the bound (`none` in moq-tokio's flags and config, `null` in JS, `max_unlimited` or a null `timeout` in the bindings). - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 8f95eedee1e..7cf42983efc 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -111,7 +111,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `moq.shutdown()` requests a drain immediately; suspending `moq.session.closed()` waits up to one second and reports its result. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.end()` takes it offline. How long to keep it up is yours to decide. A track still open at the end ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `moq.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `close()` it, or leave `use { }`, once the broadcast stops being watched. `setTarget(bps)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `MoqException.OutOfRange`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. - **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential)` or `claim.setAuth(credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. Suspending `credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 87f965ad3f3..ce6f03b5829 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -125,7 +125,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `await session.closed()` waits up to one second and reports its result. `cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `moq.media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(max_bps)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `None` while there is no estimate (keep the current rendition), and `update(max_bps)` changes the ceiling. Holding a viewer requests the estimate like `recv_bandwidth()`, and dropping it releases the claim. The session sends the summed ceilings as its PROBE target; `set_target(bps)` overrides that, `0` sends none, and `None` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, raise `moq.Error.OutOfRange`. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. - **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session.credential(token)` returns a `Credential` that `route.set_auth(credential)` or `claim.set_auth(credential)` names; setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` raises the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 6cbc0cdf4a0..f469603c2e3 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -108,7 +108,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `try await session.closed()` waits up to one second and reports its result. `session.cancel(code:)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try finish()` on its `Media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()`, or releasing its route, takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(maxBps:)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `nil` while there is no estimate (keep the current rendition), and `update(maxBps:)` changes the ceiling. Holding a viewer requests the estimate like `recvBandwidth()`, and releasing it releases the claim. The session sends the summed ceilings as its PROBE target; `setTarget(targetBps:)` overrides that, `0` sends none, and `nil` follows the viewers again. `watch`, `update`, and `setTarget` throw `MoqError.OutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. - **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `try session.credential(token)` returns a `Credential` that `try route.setAuth(credential)` or `try claim.setAuth(credential)` names (`nil` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `try await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/go/wrapper/client.go b/go/wrapper/client.go index 87f69548a89..90d984c598c 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -204,8 +204,9 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // Backoff is the retry pacing for automatic reconnects: the delay starts at // Initial, multiplies by Multiplier after each failed attempt, and caps at Max. -// After Timeout of consecutive failures the connection gives up for good. Until a -// session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a +// After Timeout of consecutive failures the connection gives up for good. Until the +// peer has accepted a session's credential or drained a session with a GOAWAY +// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a // closed session, such as a refused credential, ends the connection with its // error rather than retrying. // @@ -251,7 +252,8 @@ type Client struct { } // Dial starts a MoQ connection and returns without waiting for its first native dial to succeed. Cancel ctx -// to abort an in-flight connect. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed +// to abort an in-flight connect. Until the peer has accepted a session's credential or drained a session with a GOAWAY +// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed // dial or a closed session, such as a refused credential, closes the session with its error (see Session.Closed) // rather than retrying. func Dial(ctx context.Context, url string, config ClientConfig) (*Client, error) { diff --git a/go/wrapper/reconnect_test.go b/go/wrapper/reconnect_test.go index 574678961d2..36cae8ca0e0 100644 --- a/go/wrapper/reconnect_test.go +++ b/go/wrapper/reconnect_test.go @@ -229,6 +229,10 @@ func TestReconnectAcrossRelayRestart(t *testing.T) { if frame, err := trackConsumer.ReadFrame(ctx); err != nil || frame == nil { t.Fatalf("frame before restart: frame=%+v err=%v", frame, err) } + // Connected once the relay accepted each session, so the drop below redials rather than + // ending them as refusals. + waitConnects(t, ctx, publisher.Session(), 1) + waitConnects(t, ctx, subscriber.Session(), 1) if state := subscriber.Session().State(); !isConnected(state) { t.Fatalf("not serving before any drop: %+v", state) } @@ -236,9 +240,6 @@ func TestReconnectAcrossRelayRestart(t *testing.T) { t.Fatal("a live session must report its negotiated version") } - // Past the 1s healthy bar, so the drop redials rather than ending the sessions. - time.Sleep(time.Second) - // Drop every session and the listener, then bind a replacement on the same // port: a deploy, with an origin the replacement has never seen. current = current.restart(t, ctx) diff --git a/go/wrapper/session.go b/go/wrapper/session.go index a70218a474d..d4c8031a8bc 100644 --- a/go/wrapper/session.go +++ b/go/wrapper/session.go @@ -46,7 +46,9 @@ type ConnectionPhaseConnecting struct { Failure error } -// ConnectionPhaseConnected means a session is serving. +// ConnectionPhaseConnected means a session is serving: its peer accepted the +// credential, or on moq-lite 03/04, which never answer it, the session stayed up 1s. +// A refused session never reaches this phase. type ConnectionPhaseConnected struct { // Version names the live session's negotiated protocol, like "moq-lite-03". Version string diff --git a/js/net/src/auth_session.ts b/js/net/src/auth_session.ts index d4efcb17ada..69f3888519d 100644 --- a/js/net/src/auth_session.ts +++ b/js/net/src/auth_session.ts @@ -620,6 +620,12 @@ export interface AuthSessionProps { * session without AUTH that is live before its peer has checked the credential. */ awaiting?: boolean; + /** + * Whether the version never answers the connection credential (a moq-lite-03/04 client), + * so the peer refuses it only by closing the session. Its credential still counts as + * accepted, but {@link AuthSession.accepted} never says so. + */ + silent?: boolean; } /** @@ -657,6 +663,9 @@ export class AuthSession { // The peer's verdict on the connection credential this session presented: null once accepted, // the refusal, or undefined while it has none. #credential = new Signal(undefined); + // Whether the peer said it accepts the connection credential, which a silent version never does. + #accepted = new Signal(false); + #silent: boolean; #closed = false; // Whoever answers the peer's tokens is decided once, after the task that established @@ -665,16 +674,20 @@ export class AuthSession { #decided = yieldNow(); /** Throws {@link Duplicate} while `handle` backs another active session, or the handle's end. */ - constructor({ wire, requests, peerGrant, handle, draining, awaiting }: AuthSessionProps) { + constructor({ wire, requests, peerGrant, handle, draining, awaiting, silent }: AuthSessionProps) { this.handle = handle ?? ownHandle(); this.#state = stateOf(this.handle); this.#wire = wire; this.#requests = requests ?? false; this.#peerGrant = peerGrant; this.#draining = draining ?? (() => false); + this.#silent = silent ?? false; this.#state.attach(this); // A session without AUTH has the peer's acceptance with its handshake, unless it awaits its SETUP. - if (!wire && !awaiting) this.#credential.set(null); + if (!wire && !awaiting) { + this.#credential.set(null); + if (!silent) this.#accepted.set(true); + } // Present the connection's own credential right away, so both sides learn their // grant without waiting on the app. @@ -696,9 +709,28 @@ export class AuthSession { return this.#credential; } + /** + * Whether the peer said it accepts the connection credential: with its handshake, its SETUP, + * or the AUTH_OK to the connection's own credential (moq-lite-07). Never on a silent version. + * The session ending leaves it as it was. + */ + get accepted(): Getter { + return this.#accepted; + } + + /** + * Whether the version never answers the connection credential (a moq-lite-03/04 client), so + * only uptime suggests the peer took it. + */ + get silent(): boolean { + return this.#silent; + } + /** The peer's SETUP arrived, so it accepted the connection credential a session without AUTH presented. */ - accepted(): void { - if (!this.#wire) this.#credential.set(null); + accept(): void { + if (this.#wire) return; + this.#credential.set(null); + this.#accepted.set(true); } /** The union of this session's grants, undefined until its peer first answers. */ @@ -959,7 +991,10 @@ export class AuthSession { answered = granted = replied = true; delay = undefined; this.#replied = true; - if (presented.setup) this.#credential.set(null); + if (presented.setup) { + this.#credential.set(null); + this.#accepted.set(true); + } this.#answered(presented); this.#recompute(); continue; diff --git a/js/net/src/connection/answered.test.ts b/js/net/src/connection/answered.test.ts index 61b36d9d3a1..ef91a22a3d1 100644 --- a/js/net/src/connection/answered.test.ts +++ b/js/net/src/connection/answered.test.ts @@ -4,7 +4,7 @@ import { Signal } from "@moq/signals"; import type { Request } from "../auth.ts"; import { SessionCode, SessionError } from "../error.ts"; import * as Lite from "../lite/index.ts"; -import { createMockTransportPair, manual, waitFor } from "../mock.ts"; +import { createMockTransportPair, type MockTransport, manual, waitFor } from "../mock.ts"; import * as Path from "../path.ts"; import { wireOf } from "../wire.ts"; import { accept as acceptSession } from "./index.ts"; @@ -44,20 +44,25 @@ function relay(decide: (token: string, request: Request) => void) { /** * A lite-06 relay, which has no AUTH: it accepts any credential but `refused`, whose session it - * closes once `release` resolves, before its SETUP. + * closes once `release` resolves, before its SETUP. `dials` counts the dials, and `served` holds + * the server side of each session it accepted. */ function closing(refused: string, release: Promise) { + const served: MockTransport[] = []; + const dials = new Signal(0); const stub = function StubWebTransport(target: string | URL) { + dials.update((n) => n + 1); const pair = createMockTransportPair(Lite.ALPN_06); const url = new URL(target); if (url.searchParams.get("jwt") === refused) { void release.then(() => pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" })); } else { + served.push(pair.server); void acceptSession({ transport: pair.server, url }); } return pair.client; }; - return stub as unknown as typeof WebTransport; + return { dials, served, stub: stub as unknown as typeof WebTransport }; } function loop(versions: string[], tokens: Signal) { @@ -110,13 +115,20 @@ test("a close that refuses the connection credential answers a session without A using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); - globalThis.WebTransport = closing("old", release); - const tokens = new Signal(["old"]); + const { dials, served, stub } = closing("old", release); + globalThis.WebTransport = stub; + const tokens = new Signal(["ok"]); const reload = loop(["moq-lite-06"], tokens); try { + // The server's SETUP accepts the first credential, so the refusing session's close is + // retried rather than ending the URL. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); - // A session that stayed up past the healthy bar, so its close is retried rather than ending the URL. - await clock.advance(Duration.millis(1500)); + // Swapped while no session is live, so no session takes it in band. + served[0]?.close({ closeCode: 0, reason: "restart" }); + await clock.drive(waitFor(reload.established, (established) => established === undefined)); + tokens.set(["old"]); + await clock.drive(waitFor(dials, (n) => n >= 2)); + await clock.advance(Duration.millis(10)); const waiting = refusal(reload.answered()); resolve(); const err = await clock.drive(waiting); @@ -131,6 +143,31 @@ test("a close that refuses the connection credential answers a session without A } }); +test("a lite-07 session up past the healthy bar without an AUTH_OK never reads as connected", async () => { + using clock = manual(); + const original = globalThis.WebTransport; + const held = new Signal([]); + globalThis.WebTransport = relay((_token, request) => held.update((requests) => [...requests, request])); + const reload = loop(["moq-lite-07-wip"], new Signal([])); + const statuses: string[] = []; + const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); + try { + const [credential] = await clock.drive(waitFor(held, (requests) => requests.length > 0)); + // Uptime stands in for an answer only on moq-lite 03/04, which have none. + await clock.advance(Duration.millis(5000)); + expect(statuses).not.toContain("connected"); + expect(reload.established.peek()).toBeUndefined(); + + credential?.accept(GRANT); + await clock.drive(waitFor(reload.status, (status) => status === "connected")); + expect(reload.established.peek()).toBeDefined(); + } finally { + unsubscribe(); + reload.close(); + globalThis.WebTransport = original; + } +}); + test("a kept credential counts only once the peer refuses it", async () => { using clock = manual(); const original = globalThis.WebTransport; diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 15f9dec3782..17bc24114c6 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -442,25 +442,29 @@ for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, Se /** * Stub WebTransport for lite-06, which carries the credential in the URL: refuses `refused` - * once `release` resolves, and accepts any other credential, recording each dialed jwt. + * once `release` resolves, never having sent the SETUP that would accept it, and accepts any + * other credential, recording each dialed jwt. */ function refusing(refused: string, release: Promise) { const dialed = new Signal([]); + // The server sides of the sessions it accepted. + const served: MockTransport[] = []; const stub = function StubWebTransport(target: string | URL) { const jwt = new URL(target).searchParams.get("jwt"); dialed.update((jwts) => [...jwts, jwt]); const pair = createMockTransportPair(Lite.ALPN_06); - void accept(pair.server, new URL(target)).then(async () => { - if (jwt !== refused) return; - await release; - pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }); - }); + if (jwt === refused) { + void release.then(() => pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" })); + } else { + served.push(pair.server); + void accept(pair.server, new URL(target)); + } return pair.client; }; - return { dialed, stub: stub as unknown as typeof WebTransport }; + return { dialed, served, stub: stub as unknown as typeof WebTransport }; } -test("a first session refused before it is healthy stops the URL, even for replaced tokens", async () => { +test("a first session refused before its credential is accepted stops the URL, even for replaced tokens", async () => { using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); @@ -475,18 +479,59 @@ test("a first session refused before it is healthy stops the URL, even for repla tokens, backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); + const statuses: string[] = []; + const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); try { - await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + await clock.drive(waitFor(dialed, (dialed) => dialed.length > 0)); + await clock.advance(Duration.millis(10)); tokens.set(["new"]); resolve(); - // The refusal arrives after the session was accepted, as a relay rejects a token on - // moq-lite 03 or later, yet it stops the URL like a failed first attempt. + // The refusal arrives once the session is live, before the server's SETUP, as a relay + // rejects a token on moq-lite 03 or later, and stops the URL like a failed first attempt. const err = await clock.drive(waitFor(reload.error, (error) => error !== undefined)); expect(err).toBeInstanceOf(SessionError); expect((err as SessionError).code).toBe(SessionCode.Unauthorized); await clock.advance(Duration.millis(100)); expect(dialed.peek()).toEqual(["old"]); expect(reload.established.peek()).toBeUndefined(); + // The refused session never read as connected. + expect(statuses).not.toContain("connected"); + } finally { + unsubscribe(); + reload.close(); + globalThis.WebTransport = original; + } +}); + +test("a first session dropped right after the peer accepted its credential redials", async () => { + using clock = manual(); + const original = globalThis.WebTransport; + const dials = new Signal(0); + let first: ReturnType | undefined; + const accepted = Promise.withResolvers(); + const stub = function StubWebTransport(target: string | URL) { + dials.update((n) => n + 1); + const pair = createMockTransportPair(Lite.ALPN_06); + first ??= pair; + void accept(pair.server, new URL(target)).then(() => accepted.resolve()); + return pair.client; + }; + globalThis.WebTransport = stub as unknown as typeof WebTransport; + + const reload = new Reload({ + url: new URL("https://example.com/"), + versions: ["moq-lite-06"], + websocket: { enabled: false }, + backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, + }); + try { + const established = await clock.drive(waitFor(reload.established, (session) => session !== undefined)); + await clock.drive(accepted.promise); + // The server's SETUP accepted the credential, long before the healthy bar. + await clock.drive(waitFor(wireOf(established as Session).auth.accepted, (yes) => yes)); + first?.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }); + await clock.drive(waitFor(dials, (n) => n >= 2)); + expect(reload.error.peek()).toBeUndefined(); } finally { reload.close(); globalThis.WebTransport = original; @@ -497,10 +542,10 @@ test("a refused credential keeps redialing until the tokens change", async () => using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); - const { dialed, stub } = refusing("old", release); + const { dialed, served, stub } = refusing("old", release); globalThis.WebTransport = stub; - const tokens = new Signal(["old"]); + const tokens = new Signal(["ok"]); const reload = new Reload({ url: new URL("https://example.com/"), versions: ["moq-lite-06"], @@ -509,12 +554,19 @@ test("a refused credential keeps redialing until the tokens change", async () => backoff: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); try { - // Refused only once a session was healthy: until then, a refusal stops the URL. + // The server's SETUP accepts the first credential, which establishes the URL: until + // then, a refusal stops it. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); - await clock.advance(Duration.millis(1000)); + tokens.set(["old"]); + served[0]?.close({ closeCode: 0, reason: "restart" }); resolve(); - await clock.drive(waitFor(dialed, (jwts) => jwts.length >= 3)); - expect(dialed.peek().every((jwt) => jwt === "old")).toBe(true); + await clock.drive(waitFor(dialed, (jwts) => jwts.length >= 4)); + expect( + dialed + .peek() + .slice(1) + .every((jwt) => jwt === "old"), + ).toBe(true); expect(reload.error.peek()).toBeUndefined(); tokens.set(["new"]); await clock.drive(waitFor(dialed, (jwts) => jwts.includes("new"))); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 324af557372..749e812e8a0 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -40,10 +40,14 @@ const PROBE_MAX = Duration.secs(300); * Exponential backoff settings for the reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Nothing retries until a session to the current URL has been established, which - * takes staying up past {@link Backoff.healthy} or being drained by a GOAWAY, which no - * refusal sends: an attempt failing, or a session closing sooner, stops that URL with its error. That covers a relay refusing a session right after - * accepting it, as it does a rejected token on moq-lite 03 or later. After that, every failure is retried, + * lockstep. Nothing retries until a session to the current URL has been established: the peer + * accepts the connection credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the + * connection's own credential) or drains the session with a GOAWAY, which no refusal sends. On + * moq-lite 03/04, which answer the credential with nothing, staying up past + * {@link Backoff.healthy} establishes it too; a version with an answer waits for it however long + * the session stays up. An attempt failing, or a session closing sooner, stops that + * URL with its error. That covers a relay refusing a live session, as it does a rejected token + * on moq-lite 03 or later. After that, every failure is retried, * including an attempt that outlives {@link ConnectProps.timeout}, until * {@link Backoff.timeout} runs out, whatever the error, an auth rejection included; only a * refused GOAWAY redirect stops the current URL sooner. A new URL or a disable/re-enable starts another sequence. @@ -70,6 +74,9 @@ export type Backoff = { /** * How long a session must stay up to count as healthy (default: 1000ms; must be positive). * + * Where the peer never answers the connection credential, as on moq-lite 03/04, the first + * healthy session establishes the URL. Elsewhere only the peer's answer or a GOAWAY does, + * however long a session stays up. * A healthy session's end resets the delay and the timeout. One that ends sooner is a * failed attempt, so a peer that accepts and then drops every session still escalates the * delay and exhausts the timeout. @@ -191,8 +198,8 @@ export type ReloadStatus = "connecting" | "connected" | "disconnected"; * The reconnect loop behind a Connection handle: connects, waits for the session to end, then * redials with exponential backoff. * - * The first attempt for a URL fails fast instead: until a session to it has stayed up past - * {@link Backoff.healthy} or been drained by a GOAWAY, a failed attempt or a closed session sets {@link Reload.error} + * The first attempt for a URL fails fast instead: until a session to it was established (see + * {@link Backoff}), a failed attempt or a closed session sets {@link Reload.error} * rather than retrying, so a setup mistake such as an * untrusted certificate or no version in common surfaces at once. A page opened before its * relay is reachable therefore stays disconnected until a new URL or a disable/re-enable. @@ -213,7 +220,11 @@ export class Reload { */ tokens: Signal; - /** Current connection status. */ + /** + * Current connection status. A session reads as connected only once its peer accepts the + * credential (on moq-lite-03/04, which never answer it, once it stays up past the healthy + * bar), so a refused one never does. + */ status = new Signal("disconnected"); /** @@ -373,8 +384,8 @@ export class Reload { // loop is disabled. #givenUpHref: string | undefined; - // The configured href a session stayed up past the healthy bar for, or the peer drained with a - // GOAWAY. Until it matches, a failed + // The configured href a session's peer accepted the credential of or drained with a GOAWAY, + // or a moq-lite-03/04 session stayed up past the healthy bar for. Until it matches, a failed // attempt or a closed session gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which // resume a URL that worked. #establishedHref: string | undefined; @@ -559,7 +570,7 @@ export class Reload { effect.cleanup(() => { if (current) { // A session torn down while live, as on a page hide, may already have earned it. - if (connected !== undefined) this.#settle(href, connected); + if (connected !== undefined) this.#settle(href, current, connected); this.#end(current); if (this.established.peek() === current) this.#serve(undefined); current = undefined; @@ -623,11 +634,32 @@ export class Reload { } current = connection; this.#verdict.connected(connection); + connected = now(); - // The replacement serves now; a predecessor keeps draining its groups in flight. - this.#serve(connection); + // Serve only once the session is established: the peer accepted its credential or + // drained it with a GOAWAY. On a version that never answers (moq-lite-03/04), + // staying up past the healthy bar is the only sign, so it counts there alone. + // Until then it may still be a refusal, which never reads as connected. A close + // first is handled below. + const auth = wireOf(connection).auth; + const bar = this.#pacing().healthy; + const acceptance = auth.accepted; + if (!acceptance.peek()) { + const waiting = new AbortController(); + const waits: unknown[] = [Signal.race(acceptance), connection.closed, connection.draining]; + if (auth.silent) waits.push(sleep(bar, { signal: waiting.signal }).catch(() => {})); + await effect.race(...waits).finally(() => waiting.abort()); + if (signal.aborted) return; + } + if ( + acceptance.peek() || + connection.draining.peek() !== undefined || + (auth.silent && now().since(connected).ge(bar)) + ) { + // The replacement serves now; a predecessor keeps draining its groups in flight. + this.#serve(connection); + } this.#setDialing(false); - connected = now(); // A cancelled effect resolves undefined, so the sentinel tells the session // closing (null for clean, an Error otherwise) apart from this run being @@ -655,10 +687,10 @@ export class Reload { reached = raced; } const ended = raced instanceof Reached ? { uri: "" } : raced; - this.#settle(href, connected); + this.#settle(href, connection, connected); if (ended === null || ended instanceof Error) { - // Until a session has stayed up past the healthy bar, a close stops the URL as a - // failed attempt does: it is how a relay refuses a session it already accepted. + // Until a session was established, a close stops the URL as a failed attempt + // does: it is how a relay refuses a live session before accepting its credential. // No error kind is read. if (this.#establishedHref !== href) { console.warn(`connection closed, not retrying: ${ended ? reason(ended) : "no error"}`); @@ -747,9 +779,8 @@ export class Reload { return; } - // Attempts fail fast until a session was established: a URL that never held one past - // the healthy bar is more likely misconfigured than restarting, and only the app can - // fix that. + // Attempts fail fast until a session was established: a URL whose peer never accepted + // one is more likely misconfigured than restarting, and only the app can fix that. if (this.#establishedHref !== href) { console.warn("connection failed, not retrying:", err); this.#giveUp(error(err)); @@ -923,9 +954,14 @@ export class Reload { if (draining) this.#end(draining); } - // Mark `href` established once the session connected at `connected` stayed up past the healthy bar. - #settle(href: string, connected: Instant): void { - if (now().since(connected).ge(this.#pacing().healthy)) this.#establishedHref = href; + // A session establishes its URL once the peer accepted its credential, which a closed + // session still reports, or, on a version that never answers the credential + // (moq-lite-03/04), once it stayed up past the healthy bar. + #settle(href: string, session: Session, connected: Instant): void { + const auth = wireOf(session).auth; + if (auth.accepted.peek() || (auth.silent && now().since(connected).ge(this.#pacing().healthy))) { + this.#establishedHref = href; + } } // Resolved per use rather than at construction, so an edit to `backoff` (including one that diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index f442e612353..512b4f8b5d2 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -231,6 +231,9 @@ export class Connection implements Session { draining: () => this.#goaway.peek() !== undefined || this.#phase.kind !== "open", // Without AUTH, the server checks a client's credential before its SETUP, which the client reads after going live. awaiting: client && hasSetupStream(version) && !hasAuth(version), + // Lite-03/04 have neither, so the peer refuses the credential only by closing the + // session. Lite-01/02 answered in the negotiation SETUP already read. + silent: client && (version === Version.DRAFT_03 || version === Version.DRAFT_04), }); // Left before anything observes the close, so a successor on the same handle never // finds this session still active. @@ -605,7 +608,7 @@ export class Connection implements Session { if (!this.#client && setup.path !== undefined) throw new ProtocolViolation("Path on a URI-carrying binding"); this.#peerSetup.set(setup); - this.#auth.accepted(); + this.#auth.accept(); this.#peerRole.set(setup.role); this.#peerImplementation.resolve(setup.implementation); } else { diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index c2e33504612..1442a09c64e 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -390,8 +390,9 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState /** * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. * A null field keeps its default, zero means zero (retry at once, or give up at the first failure), - * and `maxUnlimited` lifts the ceiling. Until a session has stayed up past 1s or been drained by a - * GOAWAY, a failed dial or a closed session, such as a refused credential, ends the connection with + * and `maxUnlimited` lifts the ceiling. Until the peer has accepted a session's credential or + * drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed + * up 1s), a failed dial or a closed session, such as a refused credential, ends the connection with * its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index ca281c2ac8a..82516d69c7d 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -160,7 +160,7 @@ client = moq.Client( - `.cancel(code)`, `session.shutdown()`. Cancel immediately, or request a drain now and await `session.closed()` for its result, including a delivery timeout after one second. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - `.publish() → OriginProducer`, `.consume() → OriginConsumer`. The wired origin sides. - `.stats() → ConnectionStats`. Snapshot RTT, bandwidth estimates, and byte/packet counters. - - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends it with that error. + - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until the peer has accepted a session's credential or drained a session with a GOAWAY (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends it with that error. - `.allocator() → Allocator`. Divide the send estimate between encoders and app-owned tracks. - `.viewers() → Viewers`. Divide the receive estimate between watched broadcasts: `.watch(max_bps) → Viewer` claims up to a ceiling, `Viewer.rate()` reads its slice (`None` while there is no estimate), and dropping the viewer releases it. The summed ceilings are the PROBE target; `.set_target(bps)` overrides it (`0` sends none, `None` follows the viewers again). - `.grants() → GrantConsumer`. On `moq-lite-07-wip`, `.grant()` reads the peer's `Grant` and `async for grant in session.grants()` follows it. diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index 36702a24c8f..850c3746b66 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -345,7 +345,8 @@ def _track_info(info: TrackInfo | None) -> MoqTrackInfo | None: class Backoff: """Retry pacing for the automatic reconnect. - Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or + Until the peer has accepted a session's credential or drained a session with a GOAWAY + (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed session, such as a refused credential, ends the connection with its error rather than retrying. diff --git a/py/moq-rs/tests/test_server.py b/py/moq-rs/tests/test_server.py index 1c79515a072..f670e65360a 100644 --- a/py/moq-rs/tests/test_server.py +++ b/py/moq-rs/tests/test_server.py @@ -139,10 +139,9 @@ async def accept_loop() -> None: assert state.connects == 1 assert session.stats().version is not None - # Kill the transport under the client, like a relay restart, once past - # the 1s healthy bar so the drop redials rather than ending the session. + # Kill the transport under the client, like a relay restart. CONNECTED + # means the peer accepted the session, so the drop redials. await asyncio.wait_for(accepted.wait(), timeout=10) - await asyncio.sleep(1) sessions[0].cancel(0) # Nothing accepts the redial until the gate opens, so DISCONNECTED diff --git a/quest/m0/nightly/credential-established.md b/quest/m0/nightly/credential-established.md deleted file mode 100644 index 33eff2b9515..00000000000 --- a/quest/m0/nightly/credential-established.md +++ /dev/null @@ -1,49 +0,0 @@ -# [M] A reconnecting client is established by the credential's answer - -## Goal - -A reconnecting client (Rust `Reconnect` and `moq_tokio::Connection`, JS -`Reload`, and every binding through `moq-ffi`) counts a session as established -once the peer accepts the connection credential, instead of after it stays up -`Backoff::healthy` (1 s). A close before that answer is a refusal and ends the -policy, as today. Versions with no acceptance message keep the uptime bar. No -wire change. - -The tests [moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817) gave -real 1 s waits (Go, Python, moq-c), and the OBS tests in draft -[moq-dev/moq#9016](https://github.com/moq-dev/moq/pull/9016), drop the session -right after it is accepted instead. - -## Plan - -Found by the triage of Nightly run -[44127032469](https://github.com/moq-dev/moq/actions/runs/44127032469) -(2026-11-17): the OBS output and source tests drop a session the moment it goes -live, which #8817 now treats as a refusal. #9016 fixes them with real 1 s -waits; the maintainer chose this quest instead, and #9016 waits on it. - -Where acceptance already shows (researched 2026-11-17): - -- lite-05/06 and moq-transport: the server sends SETUP only from `ok()`, and - `moq-relay` judges the token before `ok()` - (`rs/moq-relay/src/connection.rs`), so a refused session never sends it. -- lite-07: also the AUTH_OK answering each side's empty-token Auth Stream, - which carries the grant for the connection credential. -- lite-01/02: the SERVER_SETUP reply. -- lite-03/04: nothing. They keep `Backoff::healthy` as the bar. -- `Tokens::credential` (`rs/moq-net/src/auth/tokens.rs`) already computes this - verdict across versions; `js/net/src/connection/tokens.ts`'s `Verdict` is its - counterpart. Reuse them rather than reading SETUP again. - -Decided 2026-11-17 (maintainer, /quest-plan): - -- Signal: ✅ the existing credential verdict (server SETUP, or AUTH_OK on - lite-07) (rejected: a new SETUP_OK message, which duplicates AUTH_OK; 1 s of - uptime for every version). -- ✅ Versions with no acceptance message keep a timeout, `Backoff::healthy`. -- Update `Handover.tla` and the verify-reconnect skill where `established` is - set, and the `Backoff` docs in every binding. - -## Related - -- [Connect policy state](/quest/m4/connect-policy-state.md) - moves the same policy into one place per language diff --git a/rs/libmoq/src/api.rs b/rs/libmoq/src/api.rs index 407baf96288..6eb52a56f38 100644 --- a/rs/libmoq/src/api.rs +++ b/rs/libmoq/src/api.rs @@ -1187,7 +1187,8 @@ pub struct moq_announce_event { pub enum moq_connection_phase { /// No connection has been established yet. MOQ_CONNECTION_PHASE_CONNECTING = 0, - /// A connection is serving. + /// A connection is serving: its peer accepted the credential, or on moq-lite 03/04, which + /// never answer it, the session stayed up 1s. A refused session never reaches this phase. MOQ_CONNECTION_PHASE_CONNECTED = 1, /// No connection is serving while the session redials. MOQ_CONNECTION_PHASE_DISCONNECTED = 2, @@ -1790,7 +1791,8 @@ unsafe fn connect_session( /// You should call [moq_session_close], even on error, to free up resources. /// /// Once connected, the session reconnects automatically with exponential backoff if the -/// connection drops. Until a session has stayed up past 1s or been drained by a GOAWAY, a +/// connection drops. Until the peer has accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a /// failed dial or a closed session, such as a refused credential, ends it instead, with that error as the terminal /// snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on /// each reconnect, since the origins outlive the underlying connection. diff --git a/rs/libmoq/src/test.rs b/rs/libmoq/src/test.rs index efc93bc8954..76dc5bb4863 100644 --- a/rs/libmoq/src/test.rs +++ b/rs/libmoq/src/test.rs @@ -9795,13 +9795,8 @@ fn a_credential_rides_every_reconnect() { let _first = first.accept(moq_net::auth::Grant::all()); assert_eq!(answer.recv().0, 0); - // The server ends the session once it was healthy; the client redials and presents the - // token again. - #[expect( - clippy::disallowed_methods, - reason = "the test thread has no runtime; the C API's session ages on the real clock" - )] - std::thread::sleep(Duration::from_secs(1)); + // The server ends the session the client saw connected, so accepted; the client redials + // and presents the token again. server.admit(); let again = server.token(); assert_eq!(again.token().as_ref(), b"kept", "the reconnect presents it again"); diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 91df5e9e37a..73686182c2d 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -202,7 +202,8 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// -/// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a +/// Until the peer has accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a /// closed session, such as a refused credential, ends the connection with its error rather /// than retrying. The delay starts at `initial`, multiplies by `multiplier` after each /// failed attempt, and caps at `max`. @@ -615,8 +616,9 @@ impl MoqClient { /// /// Once a session has been established, it automatically reconnects with backoff when /// the transport drops (unless [`MoqClientConfig::once`] is set), and broadcasts - /// consumed through it ride out the gap. Until a session has stayed up past 1s or been - /// drained by a GOAWAY it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the + /// consumed through it ride out the gap. Until the peer has accepted a session's credential or drained + /// a session with a GOAWAY + /// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s) it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the /// one it accepted, as a relay refusing a credential does, [`MoqSession::closed`] resolves /// with that error, so an app started before its relay is reachable must start the session /// again. @@ -737,7 +739,8 @@ pub enum MoqConnectionPhase { /// The latest failed dial, or none before the first one fails. failure: Option, }, - /// A session is serving. + /// A session is serving: its peer accepted the credential, or on moq-lite 03/04, which + /// never answer it, the session stayed up 1s. A refused session never reaches this phase. Connected { /// The live session's negotiated version, spelled like `moq-lite-03`. version: String, diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index d4ac78e218e..4fb2bfdf1a1 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7705,11 +7705,19 @@ async fn session_state_names_a_failing_first_dial() { )); assert_eq!(accepted.state().revision, terminal.revision); assert!(accepted.state_changed(terminal.revision).await.is_err()); - // The server ended the client's first session before it was healthy, which ends the - // client's session too rather than redialing. - let closed = kio::clock::timeout(TIMEOUT, session.closed()).await.unwrap(); - assert!(closed.is_err(), "the refused first session closed cleanly"); - assert!(matches!(session.state().phase, MoqConnectionPhase::Closed { .. })); + // The client reported the session connected, so the peer had accepted it, and its end is + // an outage the client redials rather than a refusal. + let dropped = kio::clock::timeout( + TIMEOUT, + wait_phase(&session, |phase| { + matches!(phase, MoqConnectionPhase::Disconnected { .. }) + }), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(dropped.connects, 1); + session.cancel(0); accepted.cancel(0); server.cancel(); } @@ -7798,8 +7806,7 @@ async fn client_reconnects_and_resumes_announcements() { .await .expect("first session timed out") .expect("accept task gone"); - // Past the 1s healthy bar, so the drop redials rather than ending the session. - kio::clock::sleep(Duration::from_secs(1)).await; + // Connected means the peer accepted the session, so the drop redials rather than ending it. first.cancel(0); let status = kio::clock::timeout( @@ -8050,9 +8057,8 @@ async fn cancelled_state_wait_does_not_swallow_the_next_transition() { Ok(status) => panic!("no transition was pending, got {status:?}"), } - // The transition the abandoned call would have eaten, once past the 1s healthy bar so - // the session redials rather than ends. - kio::clock::sleep(Duration::from_secs(1)).await; + // The transition the abandoned call would have eaten: a disconnect, since the session was + // accepted and so redials rather than ends. server_session.cancel(0); let status = kio::clock::timeout(TIMEOUT, cs.state_changed(status.revision)) diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index d8a8a432b47..33b73953aa0 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -293,8 +293,8 @@ impl Session { // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC // outages of any length, which is the trade this element wants: a pipeline nobody is watching - // should still be publishing when the relay comes back. The loop still ends when no session - // stays up past the healthy bar, such as a relay refusing the token, posting the bus error + // should still be publishing when the relay comes back. The loop still ends when the relay + // never accepted a session, such as when it refuses the token, posting the bus error // below. During an outage the pad threads keep writing (bounded // by moq-net's per-group eviction) and the relay catches up from a group boundary on // reconnect. A bounded policy is available via `ClientConfig::backoff`. diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 49f224b806b..95748af9107 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -230,8 +230,22 @@ struct Attached { /// gates can skip re-matching paths on every wakeup. epoch: u64, /// Whether the peer has accepted the connection, credential included: at once, unless - /// the session [awaits the peer's SETUP](Link::await_setup). - accepted: bool, + /// the session [awaits its answer](Link::await_acceptance) or the version + /// [has none](Link::silent). + acceptance: Acceptance, +} + +/// Whether a session's peer accepted the connection credential. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Acceptance { + /// With its handshake, or with the answer the session awaited. + Accepted, + /// Not yet: the peer's SETUP (a client without AUTH) or its AUTH_OK to the connection's + /// own credential (a lite-07 client) is still to come, and the peer can still refuse. + Awaiting, + /// Never said: the version has no answer, so the peer refuses only by closing the + /// session (a lite-03/04 client). + Silent, } /// One token as one session presents it. @@ -322,7 +336,7 @@ impl Attached { issued: BTreeMap::new(), scope: None, epoch: 0, - accepted: true, + acceptance: Acceptance::Accepted, }; attached.rescope(limit); attached @@ -1117,6 +1131,10 @@ impl Link { if !matches!(slot.phase, Phase::Pending { .. } | Phase::Granted(_)) { return; } + // The answer to the connection's own credential accepts the session. + if slot.setup && link.acceptance == Acceptance::Awaiting { + link.acceptance = Acceptance::Accepted; + } slot.phase = Phase::Granted(grant); link.replied = true; link.recompute(); @@ -1409,21 +1427,50 @@ impl Link { self.read(|link, _| link.closed.clone()) } - /// The peer accepts the connection credential only with its SETUP, which this client - /// reads after the session is live, while the server can still refuse it. - pub(crate) fn await_setup(&self) { - self.with(|link| link.accepted = false); + /// The peer accepts the connection credential only with an answer this client reads after + /// the session is live, while the server can still refuse it: its SETUP on a version + /// without AUTH, or the AUTH_OK to the connection's own credential on one with it. + pub(crate) fn await_acceptance(&self) { + self.with(|link| link.acceptance = Acceptance::Awaiting); + } + + /// The version never answers the connection credential: the peer refuses it only by + /// closing the session, however soon. + pub(crate) fn silent(&self) { + self.with(|link| link.acceptance = Acceptance::Silent); + } + + /// Whether the version never answers the connection credential, so only uptime can + /// suggest the peer took it. + pub(crate) fn is_silent(&self) -> bool { + self.read(|link, _| link.acceptance == Acceptance::Silent) } /// The peer's SETUP arrived, so it accepted the connection credential. pub(crate) fn accepted(&self) { self.with_state(|state, link| { - attached(&mut state.links, link).accepted = true; + let link = attached(&mut state.links, link); + if link.acceptance == Acceptance::Awaiting { + link.acceptance = Acceptance::Accepted; + } // Latch a set this acceptance completed, before a later end can replace its answer. state.record(); }); } + /// Ready once the peer said it accepts the connection credential, which a version without + /// that answer never does. A close afterward leaves it accepted. + pub(crate) fn poll_accepted(&self, waiter: &kio::Waiter) -> Poll<()> { + let id = self.id(); + let _ = ready_or!(self.state().poll(waiter, |state| { + match state.links.get(&id).map(|link| link.acceptance) { + Some(Acceptance::Accepted) => Poll::Ready(()), + _ => Poll::Pending, + } + })); + Poll::Ready(()) + } + /// End the session: fail every pending token here, leaving the handle's tokens for the /// next session. A handle the session created for itself ends with it. pub(crate) fn close(&self, err: Error) { diff --git a/rs/moq-net/src/auth/tokens.rs b/rs/moq-net/src/auth/tokens.rs index ca35fc6a287..fc05fafec60 100644 --- a/rs/moq-net/src/auth/tokens.rs +++ b/rs/moq-net/src/auth/tokens.rs @@ -229,7 +229,7 @@ pub(super) struct Configured { /// How the latest dial's session answers its connection credential. enum Connected { /// The version has no AUTH, so the peer accepted it with its handshake, or with its - /// SETUP when the session [awaits it](super::Link::await_setup). + /// SETUP when the session [awaits it](super::Link::await_acceptance). Accepted { link: u64 }, /// Its own AUTH stream on that session. Slot { link: u64, id: u64 }, @@ -252,12 +252,14 @@ impl Configured { Connected::Slot { link, id } => (link, id), Connected::Accepted { link } => { let link = state.links.get(link)?; - return match (link.accepted, &link.closed) { - (true, _) => Some(Ok(())), + let awaiting = link.acceptance == super::Acceptance::Awaiting; + return match (awaiting, &link.closed) { + // Accepted, or a version that never answers, whose live session counts as accepting it. + (false, _) => Some(Ok(())), // Closed before accepting it: the close is the answer, since the connection // keeps redialing with the same credential and nothing else would answer. - (false, Some(err)) => Some(Err(err.clone())), - (false, None) => None, + (true, Some(err)) => Some(Err(err.clone())), + (true, None) => None, }; } }; @@ -791,7 +793,7 @@ mod tests { let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); set.commit(&presented); let link = Link::attach(&set.handle, false, false).unwrap(); - link.await_setup(); + link.await_acceptance(); set.connect(link.id(), None); link } @@ -823,7 +825,7 @@ mod tests { let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); set.commit(&presented); let link = Link::attach(&set.handle, false, false).unwrap(); - link.await_setup(); + link.await_acceptance(); link.close(Error::Session(SessionError::Unauthorized)); set.connect(link.id(), None); refused(&set, SessionError::Unauthorized); diff --git a/rs/moq-net/src/ietf/peer.rs b/rs/moq-net/src/ietf/peer.rs index 7f4d60fd228..ce2ee2b55df 100644 --- a/rs/moq-net/src/ietf/peer.rs +++ b/rs/moq-net/src/ietf/peer.rs @@ -54,7 +54,7 @@ impl PeerSetup { /// Tell `link` once the peer's SETUP arrives that it accepted the connection credential. pub fn accepting(mut self, link: crate::auth::Link) -> Self { - link.await_setup(); + link.await_acceptance(); self.accept = Some(link); self } diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 1ce14ebc946..392c229d73d 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -328,9 +328,15 @@ pub fn start(config: Config) -> Result { .offer_ietf(our_setup.ietf) .with_implementation(peer_implementation.clone()); // Without AUTH, the server checks a client's credential before its SETUP, which the - // client reads only after going live. + // client reads only after going live. With AUTH, the AUTH_OK to the connection's own + // credential accepts it. Lite-03/04 have neither, and lite-01/02 answered in the + // negotiation SETUP already read. if client && version.has_setup_stream() && !version.has_auth() { peer_setup_slot = peer_setup_slot.accepting(auth.clone()); + } else if client && version.has_auth() { + auth.await_acceptance(); + } else if client && matches!(version, Version::Lite03 | Version::Lite04) { + auth.silent(); } // A version without a Setup Stream advertises nothing, so there is nothing to wait for. if !version.has_setup_stream() { diff --git a/rs/moq-net/src/lite/setup.rs b/rs/moq-net/src/lite/setup.rs index 4985e2a47d6..5ae9e25a319 100644 --- a/rs/moq-net/src/lite/setup.rs +++ b/rs/moq-net/src/lite/setup.rs @@ -557,7 +557,7 @@ impl PeerSetup { /// Tell `link` once the peer's SETUP arrives that it accepted the connection credential. pub fn accepting(self, link: crate::auth::Link) -> Self { - link.await_setup(); + link.await_acceptance(); self.0.lock().accept = Some(link); self } diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index f1e3cde9e8d..f30b492bf72 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -6,10 +6,15 @@ //! [`Reconnect::failed`], and otherwise polls it with the current time. The machine //! watches each live session for its close and GOAWAY itself, keeps a drained //! predecessor serving through its handover, and drops neither on the caller's behalf -//! until it reports [`Action::Done`]. +//! until it reports [`Action::Done`]. A connected session serves only once the machine +//! reports it [`Action::Established`], so a caller never shows a refusal as connected. //! -//! The policy fails fast until a session is established, which takes staying up past -//! [`Backoff::healthy`] or ending in a GOAWAY: a failed walk, or a session closed before then, ends the policy +//! The policy fails fast until a session is established: the peer accepts the connection +//! credential (its SETUP, or on moq-lite-07-wip the AUTH_OK to the connection's own +//! credential) or drains the session with a GOAWAY. On moq-lite 03/04, which answer the +//! credential with nothing, staying up past [`Backoff::healthy`] establishes it too; a +//! version with an answer waits for it however long the session stays up. A failed walk, +//! or a session closed before then, ends the policy //! with [`Action::Done`] carrying that error rather than retrying, so a setup mistake //! surfaces at once instead of looping. That covers a peer that refuses the session right //! after connecting, as a relay rejecting a token on moq-lite 03 or later does. A client @@ -56,9 +61,9 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// When to redial after a failure, and when to give up, once a session has been established. /// -/// A session is established once it stays up past `healthy` or the peer drains it with a -/// GOAWAY. Before then nothing retries: a failed walk, or a session closed sooner, ends the -/// policy. +/// A session is established once the peer accepts the connection credential or drains the +/// session with a GOAWAY, or, on a version with no answer to the credential (moq-lite +/// 03/04), once it stays up past `healthy`. Before then nothing retries: a failed walk, or a session closed sooner, ends the policy. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter, @@ -86,8 +91,9 @@ pub struct Backoff { pub max: Option, /// How long a session must stay up to count as healthy. Must be nonzero. Defaults to 1 s. /// - /// The first healthy session, or one the peer drains with a GOAWAY, establishes the - /// policy, so failures retry from then on. + /// Where the peer never answers the connection credential, as on moq-lite 03/04, the first + /// healthy session establishes the policy, so failures retry from then on. Elsewhere only + /// the peer's answer or a GOAWAY establishes it, however long a session stays up. /// A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. @@ -336,6 +342,12 @@ pub enum Action { /// [`Reconnect::failed`], polling meanwhile. Build each session's client with /// [`Attempt::client`]. Dial(Attempt), + /// The live session is established: its peer accepted the connection credential or + /// drained it with a GOAWAY, or, on a version with no answer (moq-lite 03/04), it stayed + /// up past [`Backoff::healthy`]. Until then a + /// connected session may still be a refusal, so report it as serving only from here. + /// Carries the session, which a GOAWAY may already have moved to the predecessor. + Established(Box), /// Drop the dial or probe in flight: it ran out its deadline, which already counted as /// its failure, or the predecessor's peer redirected while it headed for that peer. Abandon, @@ -377,8 +389,8 @@ pub struct Reconnect { phase: Phase, /// The failures of the walk under way. walk: Walk, - /// Whether any session stayed up past the healthy bar or ended in a GOAWAY. Until one has, - /// a failed walk or a closed session ends the policy. + /// Whether any session was established: [`Action::Established`] was reported for it. Until + /// then, a failed walk or a closed session ends the policy. established: bool, /// An old session kept serving after a GOAWAY so its in-flight groups finish. predecessor: Option, @@ -413,6 +425,8 @@ struct Live { session: Session, since: Instant, index: usize, + /// Whether [`Action::Established`] was reported for this session. + established: bool, /// Dropped once a restart is waited out: it is a latch, so it would keep firing. goaway: Option, /// Set while the session runs on a fallback transport. @@ -556,9 +570,11 @@ impl + From> Reco } let live = matches!(self.phase, Phase::Live(_)); + // A replacement serves only once established: until then it may be a refusal. + let serving = matches!(&self.phase, Phase::Live(live) if live.established); if let Some(predecessor) = self.predecessor.as_mut() { // The replacement serves, so a predecessor carrying nothing closes now. - if live { + if serving { predecessor.poll_idle(waiter); } if predecessor.poll(now, waiter) { @@ -566,7 +582,7 @@ impl + From> Reco self.queue.push_back(Action::Retired); // Nothing is serving once it retires between sessions, so a caller must // stop handing out a session that is already closed. - if !live { + if !serving { self.queue.push_back(Action::Offline(None)); } continue; @@ -594,6 +610,20 @@ impl + From> Reco deadline => return Action::Wait(earliest(deadline.as_ref().map(|(at, _)| *at), handover)), }, Phase::Live(live) => { + // Checked before the close, which may follow the peer's acceptance in one wake: + // a session the peer accepted and then dropped is an outage, not a refusal. + // Uptime stands in for an answer only on a version that has none: a peer that + // can answer and has not may still refuse. + let healthy = live + .session + .is_silent() + .then(|| live.since.saturating_add(self.config.backoff.healthy)); + let uptime = healthy.is_some_and(|healthy| now >= healthy); + if !live.established && (uptime || live.session.poll_accepted(waiter).is_ready()) { + live.established = true; + self.established = true; + return Action::Established(Box::new(live.session.clone())); + } let mut ended = None; // Checked before the close: a GOAWAY means the session is still up, and // migrating from it is not the same as reconnecting after it died. @@ -603,6 +633,10 @@ impl + From> Reco // client started while its relay drains rides out the restart, whether it // redials now or serves until the peer closes the session. self.established = true; + if !std::mem::replace(&mut live.established, true) { + self.queue + .push_back(Action::Established(Box::new(live.session.clone()))); + } // Under `Handoff`, a session drained before it was healthy is a redial that // reached the draining process rather than a replacement, so another would // only reach it again. @@ -627,7 +661,18 @@ impl + From> Reco self.end(now, ended); continue; } - match live.probe.as_mut().map(|probe| probe.poll(now)) { + if !self.queue.is_empty() { + continue; + } + // An unestablished silent session is established by uptime at the latest. + let handover = match live.established { + true => handover, + false => earliest(healthy, handover), + }; + // Only an established session is left for the preferred transport: one that may + // still be a refusal is no session to hand over. + let probe = live.probe.as_mut().filter(|_| live.established); + match probe.map(|probe| probe.poll(now)) { None => return Action::Wait(handover), Some(ProbePoll::Wait(at)) => return Action::Wait(earliest(at, handover)), Some(ProbePoll::Abandon) => return Action::Abandon, @@ -671,15 +716,13 @@ impl + From> Reco if let Some(tokens) = &self.tokens { tokens.connected(&session); } - // A predecessor that retired mid-dial queued an offline this session now contradicts. - // Its `Retired` stays, so the caller still drops the predecessor. - self.queue.retain(|action| !matches!(action, Action::Offline(_))); self.walk = Walk::default(); self.phase = Phase::Live(Box::new(Live { goaway: Some(session.draining()), session, since: now, index: dialing.index, + established: false, probe: None, })); } @@ -750,7 +793,8 @@ impl + From> Reco tracing::debug!("the fallback session was already draining"); } // The fallback session served all along, so a long one still earns the healthy reset. - self.established |= self.retry.settle( + // Only an established session is probed, so the policy already is. + self.retry.settle( now, now.saturating_duration_since(old.since), &Pacing::new(&self.config.backoff), @@ -905,9 +949,9 @@ impl + From> Reco fn walk_failed(&mut self, now: Instant, err: E, verdict: Verdict) { self.walk = Walk::default(); - // Otherwise a walk fails fast until a session was established: a peer that never held - // one past the healthy bar is more likely misconfigured (an unknown host, an untrusted - // certificate, no version in common) than restarting, and only the caller can fix that. + // Otherwise a walk fails fast until a session was established: a peer that never + // accepted one is more likely misconfigured (an unknown host, an untrusted certificate, + // no version in common) than restarting, and only the caller can fix that. // After that, every failure retries, whatever its kind. if verdict == Verdict::Unusable || self.config.once || !self.established { return self.done(Err(err)); @@ -934,12 +978,8 @@ impl + From> Reco let Poll::Ready(msg) = redirect.goaway.poll(waiter) else { return None; }; + // No need to establish the policy here: only an established session is probed. let redirect = predecessor.redirect.take()?; - // The peer accepted the fallback session and asked us to move, so its GOAWAY - // establishes the policy as one off the live session does. - if msg.is_ok() { - self.established = true; - } let target = match msg.map(|msg| { self.config .goaway @@ -982,9 +1022,6 @@ impl + From> Reco let tenure = now.saturating_duration_since(live.since); let url = self.targets[live.index].url().clone(); let pacing = Pacing::new(&self.config.backoff); - // Judged on tenure alone, so a long session that ends in a redirect loop still counts. - self.established |= tenure >= pacing.healthy; - // The connected target owns the policy, including when connecting once. A refused // redirect is terminal: the peer is leaving and named somewhere we won't go, so // redialing the old address or a fallback would ignore it. @@ -1098,11 +1135,11 @@ impl + From> Reco if self.config.once { return self.done(Err(err)); } - // Until a session has stayed up past the healthy bar, a close ends the policy as a - // failed walk does. It is how a peer refuses a session it already accepted, such as a - // rejected credential on moq-lite 03 or later, which has no SETUP reply to refuse in; - // redialing would offer the same credential forever. No error kind is read, and as - // for a refused dial, tokens replaced since change nothing. + // Until a session was established, a close ends the policy as a failed walk does. It + // is how a peer refuses a session that is already live, such as a rejected credential + // on moq-lite 03 or later, closed before the peer's SETUP or AUTH_OK; redialing would + // offer the same credential forever. No error kind is read, and as for a refused dial, + // tokens replaced since change nothing. if !self.established { return self.done(Err(err)); } diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 987b39e81d1..3c1e34cdaed 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -28,6 +28,8 @@ enum Peer { Goaway(String), /// Accept and keep serving. Serve, + /// Accept and keep serving, but never answer the connection credential. + Withhold, /// Fail the dial, judged as this. Fail(Verdict), /// Never answer. @@ -60,29 +62,53 @@ fn version() -> crate::Version { /// Run an in-memory MoQ handshake, returning the client and server sessions. async fn handshake() -> (Session, Session) { + let (client, server, _) = handshake_with(version(), false).await; + (client, server) +} + +/// Run an in-memory MoQ handshake over `version`, returning the client and server sessions. +/// With `withhold`, the server answers the client's tokens itself and never does, returning +/// the queue that keeps them unanswered while it is held. +async fn handshake_with(version: crate::Version, withhold: bool) -> (Session, Session, Option) { let (client, server) = crate::transport::memory::pair(crate::transport::memory::Config { - protocol: Some(version().alpn().to_string()), + protocol: Some(version.alpn().to_string()), ..Default::default() }); - let dial = crate::Client::new().with_versions(version().into()); - let serve = crate::Server::new().with_versions(version().into()); - futures::join!( + let dial = crate::Client::new().with_versions(version.into()); + let serve = crate::Server::new().with_versions(version.into()); + let (client, (server, requests)) = futures::join!( async { let (session, driver) = dial.connect(kio::clock::now(), client).await.unwrap(); drop(kio::clock::sim::spawn(crate::time::run(driver))); session }, async { - let (session, driver) = serve.accept(kio::clock::now(), server).await.unwrap(); + let mut handshake = serve.handshake(kio::clock::now(), server).await.unwrap(); + let requests = withhold.then(|| handshake.requests()); + let (session, driver) = handshake.ok().await.unwrap(); drop(kio::clock::sim::spawn(crate::time::run(driver))); - session + (session, requests) }, - ) + ); + (client, server, requests) } -/// The next action that is not a wait, sleeping on the simulated clock meanwhile. +/// The next action that is neither a wait nor [`Action::Established`], sleeping on the +/// simulated clock meanwhile. async fn next + From>( machine: &mut Reconnect, +) -> Action { + loop { + match next_any(machine).await { + Action::Established(_) => continue, + action => return action, + } + } +} + +/// The next action that is not a wait, sleeping on the simulated clock meanwhile. +async fn next_any + From>( + machine: &mut Reconnect, ) -> Action { let mut sleep: Option = None; kio::wait(|waiter| { @@ -116,6 +142,8 @@ struct Run { events: Vec<&'static str>, /// The server sides still serving. servers: Vec, + /// The auth queues of the servers withholding their answer. + withheld: Vec, } /// Drive `machine` against `peers` until it ends, or until `stop` says so. @@ -126,9 +154,13 @@ async fn drive( mut stop: impl FnMut(&Run) -> bool, ) -> Option> { while !stop(run) { - let attempt = match next(machine).await { + let attempt = match next_any(machine).await { Action::Dial(attempt) => attempt, Action::Done(res) => return Some(res), + Action::Established(_) => { + run.events.push("established"); + continue; + } Action::Abandon => { run.events.push("abandon"); continue; @@ -150,7 +182,7 @@ async fn drive( continue; } Action::Probe(_) => unreachable!("no test driven here reports a fallback"), - Action::Wait(_) => unreachable!("next never returns a wait"), + Action::Wait(_) => unreachable!("next_any never returns a wait"), }; let url = attempt.target().url().clone(); run.dials.push(url.clone()); @@ -158,8 +190,9 @@ async fn drive( match peer { Peer::Hang => {} Peer::Fail(verdict) => machine.failed(kio::clock::now(), TestError::Dial, *verdict), - Peer::Close | Peer::Goaway(_) | Peer::Serve => { - let (client, server) = handshake().await; + Peer::Close | Peer::Goaway(_) | Peer::Serve | Peer::Withhold => { + let (client, server, withheld) = handshake_with(version(), matches!(peer, Peer::Withhold)).await; + run.withheld.extend(withheld); run.accepts.push((url, kio::clock::now())); match peer { Peer::Close => drop(server), @@ -340,11 +373,12 @@ async fn the_first_walk_fails_fast() { assert_eq!(kio::clock::now(), started, "gave up without waiting"); } -/// A first session closed before it was healthy ends the policy with its close reason, -/// like a failed first walk: it is how a peer refuses a session it already accepted, such -/// as a rejected credential on moq-lite 03 or later. Tokens replaced since change nothing. +/// A first session closed before the peer accepted its credential ends the policy with its +/// close reason, like a failed first walk: it is how a peer refuses a session that is +/// already live, such as a rejected credential on moq-lite 03 or later. Tokens replaced +/// since change nothing. #[kio::test] -async fn a_first_session_closed_before_healthy_fails_fast() { +async fn a_first_session_closed_before_acceptance_fails_fast() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); let started = kio::clock::now(); @@ -352,7 +386,7 @@ async fn a_first_session_closed_before_healthy_fails_fast() { .unwrap() .with_tokens(tokens.clone()); let mut run = Run::default(); - drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { !run.accepts.is_empty() }) .await; @@ -369,10 +403,11 @@ async fn a_first_session_closed_before_healthy_fails_fast() { assert!(run.events.is_empty(), "retried: {:?}", run.events); } -/// A session that stayed up past the healthy bar establishes the policy: its close, and -/// the failed walk after it, back off and retry like any later failure. +/// The peer accepting the connection credential establishes the policy however soon the +/// session then drops: its close, and the failed walk after it, back off and retry like any +/// later failure. #[kio::test] -async fn a_healthy_session_establishes_the_policy() { +async fn an_accepted_session_establishes_the_policy() { let peer = url("tcp://peer:1/"); let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); @@ -381,14 +416,101 @@ async fn a_healthy_session_establishes_the_policy() { }) .await; - kio::clock::sleep(Config::default().backoff.healthy).await; + let accepted = kio::clock::now(); + kio::wait(|waiter| machine.session().expect("live").poll_accepted(waiter)).await; + assert!( + kio::clock::now() - accepted < Config::default().backoff.healthy / 10, + "the AUTH_OK came late" + ); run.servers.remove(0).abort(Error::Unauthorized); let ended = drive(&mut machine, &[(peer, Peer::Fail(Verdict::Retry))], &mut run, |run| { run.dials.len() == 3 }) .await; assert!(ended.is_none(), "an established policy gave up"); - assert_eq!(run.events, ["offline", "retrying"]); + assert_eq!(run.events, ["established", "offline", "retrying"]); +} + +/// A version that never answers the connection credential (moq-lite-04) keeps the uptime +/// bar: a session closed before it is a refusal, and one that outlives it establishes. +#[kio::test] +async fn a_silent_version_is_established_by_uptime() { + let lite04: crate::Version = "moq-lite-04".parse().unwrap(); + for healthy in [false, true] { + let mut machine = + Reconnect::::new(config(INITIAL, None), vec![url("tcp://peer:1/")], kio::clock::now()) + .unwrap(); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + let (client, server, _) = handshake_with(lite04, false).await; + machine.connected(kio::clock::now(), client); + + let healthy_bar = Config::default().backoff.healthy; + kio::clock::sleep(if healthy { healthy_bar } else { healthy_bar / 2 }).await; + assert!( + machine + .session() + .expect("live") + .poll_accepted(&kio::Waiter::noop()) + .is_pending(), + "moq-lite-04 has no answer to accept with" + ); + if healthy { + assert!(matches!(next_any(&mut machine).await, Action::Established(_))); + } + server.abort(Error::Unauthorized); + match next_any(&mut machine).await { + Action::Offline(_) => assert!(healthy, "a refusal before the bar was retried"), + Action::Done(res) => assert!(!healthy && res.is_err(), "a healthy session gave up: {res:?}"), + _ => panic!("unexpected action"), + } + } +} + +/// A GOAWAY before any answer (moq-lite-04 never sends one) establishes the session, and the +/// action carries it, since the same pass already moved it out of the live slot. +#[kio::test] +async fn an_early_goaway_establishes_the_session_it_drains() { + let lite04: crate::Version = "moq-lite-04".parse().unwrap(); + let mut machine = + Reconnect::::new(config(INITIAL, None), vec![url("tcp://peer:1/")], kio::clock::now()).unwrap(); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + let (client, server, _) = handshake_with(lite04, false).await; + machine.connected(kio::clock::now(), client.clone()); + server.goaway(goaway::Goaway::new()).unwrap(); + + let Action::Established(established) = next_any(&mut machine).await else { + panic!("the drained session was never established"); + }; + assert!(*established == client, "established another session"); + assert!(matches!(next_any(&mut machine).await, Action::Migrating)); + assert!(machine.session().is_none(), "the drained session left the live slot"); + drop(server); +} + +/// On a version that answers the connection credential (moq-lite-07-wip), uptime is no +/// answer: a session up well past the healthy bar without its AUTH_OK is never established, +/// and its close ends the policy as a refusal. +#[kio::test] +async fn an_unanswered_session_is_not_established_by_uptime() { + let peer = url("tcp://peer:1/"); + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + kio::clock::sleep(Config::default().backoff.healthy * 10).await; + let action = machine.poll(kio::clock::now(), &kio::Waiter::noop()); + assert!(matches!(action, Action::Wait(None)), "{action:?}"); + run.servers.remove(0).abort(Error::Unauthorized); + let ended = drive(&mut machine, &[(peer, Peer::Serve)], &mut run, |_| false).await; + assert!( + matches!(ended, Some(Err(TestError::Net(_)))), + "the unanswered session was retried: {ended:?}" + ); + assert_eq!(run.dials.len(), 1, "redialed after the refusal"); + assert!(run.events.is_empty(), "established or retried: {:?}", run.events); } /// A GOAWAY establishes the policy however soon it comes, since a refusal never drains a @@ -402,7 +524,8 @@ async fn a_goaway_before_healthy_establishes_the_policy() { config.restart = restart; let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); - drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + // Withheld, so only the GOAWAY can establish the policy before the relay exits. + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { !run.accepts.is_empty() }) .await; @@ -411,7 +534,7 @@ async fn a_goaway_before_healthy_establishes_the_policy() { // `Redial` follows it to a replacement on the draining relay; `Handoff` serves on, // since the session is not yet healthy. if restart == Restart::Redial { - drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + drive(&mut machine, &[(peer.clone(), Peer::Withhold)], &mut run, |run| { run.accepts.len() == 2 }) .await; @@ -759,7 +882,7 @@ async fn a_healthy_goaway_hands_over_to_the_redirect() { assert_eq!(run.accepts[1], (b.clone(), migrated), "migrated without a backoff"); assert_eq!( run.events, - ["migrating", "retired"], + ["established", "migrating", "established", "retired"], "a replacement serves, so nothing went offline" ); assert!(machine.predecessor().is_none()); @@ -779,6 +902,53 @@ async fn a_healthy_goaway_hands_over_to_the_redirect() { assert_eq!(machine.targets(), [a]); } +/// A replacement the peer has not yet accepted does not serve: a predecessor that retires +/// meanwhile leaves the connection offline, and the replacement reports +/// [`Action::Established`] only once the peer answers, however long that takes. +#[kio::test] +async fn an_unanswered_replacement_does_not_serve() { + const HANDOVER: Duration = Duration::from_millis(500); + let peer = url("tcp://peer:1/"); + let mut config = Config::default(); + config.goaway.handover = HANDOVER; + let healthy = config.backoff.healthy; + let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + kio::clock::sleep(Duration::from_secs(2)).await; + run.servers[0].goaway(goaway::Goaway::new()).unwrap(); + drive(&mut machine, &[(peer, Peer::Withhold)], &mut run, |run| { + run.accepts.len() == 2 + }) + .await; + let connected = run.accepts[1].1; + assert_eq!(run.events, ["established", "migrating"]); + assert!(matches!(next_any(&mut machine).await, Action::Retired)); + assert!(matches!(next_any(&mut machine).await, Action::Offline(None))); + assert_eq!( + kio::clock::now() - connected, + HANDOVER, + "the idle predecessor closed early" + ); + kio::clock::sleep(healthy * 10).await; + let action = machine.poll(kio::clock::now(), &kio::Waiter::noop()); + assert!( + matches!(action, Action::Wait(None)), + "established by uptime: {action:?}" + ); + + let mut requests = run.withheld.remove(0); + let request = kio::wait(|waiter| requests.poll_next(waiter)) + .await + .expect("the credential"); + let _issued = request.accept(auth::Grant::all()); + assert!(matches!(next_any(&mut machine).await, Action::Established(_))); +} + /// Our handover window caps the peer's GOAWAY deadline, and a predecessor retiring while /// no replacement serves leaves the connection offline. #[kio::test] @@ -803,7 +973,7 @@ async fn the_handover_takes_the_earlier_deadline() { run.events.contains(&"offline") }) .await; - assert_eq!(run.events, ["migrating", "retired", "offline"]); + assert_eq!(run.events, ["established", "migrating", "retired", "offline"]); assert_eq!( kio::clock::now() - drained, HANDOVER, @@ -819,10 +989,10 @@ async fn the_handover_takes_the_earlier_deadline() { ); } -/// A replacement that connects after the predecessor retired mid-dial serves, so the -/// offline the retirement queued no longer holds and is never reported. +/// A replacement that connects after the predecessor retired mid-dial does not serve until +/// it is established, so the offline the retirement queued is still reported first. #[kio::test] -async fn a_connect_cancels_the_offline_a_retirement_queued() { +async fn a_retirement_mid_dial_is_offline_until_the_replacement_is_established() { const HANDOVER: Duration = Duration::from_millis(500); let peer = url("tcp://peer:1/"); let mut config = Config::default(); @@ -843,13 +1013,10 @@ async fn a_connect_cancels_the_offline_a_retirement_queued() { let (client, _server) = handshake().await; machine.connected(kio::clock::now(), client); - assert!( - matches!( - machine.poll(kio::clock::now(), &kio::Waiter::noop()), - Action::Wait(None) - ), - "a stale offline followed the connect" - ); + let action = next_any(&mut machine).await; + assert!(matches!(action, Action::Offline(None)), "{action:?}"); + let action = next_any(&mut machine).await; + assert!(matches!(action, Action::Established(_)), "{action:?}"); assert!(machine.session().is_some()); } @@ -910,7 +1077,7 @@ async fn a_handoff_restart_moves_to_a_replacement() { }) .await; assert_eq!(run.accepts[1].1, goaway, "the redial waited"); - assert_eq!(run.events, ["migrating"]); + assert_eq!(run.events, ["established", "migrating"]); assert_eq!(settle(&mut machine).await, ["retired"]); assert!(machine.session().is_some(), "the replacement serves"); assert_eq!(run.dials.len(), 2); @@ -940,7 +1107,7 @@ async fn a_handoff_restart_redials_once_until_the_session_closes() { |run| run.accepts.len() == 2, ) .await; - assert_eq!(run.events, ["migrating"]); + assert_eq!(run.events, ["established", "migrating"]); assert_eq!(settle(&mut machine).await, ["retired"], "the policy left the session"); assert_eq!(run.dials.len(), 2, "redialed the draining process again"); @@ -1095,32 +1262,6 @@ async fn a_redirect_after_a_probe_got_through_is_followed() { assert!(machine.predecessor().is_some(), "the redirected session still drains"); } -/// The draining fallback's peer accepted that session and asked us to move, which no refusal -/// does, so its redirect establishes the policy however young the session was: the redirected -/// dial failing retries rather than ending it. -#[kio::test] -async fn a_fallback_peer_redirect_establishes_the_policy() { - let mut config = Config::default(); - // Longer than the probe takes, so the fallback session is never healthy. - config.backoff.healthy = Duration::from_secs(3600); - let (mut machine, fallback_server) = on_fallback(config).await; - probe(&mut machine).await; - assert!(machine.probed(kio::clock::now(), Ok(()))); - assert!(matches!(next(&mut machine).await, Action::Migrating)); - - fallback_server - .goaway(goaway::Goaway::redirect(url("https://peer:2/").to_string())) - .expect("the fallback peer sends a GOAWAY"); - assert!(matches!(next(&mut machine).await, Action::Abandon)); - assert!(matches!(next(&mut machine).await, Action::Dial(_))); - machine.failed(kio::clock::now(), TestError::Dial, Verdict::Retry); - let action = next(&mut machine).await; - assert!( - !matches!(action, Action::Done(_)), - "the redirected dial's failure ended the policy" - ); -} - /// A redirect from the draining fallback's peer during the backoff after a failed /// handshake keeps the wait, and the redial goes where the peer named. #[kio::test] @@ -1265,6 +1406,8 @@ async fn a_dropped_replacement_goes_offline_over_a_closing_predecessor() { assert!(matches!(next(&mut machine).await, Action::Dial(_))); let (replacement, replacement_server) = handshake().await; machine.connected(kio::clock::now(), replacement.clone()); + // Only an established replacement serves, which lets the idle predecessor close. + assert!(matches!(next_any(&mut machine).await, Action::Established(_))); // The replacement is already closed when the next poll starts the idle predecessor's // graceful close, so its close is seen over a predecessor that is closing, not retired. drop(replacement_server); @@ -1349,10 +1492,12 @@ async fn connecting_once_never_probes() { }; let (mut machine, _fallback) = on_fallback(config).await; kio::clock::sleep(PROBE_MAX * 2).await; + let waiter = kio::Waiter::noop(); assert!(matches!( - machine.poll(kio::clock::now(), &kio::Waiter::noop()), - Action::Wait(None) + machine.poll(kio::clock::now(), &waiter), + Action::Established(_) )); + assert!(matches!(machine.poll(kio::clock::now(), &waiter), Action::Wait(None))); } /// The policy must not log or report a credential riding in a dial URL. diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 800bef31e6e..d7cdf506f08 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -226,6 +226,14 @@ impl PartialEq for Session { impl Eq for Session {} +impl std::fmt::Debug for Session { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Session") + .field("version", &self.version()) + .finish_non_exhaustive() + } +} + impl Session { /// Returns the negotiated protocol version. pub fn version(&self) -> Version { @@ -472,6 +480,19 @@ impl Session { self.credential.as_ref() } + /// Ready once the peer accepted the connection credential: with its SETUP, or on + /// moq-lite-07-wip the AUTH_OK to the connection's own credential. Ready at once when the + /// handshake already carried that answer, and never on moq-lite 03/04, which have none. + pub(crate) fn poll_accepted(&self, waiter: &kio::Waiter) -> Poll<()> { + self.link.poll_accepted(waiter) + } + + /// Whether the peer never answers the connection credential, as on moq-lite 03/04, so + /// only uptime suggests it took the credential. + pub(crate) fn is_silent(&self) -> bool { + self.link.is_silent() + } + /// This session's side of its [`auth`](Self::auth) handle. pub(crate) fn link(&self) -> &auth::Link { &self.link diff --git a/rs/moq-net/verify/reconnect/Handover.cfg b/rs/moq-net/verify/reconnect/Handover.cfg index ea4c0c89aa3..0bdc4e68c55 100644 --- a/rs/moq-net/verify/reconnect/Handover.cfg +++ b/rs/moq-net/verify/reconnect/Handover.cfg @@ -12,6 +12,7 @@ VIEW View INVARIANTS NoStaleSession StatusTruthful + ConnectedMeansEstablished OfflineMeansNothingServes NoOrphan PresenceBalanced diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 9f5d29eac7a..ab8e351b476 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -20,8 +20,9 @@ (* one it refuses, no retry window, no tokens, not connecting once, and no *) (* redirect-loop scoring. A session's age stands in for its tenure: 1 is *) (* past `Backoff::healthy`. Until a session has been established, which *) -(* takes one past that bar or a GOAWAY, a failed dial or a closed session *) -(* ends the policy (`Reconnect::established`). *) +(* takes the peer accepting its credential or a GOAWAY, or on a silent *) +(* version (moq-lite 03/04, no answer) one past that bar, a failed dial or *) +(* a closed session ends the policy (`Reconnect::established`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -59,7 +60,7 @@ VARIABLES started, \* `Presence::sessions_started` ended, \* `Presence::sessions_ended` misdials, \* 1 once a dial went elsewhere than an unheeded redirect named - established \* `Reconnect::established`: a session stayed up past the healthy bar or got a GOAWAY + established \* `Reconnect::established`: a session was reported established policyVars == <> @@ -71,11 +72,16 @@ vars == <> \* "abort" for `Session::abort`, "drain" for a `start_close` that finished), \* whether a `start_close` began, the GOAWAY it got, its age, whether it \* carries no subscription or fetch, whether the policy acted on its GOAWAY, -\* whether we sent our own, and whether the policy watches its GOAWAY as the -\* predecessor (`Predecessor::redirect`, set when a probe moved us off it). +\* whether we sent our own, whether the policy watches its GOAWAY as the +\* predecessor (`Predecessor::redirect`, set when a probe moved us off it), +\* whether the peer accepted its credential (`Session::poll_accepted`), which a +\* close leaves as it was, whether its version never answers the credential +\* (`Session::is_silent`), and whether the policy reported it established +\* (`Live::established`), the only point a caller serves it from. Fresh(e) == [ep |-> e, open |-> TRUE, by |-> "", closing |-> FALSE, goaway |-> NoGoaway, age |-> 0, idle |-> FALSE, heeded |-> FALSE, - sent |-> FALSE, watched |-> FALSE] + sent |-> FALSE, watched |-> FALSE, accepted |-> FALSE, silent |-> FALSE, + served |-> FALSE] Unused == [Fresh(Start) EXCEPT !.open = FALSE] Timers == {"dial", "backoff", "probe", "handover"} @@ -113,6 +119,16 @@ Abort(sx, s) == IF s # None /\ sx[s].open (* predecessor once the pass ends, which `Reconnect::predecessor` reads. *) (***************************************************************************) +\* `Action::Established`: `Shared::connected` with the live session `s`. +OnEstablished(s) == + /\ ended' = IF lc = "live" THEN ended + 1 ELSE ended + /\ spred' = IF lc \in {"live", "offline"} THEN spred ELSE None + /\ started' = started + 1 + /\ lc' = "live" + /\ lcSess' = s + /\ migrating' = FALSE + /\ UNCHANGED probing + \* `Shared::predecessor`. Mirror(p) == spred' = IF lc \in {"live", "offline"} THEN p ELSE spred @@ -155,8 +171,11 @@ OnDone == /\ probing' = FALSE /\ UNCHANGED started +\* `p` is the predecessor, or for `Established` the session the action carries, +\* which a GOAWAY in the same pass already moved out of the live slot. Apply(a, p) == - CASE a = "Migrating" -> OnMigrating(p) + CASE a = "Established" -> OnEstablished(p) + [] a = "Migrating" -> OnMigrating(p) [] a = "Retired" -> OnRetired [] a = "Offline" -> OnOffline [] a = "Done" -> OnDone @@ -186,6 +205,15 @@ Pop == dial, dialTo, misdials>> /\ UNCHANGED lifeVars +\* A replacement serves only once established: until then it may be a refusal. +Serving == phase = "live" /\ sess[live].served + +\* The live session is due its `Action::Established`, which `poll` returns +\* before reading its GOAWAY or close. +\* Uptime stands in for the peer's answer only on a version that has none. +Healthy(s) == sess[s].silent /\ sess[s].age >= 1 +EstablishDue == phase = "live" /\ ~sess[live].served /\ (sess[live].accepted \/ Healthy(live)) + \* The predecessor closed or overstayed its handover: `Predecessor::poll`. PredDue == pred # None /\ (~sess[pred].open \/ timer["handover"]) @@ -194,7 +222,7 @@ Retire == /\ PredDue /\ sess' = Abort(sess, pred) /\ pred' = None - /\ Emit(IF phase = "live" THEN <<"Retired">> ELSE <<"Retired", "Offline">>, None) + /\ Emit(IF Serving THEN <<"Retired">> ELSE <<"Retired", "Offline">>, None) /\ UNCHANGED <> /\ UNCHANGED lifeVars @@ -205,7 +233,7 @@ Retire == IdleClose == /\ queue = <<>> /\ ~PredDue - /\ phase = "live" + /\ Serving /\ pred # None /\ sess[pred].idle /\ ~sess[pred].closing @@ -252,8 +280,8 @@ FollowPred == /\ UNCHANGED <> /\ UNCHANGED <> - \* Its GOAWAY establishes the policy, as one off the live session does. - /\ established' = TRUE + \* Only an established session is probed, so the policy already is. + /\ UNCHANGED established \* `Phase::Next`: ask for a dial. Dial == @@ -309,35 +337,40 @@ BackoffEnds == \* `Reconnect::end` for a GOAWAY naming `uri` off the live session `s`. The \* GOAWAY establishes the policy however soon it came, as `Reconnect::poll` does \* on its receipt: the peer accepted the session, which a refusal never does. +\* A session not yet reported established is, first. EndGoaway(s, uri) == + LET first == IF sess[s].served THEN <<>> ELSE <<"Established">> IN IF uri = Refused THEN \* A refused redirect is terminal. /\ Done - /\ Emit(<<"Done">>, None) + /\ Emit(first \o <<"Done">>, s) /\ established' = TRUE - /\ UNCHANGED <> + /\ sess' = [sess EXCEPT ![s].served = TRUE] + /\ UNCHANGED <> ELSE /\ established' = TRUE /\ target' = IF uri = "" THEN target ELSE uri \* An older predecessor is retired outright, with no `Retired`. - /\ sess' = [Abort(sess, pred) EXCEPT ![s].heeded = TRUE] + /\ sess' = [Abort(sess, pred) EXCEPT ![s].heeded = TRUE, ![s].served = TRUE] /\ pred' = s /\ live' = None \* A healthy session hands over at once; one that drained too soon backs off. /\ phase' = IF sess[s].age >= 1 THEN "start" ELSE "backoff" /\ timer' = [timer EXCEPT !["handover"] = FALSE, !["backoff"] = FALSE] /\ probe' = "none" - /\ Emit(<<"Migrating">>, s) + /\ Emit(first \o <<"Migrating">>, s) /\ UNCHANGED watch \* `Reconnect::closed`. Until a session has been established, a close ends the -\* policy as a failed dial does: it is how a peer refuses a session it already -\* accepted. After that, every close retries, whatever its error. A predecessor +\* policy as a failed dial does: it is how a peer refuses a live session before +\* accepting its credential. `poll` reads the acceptance and the healthy bar +\* before the close, so a session either establishes would already have. After +\* that, every close retries, whatever its error. A predecessor \* that is not closing gracefully serves again, so the policy migrates back \* onto it; the `Retrying` that follows changes no status, so it is left out. EndClosed == - /\ established' = (established \/ sess[live].age >= 1) + /\ UNCHANGED established /\ probe' = "none" - /\ IF established' + /\ IF established THEN /\ live' = None /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] @@ -358,17 +391,34 @@ WaitOut(s) == /\ Restart = "handoff" /\ sess[s].age = 0 +\* `Action::Established`, from the peer's acceptance, or the healthy bar on a +\* silent version. +Establish == + /\ queue = <<>> + /\ ~PredDue + /\ EstablishDue + /\ sess' = [sess EXCEPT ![live].served = TRUE] + /\ established' = TRUE + /\ Emit(<<"Established">>, live) + /\ UNCHANGED <> + LiveEnds == /\ queue = <<>> /\ ~PredDue /\ phase = "live" + /\ ~EstablishDue /\ \/ /\ Goaway(live) /\ WaitOut(live) \* Drop the latch and serve until the peer closes. The GOAWAY establishes \* the policy here too, so that close retries. /\ watch' = FALSE - /\ UNCHANGED <> - /\ CallerSame + /\ UNCHANGED <> + /\ IF sess[live].served + THEN /\ UNCHANGED <> + /\ CallerSame + ELSE /\ sess' = [sess EXCEPT ![live].served = TRUE] + /\ Emit(<<"Established">>, live) /\ established' = TRUE \/ /\ Goaway(live) /\ ~WaitOut(live) @@ -378,11 +428,12 @@ LiveEnds == /\ EndClosed /\ UNCHANGED <> -\* `Reconnect::probe`: ask for a probe of the preferred transport. +\* `Reconnect::probe`: ask for a probe of the preferred transport, only off an +\* established session. ProbeAsk == /\ queue = <<>> /\ ~PredDue - /\ phase = "live" + /\ Serving /\ ~Goaway(live) /\ sess[live].open /\ probe = "due" @@ -410,35 +461,32 @@ ProbeAbandon == /\ UNCHANGED lifeVars Policy == Pop \/ Retire \/ IdleClose \/ FollowPred \/ Dial \/ Abandon \/ BackoffEnds - \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon + \/ Establish \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon (***************************************************************************) (* What `run` reports back to the policy. *) (***************************************************************************) -\* A dial connected: `Shared::connected`, then `Reconnect::connected`, and -\* `Reconnect::fallback` when it landed on the fallback transport. +\* A dial connected: `Reconnect::connected`, and `Reconnect::fallback` when it +\* landed on the fallback transport. Nothing is published until the policy +\* reports the session established. Connected == /\ dial # "none" /\ used < MaxSessions /\ LET s == used + 1 IN /\ used' = s /\ dial' = "none" - /\ sess' = [sess EXCEPT ![s] = Fresh(dialTo)] - \* `Shared::connected` - /\ ended' = IF lc = "live" THEN ended + 1 ELSE ended - /\ spred' = IF lc \in {"live", "offline"} THEN spred ELSE None - /\ started' = started + 1 - /\ lc' = "live" - /\ lcSess' = s - /\ migrating' = FALSE - \* `Reconnect::connected`, ignored unless a dial is in flight. It drops - \* the `Offline` a predecessor that retired mid-dial left queued. + \* Any version: one that answers the credential, or a silent one. + /\ \E silent \in BOOLEAN : sess' = [sess EXCEPT ![s] = [Fresh(dialTo) EXCEPT !.silent = silent]] + /\ UNCHANGED <> + \* `Reconnect::connected`, ignored unless a dial is in flight. An `Offline` + \* a predecessor that retired mid-dial left queued stands: nothing serves + \* until the session is established. /\ IF phase = "dialing" THEN /\ phase' = "live" /\ live' = s /\ watch' = TRUE - /\ queue' = SelectSeq(queue, LAMBDA a : a # "Offline") + /\ UNCHANGED queue /\ \E fallback \in BOOLEAN : probe' = IF fallback THEN "wait" ELSE "none" ELSE UNCHANGED <> @@ -499,9 +547,8 @@ ProbeGetsThrough == /\ dial' = "handshake" /\ dialTo' = sess[live].ep /\ Emit(Append(queue, "Migrating"), live) - \* The fallback served all along, so a long one establishes the policy. - /\ established' = (established \/ sess[live].age >= 1) - /\ UNCHANGED <> + \* Only an established session is probed, so the policy already is. + /\ UNCHANGED <> Caller == Connected \/ Failed \/ ProbeFails \/ ProbeGetsThrough @@ -522,11 +569,14 @@ PeerClose == /\ sess' = [sess EXCEPT ![s].open = FALSE, ![s].by = IF sess[s].closing THEN "drain" ELSE "peer"] -\* The live session passes the healthy bar, or the predecessor stops carrying -\* a subscription or fetch. Only the policy reads either, and only there. +\* The live session passes the healthy bar, its peer accepts its credential, or +\* the predecessor stops carrying a subscription or fetch. Only the policy reads +\* any of them, and only there. Ages == \/ /\ Open(live) /\ sess[live].age = 0 /\ sess' = [sess EXCEPT ![live].age = 1] + \/ /\ Open(live) /\ ~sess[live].accepted + /\ sess' = [sess EXCEPT ![live].accepted = TRUE] \/ /\ Open(pred) /\ ~sess[pred].idle /\ sess' = [sess EXCEPT ![pred].idle = TRUE] @@ -579,7 +629,8 @@ Spec == Init /\ [][Next]_vars \* field after it closes: `LiveEnds` still reads its GOAWAY and age. Seen(s) == IF sess[s].open \/ s = live THEN sess[s] - ELSE [Unused EXCEPT !.by = sess[s].by, !.closing = sess[s].closing] + ELSE [Unused EXCEPT !.by = sess[s].by, !.closing = sess[s].closing, + !.served = sess[s].served] Due(t) == CASE t = "dial" -> timer[t] /\ phase = "dialing" [] t = "backoff" -> timer[t] /\ phase = "backoff" @@ -610,11 +661,20 @@ NoStaleSession == StatusTruthful == Parked => /\ lc = "live" /\ ~migrating => live = lcSess - /\ lc = "live" /\ migrating => pred = lcSess /\ phase # "live" - /\ phase = "live" => lc = "live" /\ lcSess = live /\ ~migrating + /\ lc = "live" /\ migrating => pred = lcSess /\ ~Serving + /\ Serving => lc = "live" /\ lcSess = live /\ ~migrating /\ (lc = "closed") = (phase = "done") /\ lc # "closed" => spred = pred +\* A session reads as connected only once established: the peer accepted its +\* credential or drained it with a GOAWAY, or on a silent version it stayed up +\* past the healthy bar. A refusal never does, nor does uptime alone on a +\* version that answers. +ConnectedMeansEstablished == + /\ Parked /\ lc = "live" => sess[lcSess].served + /\ \A s \in 1..used : + sess[s].served => sess[s].accepted \/ Healthy(s) \/ sess[s].goaway # NoGoaway + \* `Action::Offline`: nothing serves anymore. A predecessor closing gracefully \* already carries nothing and takes no new work. OfflineMeansNothingServes == @@ -633,7 +693,10 @@ PresenceBalanced == \* The policy fails fast: no failed dial or closed session backs off before a \* session has been established, so a setup mistake or a refusal ends the \* policy instead of looping. A GOAWAY establishes it: no refusal sends one. -FirstWalkFailsFast == phase = "backoff" => established +\* Only a session reported established establishes the policy. +FirstWalkFailsFast == + /\ phase = "backoff" => established + /\ established => \E s \in 1..used : sess[s].served \* A dial never goes elsewhere than where a session the policy still holds \* was redirected and the policy never acted on it: the peer is leaving, and diff --git a/rs/moq-net/verify/reconnect/RestartHandoff.cfg b/rs/moq-net/verify/reconnect/RestartHandoff.cfg index 75377d9e5a3..39214d15354 100644 --- a/rs/moq-net/verify/reconnect/RestartHandoff.cfg +++ b/rs/moq-net/verify/reconnect/RestartHandoff.cfg @@ -14,6 +14,7 @@ VIEW View INVARIANTS NoStaleSession StatusTruthful + ConnectedMeansEstablished OfflineMeansNothingServes NoOrphan PresenceBalanced diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 72fb6bd95a3..41a53d70956 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -653,8 +653,8 @@ mod tests { .await .expect("the admission outlived the drain deadline"); running.await.unwrap().unwrap(); - // The client may already count the session established (SETUP precedes - // auth); either way the refusal ends it. + // The paused clock may carry the session past the healthy bar before the admission + // ends, which establishes it; either way the refusal ends it. if let Ok(connection) = connecting.await.unwrap() { assert!(connection.closed().await.is_err(), "an abandoned admission is refused"); } @@ -683,10 +683,9 @@ mod tests { let all = moq_auth::Pattern::all; let (_lease, consumer) = moq_auth::lease::Producer::new(moq_auth::Grant::new(all(), all())); admission.grant(consumer); + // The relay accepts the session and drains it at once, so the one-shot loop may already + // be over: only a GOAWAY ends it cleanly, where a refusal or a close is an error. let connection = connecting.await.unwrap().unwrap(); - let draining = connection.draining().unwrap(); - let goaway = kio::wait(|waiter| draining.poll(waiter)).await.unwrap(); - assert_eq!(goaway.uri(), ""); connection.closed().await.unwrap(); running.await.unwrap().unwrap(); } diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index 7088f0fae1d..15a477cbdc0 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -447,7 +447,14 @@ async fn drain_on_the_trigger(version: Option) { // Hold the next real connection at authentication. Its admission keeps the // drain open even after the first session leaves, without an unrelated peer. - let arriving = tokio::spawn(connect(client, url)); + // Reconnecting, so the GOAWAY that lands with the relay's acceptance leaves the drained + // session readable as the predecessor, where a one-shot loop ends on it. + let arriving = tokio::spawn(async move { + kio::clock::timeout(TIMEOUT, client.with_reconnect(true).connect(url).established()) + .await + .expect("connect timeout") + .expect("connect failed") + }); let pending = admissions.next().await.expect("pending admission"); // The relay fixes its deadline inside `start`, so these bracket it: what is // left of each bounds what the relay tells the arrival from below and above. @@ -506,10 +513,8 @@ async fn drain_on_the_trigger(version: Option) { "{label}: an arrival mid-drain got {timeout:?}, short of the {least:?} left of the window" ); } - kio::clock::timeout(TIMEOUT, arrival.closed()) - .await - .unwrap_or_else(|_| panic!("{label}: the arrival never closed")) - .expect("a one-shot session leaves cleanly on GOAWAY"); + // A reconnecting arrival would redial the draining relay, so leave instead. + drop(arrival); // `run` joins the worker threads before returning, so returning at all is // the clean join; the rebind proves every worker let go of the port. diff --git a/rs/moq-relay/tests/integration/shutdown_signal.rs b/rs/moq-relay/tests/integration/shutdown_signal.rs index cf95cae703e..a80b6a84e35 100644 --- a/rs/moq-relay/tests/integration/shutdown_signal.rs +++ b/rs/moq-relay/tests/integration/shutdown_signal.rs @@ -349,7 +349,9 @@ async fn a_session_arriving_mid_drain_gets_what_is_left_inner() { // then told to leave at once. kio::clock::sleep(DRAIN_TIMEOUT / 2).await; let dialed = kio::clock::now(); - let arrival = connect(&client, port).await; + // Reconnecting, so the GOAWAY that lands with the relay's acceptance leaves the drained + // session readable as the predecessor, where a one-shot loop ends on it. + let arrival = connect(&client.clone().with_reconnect(true), port).await; let draining = arrival.draining().expect("connected"); let goaway = kio::clock::timeout(Duration::from_secs(1), kio::wait(|waiter| draining.poll(waiter))) .await diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 32a19a3f2af..ba6ffa51376 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -44,8 +44,10 @@ fn verdict(client: &Client, url: &url::Url, err: &Error) -> Verdict { /// Three knobs, each with one job: `initial` and `multiplier` set the wait before /// each retry, `max` caps it, and `healthy` decides which sessions count as failed /// attempts. See [`moq_net::reconnect::Backoff`], which this resolves to, for how -/// the policy applies them. Until a session has stayed up past `healthy` or been drained by -/// a GOAWAY, a failed dial or a closed session ends the connection instead of retrying. +/// the policy applies them. Until the peer has accepted a session's credential or drained a +/// session with a GOAWAY (or, on moq-lite 03/04, which never answer the credential, a session +/// stayed up past `healthy`), a failed dial or a closed session ends the connection instead +/// of retrying. #[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -89,7 +91,8 @@ pub struct Backoff { /// How long a session must stay up to count as healthy. Defaults to 1s; zero is refused. /// - /// A healthy session's end resets the delay and the timeout. One that ends sooner is a + /// A healthy session establishes the connection even when its peer never answered the + /// credential, as on moq-lite 03/04. A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. Set by the embedding binary, not configured. #[usage(skip)] @@ -253,7 +256,10 @@ impl Backoff { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Status { - /// A session connected (the first connect, or a reconnect after a drop). + /// A session connected and its peer accepted the connection credential (the first + /// connect, or a reconnect after a drop). On moq-lite 03/04, which never answer the + /// credential, the session must stay up past [`Backoff::healthy`] instead. A session the + /// peer refuses never reports it. Connected, /// An established session dropped; a reconnect attempt follows. Disconnected, @@ -896,8 +902,9 @@ pub struct Snapshot { /// /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) -/// redials with exponential backoff. Until a session has stayed up past the healthy bar or -/// been drained by a GOAWAY, nothing retries: a failed dial, or a session the peer closes, as a relay refusing a +/// redials with exponential backoff. Until the peer has accepted a session's credential or +/// drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), nothing retries: a failed dial, or a session the peer closes, as a relay refusing a /// credential does, stops the loop with its error, so a setup mistake surfaces at once. The read surface /// mirrors [`moq_net::Session`] so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), @@ -1119,6 +1126,9 @@ impl Connection { let (mut probe, mut fallback): (Option, Option) = (None, None); // Mirrors the live session's estimates while it serves. let mut forward: Option = None; + // The transport of the connected session the policy has yet to establish, published + // only once it does, so a refused session never shows as connected. + let mut pending: Option = None; let mut timer: Option = None; // Consecutive walks that failed since the last session, and whether a dial failed // since then, which tells a failed walk from a session that ended too soon. @@ -1135,8 +1145,7 @@ impl Connection { Ok(crate::client::Dialed { session, transport }) => { tracing::info!(peer = %Endpoint(&url), %transport, "connected"); (failed_walks, dial_failed) = (0, false); - shared.connected(&session, transport); - forward = Some(Forward::new(&session)); + pending = Some(transport); policy.connected(kio::clock::now(), session); if client.on_fallback(&url, transport) { policy.fallback(kio::clock::now()); @@ -1195,6 +1204,12 @@ impl Connection { dial = Some((url, Box::pin(async move { client.dial(attempt).await }))); } // The policy already counted the timeout; dropping the dial cancels it. + Action::Established(session) => { + if let Some(transport) = pending.take() { + shared.connected(&session, transport); + forward = Some(Forward::new(&session)); + } + } Action::Abandon => { if dial.is_some() { dial_failed = true; @@ -1256,24 +1271,27 @@ impl Connection { /// Poll until a session is established (the first connect, or the current one). /// /// `Ready(Ok(session))` while a session is live (during a GOAWAY handover this is the - /// old session, which still serves), `Ready(Err)` once the loop has stopped, `Pending` + /// old session, which still serves), or once the loop has stopped after one was, since + /// a peer can accept a one-shot session and end it before this is polled. `Ready(Err)` + /// once the loop has stopped without one, as on a refused credential, `Pending` /// otherwise. pub fn poll_established(&self, waiter: &kio::Waiter) -> Poll> { match ready!(self.state.poll(waiter, |state| match &state.lifecycle { Lifecycle::Live(_) => Poll::Ready(Ok(())), - Lifecycle::Closed(_) => Poll::Ready(Err(terminal(state))), + Lifecycle::Closed(_) => Poll::Ready(stopped(state)), Lifecycle::Connecting { .. } | Lifecycle::Offline { .. } => Poll::Pending, })) { Ok(res) => Poll::Ready(res), - Err(state) => Poll::Ready(Err(terminal(&state))), + Err(state) => Poll::Ready(stopped(&state)), } } /// Wait until a session is established, handing the connection back. /// - /// Returns as soon as a session is live, so the first call waits out the initial dial - /// (surfacing its error if the loop gives up, e.g. on an auth failure or in one-shot - /// mode). Useful when a caller wants dial errors up front rather than through + /// Returns once a session reports [`Status::Connected`], so the first call waits out the + /// initial dial and the peer's answer to its credential. If the loop gives up before any + /// session did, as when the peer refuses the credential, it returns that error instead. + /// Useful when a caller wants dial errors up front rather than through /// [`closed`](Self::closed). /// /// It consumes and returns the connection so the reconnecting chain, @@ -1566,6 +1584,14 @@ fn poll_forward(bw: &mut Option, out: &BandwidthProducer, wai /// this reports [`Error::Stopped`]: [`Connection::closed`] treats that as the `Ok` /// it is, and a watcher that only has an error to go on can still tell it apart /// from a connection that failed. +/// What [`Connection::poll_established`] reports for a stopped loop: whether any session was. +fn stopped(state: &State) -> crate::Result<()> { + match state.connects { + 0 => Err(terminal(state)), + _ => Ok(()), + } +} + fn terminal(state: &State) -> Error { match &state.lifecycle { Lifecycle::Closed(Some(err)) => err.clone(), @@ -2041,8 +2067,9 @@ mod tests { let (accepted, mut sessions) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(async move { while let Ok(request) = server.accept().await { - let _ = accepted.send(()); - let _ = request.ok().await; + // Held open: a session the server drops before the client reads its answer + // is a refusal. + let _ = accepted.send(request.ok().await); } }); @@ -2338,15 +2365,19 @@ mod tests { let mut request = kio::clock::timeout(WAIT, listener.accept()).await.unwrap().unwrap(); let mut requests = request.requests(); let session = request.ok().await.unwrap(); - loop { + // Both, in either order: the connection's own credential, whose AUTH_OK establishes + // the session, and the token. + let (mut credential, mut presented) = (false, false); + while !(credential && presented) { let request = kio::clock::timeout(WAIT, requests.next()).await.unwrap().unwrap(); if request.token().is_empty() { issued.push(request.accept(moq_net::auth::Grant::default())); + credential = true; continue; } assert_eq!(request.token().as_ref(), b"token", "round {round}"); issued.push(request.accept(moq_net::auth::Grant::all())); - break; + presented = true; } if let Some(added) = added.take() { token = Some(added.await.unwrap().expect("granted")); diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index d316d5e3fa6..36e0677de17 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -277,10 +277,13 @@ async fn sni_test() { let fingerprints = server.certificates().fingerprints(); assert_eq!(fingerprints.len(), 2, "expected both certificates to be served"); - // Keep accepting so the client handshakes complete, then drop the server. + // Keep accepting and holding each session, so the client sees it accepted. let server_handle = tokio::spawn(async move { let mut server = server; - while server.accept().await.is_ok() {} + let mut sessions = Vec::new(); + while let Ok(request) = server.accept().await { + sessions.push(request.ok().await); + } }); let dial = |host_name: &str, fingerprint: &str| { diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index 2176e227d96..3a7be1066d8 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -2671,10 +2671,14 @@ async fn one_shot_connect_surfaces_the_session_close() { let accepts = Arc::new(AtomicUsize::new(0)); let server_accepts = accepts.clone(); let pub_origin = moq_tokio::origin::spawn(); + // Each session is aborted once the client is established on it: one aborted sooner is a + // refusal, which `established` reports instead. + let (established, mut wait_established) = tokio::sync::mpsc::unbounded_channel::<()>(); let server_handle = tokio::spawn(async move { while let Ok(request) = server.accept().await { server_accepts.fetch_add(1, Ordering::SeqCst); let session = request.with_publisher(&pub_origin).ok().await?; + wait_established.recv().await; session.abort(moq_net::Error::Cancel); } Ok::<_, anyhow::Error>(()) @@ -2693,6 +2697,7 @@ async fn one_shot_connect_surfaces_the_session_close() { .await .expect("connect timed out") .expect("connect failed"); + established.send(()).unwrap(); // The server aborts the session; one-shot mode turns that into the terminal error. kio::clock::timeout(TIMEOUT, connection.closed()) @@ -3346,9 +3351,12 @@ async fn goaway_timeout_force_close_moq_transport_22_quic() { let client = client_config.init(Default::default()).expect("failed to init client"); let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); + // Sent once the client is established, so the one-shot loop is not over before it is. + let (established, wait_established) = tokio::sync::oneshot::channel::<()>(); let server_handle = tokio::spawn(async move { let request = server.accept().await.expect("no incoming connection"); let session = request.with_publisher(&pub_origin).ok().await?; + let _ = wait_established.await; session .goaway(moq_net::goaway::Goaway::new().with_timeout(Some(Duration::from_millis(200)))) @@ -3366,6 +3374,7 @@ async fn goaway_timeout_force_close_moq_transport_22_quic() { // Observe the GOAWAY but do NOT leave. let draining = connection.draining().expect("connected"); + established.send(()).unwrap(); let goaway = kio::clock::timeout(TIMEOUT, kio::wait(|waiter| draining.poll(waiter))) .await .expect("goaway timed out") @@ -3444,10 +3453,10 @@ async fn zero_initial_backoff_still_gives_up_on_a_flapping_peer() { } /// A rejection at the MoQ layer lands after the transport handshake, so the dial -/// itself succeeds and the rejection arrives as the session's close. That close -/// rides the specified UNAUTHORIZED session code, so it decodes back into a typed -/// error rather than an opaque transport string, even in one-shot mode where the -/// session ending is what ends the connection. +/// itself succeeds and the rejection arrives as the session's close, before the client +/// ever reports it connected. That close rides the specified UNAUTHORIZED session code, so +/// it decodes back into a typed error rather than an opaque transport string, which +/// `established` returns. #[tokio::test] async fn session_close_surfaces_a_rejection_code() { let (mut server, addr) = test_server().await; @@ -3460,14 +3469,11 @@ async fn session_close_surfaces_a_rejection_code() { Ok::<_, anyhow::Error>(()) }); - let (_client, connection) = kio::clock::timeout(TIMEOUT, connect_once(test_client(), url)) - .await - .expect("connect timed out") - .expect("connect failed"); - let err = kio::clock::timeout(TIMEOUT, connection.closed()) - .await - .expect("close timed out") - .expect_err("a rejected session must surface as an error"); + let err = expect_connect_err( + kio::clock::timeout(TIMEOUT, connect_once(test_client(), url)) + .await + .expect("connect timed out"), + ); assert_connect_error(&err, moq_tokio::ConnectError::Unauthorized); server_handle.abort(); @@ -3475,7 +3481,8 @@ async fn session_close_surfaces_a_rejection_code() { /// The same rejection of a first session under the reconnect loop ends it with the code, /// as a refused first dial does: on these versions the refusal arrives after ALPN, so the -/// session connected first, yet redialing would only present the same credential again. +/// transport connected first, yet redialing would only present the same credential again. +/// The refused session never reports as connected. #[tokio::test] async fn reconnect_ends_on_a_first_session_rejection() { for code in [ @@ -3501,6 +3508,11 @@ async fn reconnect_ends_on_a_first_session_rejection() { .await .unwrap_or_else(|_| panic!("{code:?} kept the loop redialing")) .expect_err("a rejected first session must surface as an error"); + assert_eq!( + connection.connects(), + 0, + "{code:?} showed the refused session as connected" + ); server_handle.abort(); let mut count = 0; while accepts.try_recv().is_ok() { diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 0d2f24aa334..444114ebf8b 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -91,8 +91,9 @@ public typealias ConnectionStats = MoqFFI.MoqConnectionStats /// Retry pacing for the automatic reconnect; see `Client(backoff:)`. A `nil` /// field keeps its default, zero means zero (retry at once, or give up at the -/// first failure), and `maxUnlimited` lifts the ceiling. Until a session has -/// stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed +/// first failure), and `maxUnlimited` lifts the ceiling. Until the peer has +/// accepted a session's credential or drained a session with a GOAWAY +/// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed /// session, such as a refused credential, ends the connection with its error /// rather than retrying. public typealias Backoff = MoqFFI.MoqBackoff diff --git a/swift/Sources/Moq/Client.swift b/swift/Sources/Moq/Client.swift index 1be3e289c78..9136cab1567 100644 --- a/swift/Sources/Moq/Client.swift +++ b/swift/Sources/Moq/Client.swift @@ -65,7 +65,8 @@ public final class Client: Sendable { } /// Start a connection and return its session without waiting for the native first dial to succeed. - /// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed + /// Until the peer has accepted a session's credential or drained a session with a GOAWAY + /// (or, on moq-lite 03/04, which never answer it, a session stayed up 1s), a failed dial or a closed /// session, such as a refused credential, closes the session with its error rather than retrying. public func connect(to url: String) async throws -> Session { Session(try await ffi.connect(url: url)) diff --git a/test/drill/mutations/reconnect-stops-after-session-loss.patch b/test/drill/mutations/reconnect-stops-after-session-loss.patch index 09322a59de6..87d79be4499 100644 --- a/test/drill/mutations/reconnect-stops-after-session-loss.patch +++ b/test/drill/mutations/reconnect-stops-after-session-loss.patch @@ -8,7 +8,7 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs -@@ -1088,7 +1088,8 @@ impl + From> Reco +@@ -1104,7 +1104,8 @@ impl + From> Reco fn closed(&mut self, now: Instant, err: Error, healthy: bool, url: &Url) { let err = E::from(err); // Connecting once, the session ending ends the policy with its close reason. @@ -17,4 +17,4 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs + if self.config.once || healthy { return self.done(Err(err)); } - // Until a session has stayed up past the healthy bar, a close ends the policy as a + // Until a session was established, a close ends the policy as a failed walk does. It diff --git a/test/interop/README.md b/test/interop/README.md index d3d72dc2ce8..b00e364cb51 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -124,8 +124,8 @@ released relays and clients, skip it. Whenever the browser subscriber is in the run, the matrix ends with a close-code case: Chromium dials the relay with a token its public rules refuse, and `WebTransport.closed` must carry the relay's code and reason. A WebSocket-only -`Connection` counts the refused session as connected, but it never stayed up -past the healthy bar, so the handle must stop at it without redialing and +`Connection` never reports the refused session as connected, since the relay +never accepted its credential, and must stop at it without redialing and publish the same session error. Chromium treats a server's HTTP/3 control stream ending as fatal, so a server that ends it under the close capsule loses both; no Rust peer is that strict. diff --git a/test/interop/clients/js/src/close.ts b/test/interop/clients/js/src/close.ts index 7385af7292b..d26af09522b 100644 --- a/test/interop/clients/js/src/close.ts +++ b/test/interop/clients/js/src/close.ts @@ -45,8 +45,9 @@ export async function refused(relay: string): Promise { /** * A WebSocket refusal reaches the reconnecting handle as its session error. * - * The relay closes a session that ALPN already established. It never stayed up past the healthy - * bar, so the handle stops at once with that close, as after a refused dial, instead of redialing. + * The relay closes a session that ALPN already established, never having accepted its credential, + * so the handle stops at once with that close, as after a refused dial, instead of redialing, and + * never reports it connected. */ export async function refusedWebSocket(relay: string): Promise { const url = new URL(relay); @@ -58,7 +59,7 @@ export async function refusedWebSocket(relay: string): Promise { share: false, backoff: { initial: Duration.millis(10), max: Duration.millis(10), timeout: null }, }); - // The refused session counted as connected first; a second one means the handle redialed. + // The refused session never reads as connected, so any connect means the relay admitted one. let connected = 0; const dispose = connection.status.subscribe((status) => { if (status === "connected") connected++; @@ -66,8 +67,7 @@ export async function refusedWebSocket(relay: string): Promise { try { let error = connection.error.peek(); while (!error) error = await connection.error.changed(); - if (connected !== 1) - return { error: `stopped after ${connected} session(s) instead of the refused one: ${error}` }; + if (connected !== 0) return { error: `the refused session read as connected ${connected} time(s): ${error}` }; if (!(error instanceof MoqError.Session)) return { error: String(error) }; return { closeCode: error.code, reason: error.message }; } finally {