From 48fe715c5f0b91ae0bb27862135d1afeaf3ec4fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 16:21:20 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m1/reconnect-fail-fast.md Quest-Claim: 51ba98dc8691a3bd From 12bdde72daef41915a06ff3b6c1f5bbfa582c706 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:01:23 -0800 Subject: [PATCH 2/8] wip: first walk fails fast --- .claude/skills/verify-reconnect/SKILL.md | 4 +- cpp/obs/src/moq-dock.cpp | 3 +- cpp/obs/src/moq-settings.cpp | 3 +- cpp/obs/test/moq-output-test.cpp | 29 ++-- dart/moq/lib/src/aliases.dart | 2 + dart/moq/lib/src/client.dart | 5 +- doc/lib/cpp/index.md | 2 +- doc/lib/dart/index.md | 2 +- doc/lib/go/index.md | 2 +- doc/lib/index.md | 2 +- doc/lib/kt/index.md | 2 +- doc/lib/py/index.md | 2 +- doc/lib/rs/moq-net.md | 2 +- doc/lib/swift/index.md | 2 +- go/wrapper/client.go | 7 +- go/wrapper/reconnect_test.go | 33 ++-- js/net/src/connection/pool.test.ts | 93 +++++++----- js/net/src/connection/reload.test.ts | 142 +++++++++++++----- js/net/src/connection/reload.ts | 73 ++++++--- .../kotlin/dev/moq/Aliases.kt | 2 + py/moq-rs/README.md | 2 +- py/moq-rs/moq/_records.py | 3 + py/moq-rs/tests/test_server.py | 16 +- quest/m1/README.md | 1 - quest/m1/reconnect-fail-fast.md | 62 -------- quest/m2/reconnect-no-error-kinds.md | 6 +- rs/moq-ffi/src/session.rs | 23 +-- rs/moq-ffi/src/test.rs | 38 +++-- rs/moq-net/src/client.rs | 6 + rs/moq-net/src/reconnect.rs | 22 ++- rs/moq-net/src/reconnect/tests.rs | 77 +++++++--- rs/moq-net/src/setup.rs | 13 +- rs/moq-net/verify/reconnect/Handover.cfg | 1 + rs/moq-net/verify/reconnect/Handover.tla | 31 +++- rs/moq-net/verify/reconnect/RestartWait.cfg | 1 + rs/moq-tokio/src/connection.rs | 3 +- swift/Sources/Moq/Aliases.swift | 4 +- swift/Sources/Moq/Client.swift | 2 + 38 files changed, 451 insertions(+), 272 deletions(-) delete mode 100644 quest/m1/reconnect-fail-fast.md diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index d6bb2cfc4c2..2d3195631d3 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -9,7 +9,7 @@ description: Model-check the client connect policy (`rs/moq-net/src/reconnect.rs ## Rules -Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan` and `RedirectFollowed` are checked only while `run` waits with the queue drained, which is when another task can read the handle. +Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `RedirectFollowed`, and `FirstWalkFailsFast` are checked only while `run` waits with the queue drained, which is when another task can read the handle. | Rule | Holds when | Harm if broken | | --- | --- | --- | @@ -18,6 +18,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan` and | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"). | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | +| `FirstWalkFailsFast` | Nothing backs off before a session has been established: a failed first walk ends the policy. | A setup mistake (an untrusted certificate, no version in common) retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run @@ -54,6 +55,7 @@ Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07 | Code | Model | | --- | --- | | `reconnect.rs` `Reconnect::poll` (queue, predecessor, phases) | `Pop`, `Retire`, `IdleClose`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | +| `reconnect.rs` `established`, and `walk_failed` ending the first walk | `Established`, `Failed`, `Abandon` | | `reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `reconnect.rs` `Restart::Wait` | `LiveEnds` with `Restart = "wait"` (`RestartWait.cfg`) | diff --git a/cpp/obs/src/moq-dock.cpp b/cpp/obs/src/moq-dock.cpp index 02eb02b029f..733f0f5f21a 100644 --- a/cpp/obs/src/moq-dock.cpp +++ b/cpp/obs/src/moq-dock.cpp @@ -1212,7 +1212,8 @@ void MoQDock::UpdateStatus() const bool everConnected = moq && (moq->IsLiveSession() || obs_output_get_connect_time_ms(output) > 0); if (!everConnected) { if (!failText.isEmpty()) { - // The first dial is still retrying; one that gave up stops the output instead. + // A first dial that reaches no session stops the output, so this shows only + // until that stop lands. status->setText(QString("● Connecting… · %1").arg(failText)); status->setStyleSheet("color: #d08b1d;"); statsBox->setText(failText); diff --git a/cpp/obs/src/moq-settings.cpp b/cpp/obs/src/moq-settings.cpp index 6bf25c5ef22..3528bfebc81 100644 --- a/cpp/obs/src/moq-settings.cpp +++ b/cpp/obs/src/moq-settings.cpp @@ -206,7 +206,8 @@ const std::vector &Fields() 600000, 1000, "Ceiling on the growing reconnect delay.")); f.push_back(Number(BACKOFF_TIMEOUT, "Give up after (ms)", Millis(d.backoff.timeout, BACKOFF_TIMEOUT), 0, 3600000, 1000, - "Total time to keep retrying before the stream fails. 0 retries forever.")); + "Total time to keep reconnecting after a drop before the stream fails. 0 retries " + "forever. A first connect that reaches no server fails the stream at once.")); // QUIC transport. f.push_back(Choose( diff --git a/cpp/obs/test/moq-output-test.cpp b/cpp/obs/test/moq-output-test.cpp index e263fd6091d..c20001c8676 100644 --- a/cpp/obs/test/moq-output-test.cpp +++ b/cpp/obs/test/moq-output-test.cpp @@ -1008,8 +1008,9 @@ int main() } printf("names the failure while reconnecting: ok\n"); - // While the first dial keeps failing, the dock names why instead of a bare - // "Connecting…". The relay is gone before the start, so nothing answers. + // A first dial that reaches no session fails the start for good rather than + // retrying: nothing has ever connected, so the cause is likely setup. The relay + // is gone before the start, so nothing answers. { std::string url; { @@ -1019,21 +1020,21 @@ int main() reset(url); MoQOutput o(nullptr, OUTPUT); CHECK(o.Start()); - std::string reason; - CHECK(WaitFor([&] { - int code = 0; - o.CopyLastFailure(&code, &reason); - return !reason.empty(); - })); - // Still retrying rather than failed, so this is the first-dial path. - CHECK(!o.IsLiveSession()); - CHECK(signalCount() == 0); - o.Stop(); + CHECK(WaitFor([] { return signalCount() == 1; })); + // Never reached the server, so OBS must not retry the endpoint. + CHECK(signalAt(0).code == OBS_OUTPUT_CONNECT_FAILED); + CHECK(signalAt(0).begin_capture_at == 1); + CHECK(!signalAt(0).last_error.empty()); int code = 0; + std::string reason; o.CopyLastFailure(&code, &reason); - CHECK(reason.empty()); + CHECK(reason == signalAt(0).last_error); + CHECK(!o.IsLiveSession()); + // The failure retired the attempt, so nothing it queued can report again. + MoQOutputStatsTest::Drain(o); + CHECK(signalCount() == 1); } - printf("names the failure while the first dial retries: ok\n"); + printf("a failing first dial fails the start: ok\n"); // Stopping while the connect is still pending cancels it: nothing but the stop // reaches OBS, then or later. diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index ab7391cf40e..722d967228e 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -255,6 +255,8 @@ typedef ConnectionPhaseClosed = ClosedMoqConnectionPhase; typedef ConnectionState = MoqConnectionState; /// Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. +/// It applies only once a session has been established: a first dial that +/// reaches no session ends the connection with its error rather than retrying. typedef Backoff = MoqBackoff; /// Whether a protocol code is from the session or stream registry. diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 77c91be2a0b..849e318e440 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -60,8 +60,9 @@ final class ConnectOptions { /// The WebSocket fallback raced against QUIC for `http(s)` URLs. final WebSocketConfig? websocket; - /// Set true for a one-shot dial. By default the session redials with - /// backoff whenever the transport drops. + /// Set true for a one-shot dial. By default an established session redials + /// with backoff whenever the transport drops; a first dial that reaches no + /// session ends it with that error instead. final bool once; /// Retry pacing for the automatic reconnect. diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index fef1c6f21c1..ca9fc3b6e2e 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -255,7 +255,7 @@ Set `ClientConfig::tokens` instead of putting the credential in the URL. The fir ## Stats -`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its first-dial failures and reconnects; independent watchers keep their own revisions. +`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (a first dial that reaches no session ends the session with that error rather than retrying); independent watchers keep their own revisions. ## Shutdown diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index f65491913f1..95f60a2117e 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -156,7 +156,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` drains for up to one second and reports its result. `session.shutdown()` requests the drain synchronously; `await session.closed()` waits for its result. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements and grants also release their consumer; server listeners leave their shared server usable. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` or disposing its `MoqRoute` takes it offline. How long to keep it up is yours to decide. A track still open at the close, or released without a finish, ends with `Unroutable` (`Internal` over moq-transport, which has no such code). Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe first-dial failures and reconnects. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `client.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps: ...)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps: ...)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `dispose()` it once the broadcast stops being watched. `setTarget(targetBps: ...)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `OutOfRangeMoqException`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. - **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 8f3a4e95c1c..a6e7d9aa1a9 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -133,7 +133,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown()` requests that drain immediately; `session.Closed(ctx)` waits for its result, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `Finish()` on its `media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.Close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close, or released without a finish, ends with `Unroutable` (`Internal` over moq-transport, which has no such code). Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe first-dial failures and reconnects. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `Session().Viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `Watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `Rate()` reads its slice in bits per second, nil while there is no estimate, so hold the current selection. `Update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `Close()` it once the broadcast stops being watched, from the goroutine that last used it. `SetTarget(&bps)` sends another target, zero sends none, and nil follows the viewers again. `Watch`, `Update`, and `SetTarget` return `moq.ErrOutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. Client viewers survive reconnects and read nil while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Cancel()` it when done. - **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. diff --git a/doc/lib/index.md b/doc/lib/index.md index eb8b791553b..f75f70c36a2 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -66,7 +66,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. +- **Reconnect** automatically with backoff when the transport drops. The first dial fails fast instead: a relay that never answered with a session ends the connection with its error, for the app to retry. - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 369105923ba..27679f8ebfc 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -110,7 +110,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `moq.shutdown()` requests a drain immediately; suspending `moq.session.closed()` waits up to one second and reports its result. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.end()` takes it offline. How long to keep it up is yours to decide. A track still open at the end, or released without a finish, ends with `Unroutable` (`Internal` over moq-transport, which has no such code). Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe first-dial failures and reconnects. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `moq.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `close()` it, or leave `use { }`, once the broadcast stops being watched. `setTarget(bps)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `MoqException.OutOfRange`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. - **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 51ca45f9b3c..db331e91f5c 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -125,7 +125,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `await session.closed()` waits up to one second and reports its result. `cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `moq.media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close, or released without a finish, ends with `Unroutable` (`Internal` over moq-transport, which has no such code). Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe first-dial failures and reconnects. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(max_bps)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `None` while there is no estimate (keep the current rendition), and `update(max_bps)` changes the ceiling. Holding a viewer requests the estimate like `recv_bandwidth()`, and dropping it releases the claim. The session sends the summed ceilings as its PROBE target; `set_target(bps)` overrides that, `0` sends none, and `None` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, raise `moq.Error.OutOfRange`. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. - **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 2925131f324..6b78f82085a 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -29,7 +29,7 @@ What you use it for, beyond what the concept page already describes: - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. - **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). -- **Reconnect policy comes with it.** With the `async` feature, `moq_net::Reconnect` decides which of a peer's addresses to dial, how long each dial and handshake may take, when to back off and give up, and how to follow a GOAWAY, without doing I/O: you dial each `Attempt` it asks for and report back. `moq-tokio`'s `Connection` runs it, so a client on another runtime gets the same policy rather than a copy. +- **Reconnect policy comes with it.** With the `async` feature, `moq_net::Reconnect` decides which of a peer's addresses to dial, how long each dial and handshake may take, when to back off and give up, and how to follow a GOAWAY, without doing I/O: you dial each `Attempt` it asks for and report back. `moq-tokio`'s `Connection` runs it, so a client on another runtime gets the same policy rather than a copy. The first walk of the peer's addresses fails fast: if it yields no session, the policy ends with that walk's error instead of retrying, so a setup mistake such as an untrusted certificate or no version in common surfaces at once, and a client started before its relay is reachable exits for its supervisor to restart. Once a session has been established, later failures back off and retry, so relay restarts still recover. - **A session bounds what its peer can hold.** Each request holds a stream, so the transport's stream credit bounds subscriptions and FETCHes. Once that credit runs out, a waiting open resets the lowest-priority group stream below it, which the subscriber sees dropped, so the credit it frees goes to the higher-priority group. `session::Limits` (via `Client::with_limits` and `Server::with_limits`) bounds the announcements per session. The default suits a relay mesh, so lower it for untrusted peers. Past it, the session closes with `TOO_MANY_REQUESTS`. Peer-declared lengths are capped before they are buffered. - **Authorization is in band.** On moq-lite 07 (`moq-lite-07-wip`, opt-in) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `session.auth()` presents tokens without reconnecting and watches the grant the peer sent back. `add(token)` resolves with the token once the peer takes it: on its AUTH stream, or on moq-transport without AUTH, where only a request that names it carries it. A handle passed to `Client::with_auth` outlives each session, so its tokens, and the routes naming them, carry across your own reconnects. A server verifies tokens itself by taking `handshake.requests()` before `ok()`. `authorize(&grant)` narrows or widens a live session on any version: what falls outside resets with `Unauthorized` and the session stays up. A client that publishes outside its grant closes the session, naming the path. - **A graceful close waits.** `session.close().await` withdraws announcements and drains for up to one second. `moq-lite-05` and newer wait for the subscriber to receive finished tails; older Lite versions and IETF wait only for transport ACKs. `abort` ends immediately. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index b35e5fe4976..0109d423bb1 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -107,7 +107,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `try await session.closed()` waits up to one second and reports its result. `session.cancel(code:)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try finish()` on its `Media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()`, or releasing its route, takes it offline. How long to keep it up is yours to decide. A track still open at the close, or released without a finish, ends with `Unroutable` (`Internal` over moq-transport, which has no such code). Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe first-dial failures and reconnects. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(maxBps:)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `nil` while there is no estimate (keep the current rendition), and `update(maxBps:)` changes the ceiling. Holding a viewer requests the estimate like `recvBandwidth()`, and releasing it releases the claim. The session sends the summed ceilings as its PROBE target; `setTarget(targetBps:)` overrides that, `0` sends none, and `nil` follows the viewers again. `watch`, `update`, and `setTarget` throw `MoqError.OutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. - **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. diff --git a/go/wrapper/client.go b/go/wrapper/client.go index 61e1ccef700..9b76bbb1c68 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -205,7 +205,9 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // Backoff is the retry pacing for automatic reconnects: the delay starts at // Initial, multiplies by Multiplier after each failed attempt, and caps at Max. -// After Timeout of consecutive failures the connection gives up for good. +// After Timeout of consecutive failures the connection gives up for good. It +// applies only once a session has been established: a first dial that reaches no +// session ends the connection with its error rather than retrying. // // Every field is optional: the zero value means the native default (1s, x2, 5s, // and no give-up), so a partial Backoff overrides only what it sets. Pass @@ -270,7 +272,8 @@ type Client struct { } // Dial starts a MoQ connection and returns without waiting for its first native dial to succeed. Cancel ctx -// to abort an in-flight connect. +// to abort an in-flight connect. If that first dial reaches no session, the session closes with its error +// (see Session.Closed) rather than retrying; only an established session reconnects. func Dial(ctx context.Context, url string, config ClientConfig) (*Client, error) { native, err := config.ffi() if err != nil { diff --git a/go/wrapper/reconnect_test.go b/go/wrapper/reconnect_test.go index e8ed7ea205f..20e7cc7a18d 100644 --- a/go/wrapper/reconnect_test.go +++ b/go/wrapper/reconnect_test.go @@ -342,7 +342,8 @@ func TestDialConnectionOptions(t *testing.T) { } } -// A public wrapper owns its first dial through Session and observes repeated failures. +// A public wrapper owns its first dial through Session and observes it fail: a first dial +// that reaches no session ends the session with its error rather than retrying. func TestFirstDialState(t *testing.T) { silent, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { @@ -363,28 +364,30 @@ func TestFirstDialState(t *testing.T) { } session := client.Session() defer session.Cancel(0) - state := session.State() - if _, ok := state.Phase.(moq.ConnectionPhaseConnecting); !ok || state.Connects != 0 { - t.Fatalf("initial state: %+v", state) - } - for i := 0; i < 2; i++ { - state, err = session.StateChanged(ctx, state.Revision) - if err != nil { - t.Fatal(err) - } - if phase, ok := state.Phase.(moq.ConnectionPhaseConnecting); !ok || phase.Failure == nil || state.Connects != 0 { - t.Fatalf("failed dial state: %+v", state) - } + initial := session.State() + if _, ok := initial.Phase.(moq.ConnectionPhaseConnecting); !ok || initial.Connects != 0 { + t.Fatalf("initial state: %+v", initial) } // A canceled waiter must leave both the session and another observer live. canceled, stop := context.WithCancel(ctx) stop() - if _, err := session.StateChanged(canceled, state.Revision); !errors.Is(err, context.Canceled) { + if _, err := session.StateChanged(canceled, initial.Revision); !errors.Is(err, context.Canceled) { t.Fatalf("canceled waiter: %v", err) } - if _, err := session.StateChanged(ctx, state.Revision); err != nil { + state, err := session.StateChanged(ctx, initial.Revision) + if err != nil { t.Fatalf("independent waiter: %v", err) } + if phase, ok := state.Phase.(moq.ConnectionPhaseClosed); !ok || !errors.Is(phase.Failure, moq.ErrConnect) || state.Connects != 0 { + t.Fatalf("failed dial state: %+v", state) + } + // The terminal revision never changes again, so a wait there reports the failure. + if _, err := session.StateChanged(ctx, state.Revision); !errors.Is(err, moq.ErrConnect) { + t.Fatalf("terminal waiter: %v", err) + } + if err := session.Closed(ctx); !errors.Is(err, moq.ErrConnect) { + t.Fatalf("closed: %v", err) + } } // isConnected reports whether a session serves with a negotiated version. diff --git a/js/net/src/connection/pool.test.ts b/js/net/src/connection/pool.test.ts index 59792302de1..643845b9e00 100644 --- a/js/net/src/connection/pool.test.ts +++ b/js/net/src/connection/pool.test.ts @@ -189,15 +189,50 @@ test("switching URLs switches origins", async () => { handle.close(); }); -test("a shared connection outlasts an outage longer than a finite retry window", async () => { - let offline = true; - const stub = function StubWebTransport() { - if (offline) throw new Error("relay is down"); +/** + * Accept each dial until `outage` takes the relay down, after which each dial throws or hangs + * until `recover`. A URL's first attempt is never retried, so `outage` waits for the handle's + * session and holds it past the default healthy bar before severing it: the loop then retries + * with a fresh backoff and retry window, both counting from the outage. + */ +function stubOutage(down: "throw" | "hang") { + let offline = false; + let attempts = 0; + const servers: Promise<{ abort(): void }>[] = []; + class Hung { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() {} + } + globalThis.WebTransport = function StubWebTransport() { + attempts++; + if (offline) { + if (down === "throw") throw new Error("relay is down"); + return new Hung(); + } const pair = createMockTransportPair(Lite.ALPN_05); - void accept(pair.server, url); + servers.push(accept(pair.server, url)); return pair.client; + } as unknown as typeof WebTransport; + return { + attempts: () => attempts, + outage: async (handle: Connection) => { + await advance(); + expect(handle.status.peek()).toBe("connected"); + await advance(1000); + offline = true; + (await servers.at(-1))?.abort(); + await advance(); + expect(handle.status.peek()).not.toBe("connected"); + }, + recover: () => { + offline = false; + }, }; - globalThis.WebTransport = stub as unknown as typeof WebTransport; +} + +test("a shared connection outlasts an outage longer than a finite retry window", async () => { + const relay = stubOutage("throw"); // Keep the fallback local too, so offline attempts are driven entirely by fake time. globalThis.WebSocket = function OfflineWebSocket() { @@ -206,6 +241,7 @@ test("a shared connection outlasts an outage longer than a finite retry window", const handle = new Connection({ url }); try { + await relay.outage(handle); await advance(500); expect(handle.status.peek()).toBe("disconnected"); @@ -213,7 +249,7 @@ test("a shared connection outlasts an outage longer than a finite retry window", await advance(60_000); expect(handle.status.peek()).not.toBe("connected"); - offline = false; + relay.recover(); await advance(10_000); expect(handle.status.peek()).toBe("connected"); } finally { @@ -456,12 +492,7 @@ test("exhausted retries then a new URL recovers a private handle", async () => { }); test("a private partial delay keeps the unlimited retry timeout", async () => { - let attempts = 0; - const stub = function StubWebTransport() { - attempts++; - throw new Error("relay is down"); - }; - globalThis.WebTransport = stub as unknown as typeof WebTransport; + const relay = stubOutage("throw"); const handle = new Connection({ url, @@ -470,8 +501,10 @@ test("a private partial delay keeps the unlimited retry timeout", async () => { delay: { initial: Duration.millis(1000), multiplier: 1, max: Duration.millis(1000) }, }); try { + await relay.outage(handle); + const before = relay.attempts(); await advance(60_000); - expect(attempts).toBeGreaterThanOrEqual(3); + expect(relay.attempts() - before).toBeGreaterThanOrEqual(3); expect(handle.error.peek()).toBeUndefined(); } finally { handle.close(); @@ -479,27 +512,20 @@ test("a private partial delay keeps the unlimited retry timeout", async () => { }); test("a private handle with an explicitly undefined timeout outlasts the outage", async () => { - let offline = true; - let attempts = 0; - const stub = function StubWebTransport() { - attempts++; - if (offline) throw new Error("relay is down"); - const pair = createMockTransportPair(Lite.ALPN_05); - void accept(pair.server, url); - return pair.client; - }; - globalThis.WebTransport = stub as unknown as typeof WebTransport; + const relay = stubOutage("throw"); // What a caller building options from optional values produces: the undefined means "the // default", which for a Connection is to retry for as long as the handle is held. const handle = new Connection({ url, share: false, websocket: { enabled: false }, delay: { timeout: undefined } }); try { + await relay.outage(handle); + const before = relay.attempts(); // Long enough that any finite fallback window would have given up. await advance(60_000); - expect(attempts).toBeGreaterThan(3); + expect(relay.attempts() - before).toBeGreaterThan(3); expect(handle.error.peek()).toBeUndefined(); - offline = false; + relay.recover(); await advance(10_000); expect(handle.status.peek()).toBe("connected"); } finally { @@ -830,16 +856,7 @@ test("GOAWAY keeps version pools distinct at the redirected URL", async () => { }); test("a private handle's timeout bounds each attempt", async () => { - let attempts = 0; - class Hung { - ready = new Promise(() => {}); - closed = new Promise(() => {}); - close() {} - } - globalThis.WebTransport = function StubWebTransport() { - attempts++; - return new Hung(); - } as unknown as typeof WebTransport; + const relay = stubOutage("hang"); const handle = new Connection({ url, @@ -849,10 +866,12 @@ test("a private handle's timeout bounds each attempt", async () => { delay: { initial: Duration.millis(10), multiplier: 1, max: Duration.millis(10) }, }); try { + await relay.outage(handle); + const before = relay.attempts(); // Each hung dial is cut at 50 ms and redialed after the backoff; the default 30 s // timeout would still be holding the first. await advance(1_000); - expect(attempts).toBeGreaterThanOrEqual(3); + expect(relay.attempts() - before).toBeGreaterThanOrEqual(3); } finally { handle.close(); } diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index f5e188bbc28..3d6af3dac35 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -490,7 +490,8 @@ test("a refused credential replaced since its dial redials with the new one", as test("new tokens redial a URL that stopped on refusing the old ones", async () => { using clock = manual(); const original = globalThis.WebTransport; - const { dialed, stub } = refusing("old", Promise.resolve()); + const { promise: release, resolve } = Promise.withResolvers(); + const { dialed, stub } = refusing("old", release); globalThis.WebTransport = stub; const tokens = new Signal(["old"]); @@ -502,6 +503,9 @@ test("new tokens redial a URL that stopped on refusing the old ones", async () = delay: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); try { + // Refused only once a session is up: a URL's first attempt failing stops it whatever the cause. + await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + resolve(); await clock.drive(waitFor(reload.error, (error) => error !== undefined)); expect(dialed.peek()).toEqual(["old"]); tokens.set(["new"]); @@ -541,6 +545,7 @@ test("an unauthorized session close during setup stops retrying", async () => { enabled: true, url, websocket: { enabled: false }, + tokens: ["old"], // Unlimited retries prove that `closed` settles only because the rejection is terminal. delay: { initial: Duration.millis(1), multiplier: 1, max: Duration.millis(1), timeout: null }, }); @@ -553,10 +558,15 @@ test("an unauthorized session close during setup stops retrying", async () => { expect(attempts).toBe(1); expect(reload.closed.peek()).toBeUndefined(); + // The first attempt failing stops the URL, yet new credentials still earn a dial. + reload.tokens.set(["new"]); + await waitUntil(() => attempts === 2); + await waitUntil(() => reload.error.peek() !== undefined); + // A new URL starts another sequence on the same loop. reload.url.set(new URL("https://example.com/changed")); - await waitUntil(() => attempts >= 2); - expect(attempts).toBeGreaterThanOrEqual(2); + await waitUntil(() => attempts >= 3); + expect(attempts).toBeGreaterThanOrEqual(3); } finally { reload.close(); globalThis.WebTransport = original; @@ -571,7 +581,15 @@ test("a setup stream reset is not mistaken for an unauthorized session", async ( const original = globalThis.WebTransport; const url = new URL("https://example.com/"); const attempts = new Signal(0); + let dials = 0; + let first: Promise<{ abort(): void }> | undefined; const stub = function StubWebTransport() { + // The first dial connects: a URL's first attempt failing stops it whatever the cause. + if (dials++ === 0) { + const pair = createMockTransportPair(Lite.ALPN_06); + first = accept(pair.server, url); + return pair.client; + } // No ALPN, so the handshake exchanges SETUP over a bidi stream we can reset. const pair = createMockTransportPair(""); void (async () => { @@ -597,7 +615,10 @@ test("a setup stream reset is not mistaken for an unauthorized session", async ( }); try { + await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + (await first)?.abort(); await clock.drive(waitFor(attempts, (n) => n >= 2)); + expect(reload.error.peek()).toBeUndefined(); } finally { reload.close(); globalThis.WebTransport = original; @@ -1231,10 +1252,11 @@ async function elapse(clock: Manual, pred: () => boolean): Promise { } // Stub WebTransport so each dial either hangs, as one placed on a dead target does, or -// connects to a relay that accepts it. -function dialPlan(plan: (dial: number) => "hang" | "ok", url: URL) { +// connects to a relay that accepts it, which `drop` severs. +function dialPlan(plan: (dial: number) => "hang" | "ok") { const original = globalThis.WebTransport; const origin = new OriginProducer(); + const servers: Promise<{ abort(): void }>[] = []; let dials = 0; let hungCloses = 0; class Hung { @@ -1244,16 +1266,18 @@ function dialPlan(plan: (dial: number) => "hang" | "ok", url: URL) { hungCloses++; } } - globalThis.WebTransport = function StubWebTransport() { + globalThis.WebTransport = function StubWebTransport(target: string | URL) { const dial = dials++; if (plan(dial) === "hang") return new Hung(); const pair = createMockTransportPair(Lite.ALPN_06); - void accept(pair.server, url, { publish: origin.consume() }); + servers.push(accept(pair.server, new URL(target), { publish: origin.consume() })); return pair.client; } as unknown as typeof WebTransport; return { dials: () => dials, hungCloses: () => hungCloses, + // Sever the latest accepted session from the server. + drop: async () => (await servers.at(-1))?.abort(), restore: () => { globalThis.WebTransport = original; origin.close(); @@ -1261,24 +1285,71 @@ function dialPlan(plan: (dial: number) => "hang" | "ok", url: URL) { }; } +// Connect, hold the session past the default healthy bar, then sever it from the server. A URL's +// first attempt is never retried, so retrying needs a session first; ending a healthy one +// leaves the backoff and the retry window fresh, both counting from the drop. +async function outlive(clock: Manual, reload: Reload, plan: ReturnType) { + await waitUntil(() => reload.status.peek() === "connected"); + await clock.advance(Duration.millis(1000)); + await plan.drop(); + await waitUntil(() => reload.status.peek() !== "connected"); +} + +test("a URL's first attempt failing stops it until the URL changes", async () => { + const clock = new Manual(); + const installed = clock.install(); + const warned = spyOn(console, "warn").mockImplementation(() => {}); + const plan = dialPlan((dial) => (dial === 0 ? "hang" : "ok")); + const reload = new Reload({ + url: new URL("https://relay.example/room"), + websocket: { enabled: false }, + timeout: Duration.millis(30), + // An unbounded window and a short backoff, so only the first-attempt rule can stop it. + delay: { initial: Duration.millis(10), multiplier: 1, max: Duration.millis(10), timeout: null }, + }); + try { + await waitUntil(() => plan.dials() === 1); + expect(await elapse(clock, () => reload.error.peek() !== undefined)).toBe(30); + expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout); + expect(reload.status.peek()).toBe("disconnected"); + + await clock.advance(Duration.millis(1000)); + expect(plan.dials()).toBe(1); + expect(plan.hungCloses()).toBe(1); + + // A new URL needs its own first session, and dials it. + reload.url.set(new URL("https://relay.example/other")); + await waitUntil(() => reload.status.peek() === "connected"); + expect(plan.dials()).toBe(2); + expect(reload.error.peek()).toBeUndefined(); + } finally { + reload.close(); + plan.restore(); + warned.mockRestore(); + installed.close(); + } +}); + test("an attempt that outlives the connect timeout is retried after the backoff", async () => { const clock = new Manual(); const installed = clock.install(); const url = new URL("https://relay.example/room"); // Scaled down: the attempt times out before a WebTransport hedge would dial. - const plan = dialPlan((dial) => (dial === 0 ? "hang" : "ok"), url); + const plan = dialPlan((dial) => (dial === 1 ? "hang" : "ok")); const statuses: string[] = []; const reload = new Reload({ url, websocket: { enabled: false }, timeout: Duration.millis(50), - delay: { initial: Duration.millis(20) }, + delay: { initial: Duration.millis(20), multiplier: 1 }, }); const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); try { - await waitUntil(() => plan.dials() === 1); - const waited = await elapse(clock, () => plan.dials() === 2); - // The 50 ms timeout, then the first backoff with its equal jitter (10 to 20 ms). + await outlive(clock, reload, plan); + await elapse(clock, () => plan.dials() === 2); + statuses.length = 0; + const waited = await elapse(clock, () => plan.dials() === 3); + // The 50 ms timeout, then the backoff with its equal jitter (10 to 20 ms). expect(waited).toBeGreaterThanOrEqual(60); expect(waited).toBeLessThanOrEqual(70); expect(plan.hungCloses()).toBe(1); @@ -1294,32 +1365,28 @@ test("an attempt that outlives the connect timeout is retried after the backoff" } }); -// Every attempt hangs until its 30 ms connect timeout. The window counts from the first dial, as -// moq-tokio's does, and no attempt starts once it has passed. +// A healthy session ends at 0, then every attempt hangs until its 30 ms connect timeout. The +// window counts from the end of that session, as moq-tokio's does, and no attempt starts once +// it has passed. const windows: [string, ReloadDelay, number, number][] = [ - // Dials at 0, 40, and 80; the last runs past the window and gives up when it fails. + // Dials at 10, 50, and 90; the last runs past the window and gives up when it fails. [ "an attempt under way finishes past the window", { initial: Duration.millis(10), timeout: Duration.millis(100) }, - 110, + 120, 3, ], - // The first failure at 30 ms leaves 30 ms, less than the 50 ms wait, so it gives up at the deadline. + // The first failure at 50 ms leaves 10 ms, less than the 20 ms wait, so it gives up at the deadline. [ "a wait that reaches the deadline gives up there", - { initial: Duration.millis(50), timeout: Duration.millis(60) }, + { initial: Duration.millis(20), timeout: Duration.millis(60) }, 60, 1, ], - // Zero is a real bound, not "unlimited": the first failure at 30 ms gives up. - [ - "a zero window gives up when the first attempt fails", - { initial: Duration.millis(10), timeout: Duration.ZERO }, - 30, - 1, - ], + // Zero is a real bound, not "unlimited": the session's end gives up without a redial. + ["a zero window gives up when the session ends", { initial: Duration.millis(10), timeout: Duration.ZERO }, 0, 0], ]; -for (const [name, delay, gaveUp, dials] of windows) { +for (const [name, delay, gaveUp, hung] of windows) { test(`retry window: ${name}`, async () => { const clock = new Manual(); const installed = clock.install(); @@ -1327,7 +1394,7 @@ for (const [name, delay, gaveUp, dials] of windows) { const random = spyOn(Math, "random").mockReturnValue(1); const warned = spyOn(console, "warn").mockImplementation(() => {}); const url = new URL("https://relay.example/room"); - const plan = dialPlan(() => "hang", url); + const plan = dialPlan((dial) => (dial === 0 ? "ok" : "hang")); const reload = new Reload({ url, websocket: { enabled: false }, @@ -1335,11 +1402,12 @@ for (const [name, delay, gaveUp, dials] of windows) { delay: { multiplier: 1, max: Duration.millis(1000), ...delay }, }); try { - await waitUntil(() => plan.dials() === 1); + await outlive(clock, reload, plan); expect(await elapse(clock, () => reload.error.peek() !== undefined)).toBe(gaveUp); - expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout); - expect(plan.dials()).toBe(dials); - expect(plan.hungCloses()).toBe(plan.dials()); + // The last failure: a hung attempt's timeout, or the session's own end. + expect(reload.error.peek() instanceof ConnectTimeout).toBe(hung > 0); + expect(plan.dials() - 1).toBe(hung); + expect(plan.hungCloses()).toBe(hung); } finally { reload.close(); plan.restore(); @@ -1358,7 +1426,7 @@ test("retry window: a backoff that fires past the window gives up", async () => const random = spyOn(Math, "random").mockReturnValue(1); const warned = spyOn(console, "warn").mockImplementation(() => {}); const url = new URL("https://relay.example/room"); - const plan = dialPlan(() => "hang", url); + const plan = dialPlan((dial) => (dial === 0 ? "ok" : "hang")); const reload = new Reload({ url, websocket: { enabled: false }, @@ -1371,15 +1439,17 @@ test("retry window: a backoff that fires past the window gives up", async () => }, }); try { - await waitUntil(() => plan.dials() === 1); - // The first attempt times out at 30 ms and schedules the next dial for 40 ms. - await clock.advance(Duration.millis(30)); + // A healthy session ends at 0; the window closes at 100 ms. + await outlive(clock, reload, plan); + // The redial at 10 ms times out at 40 ms and schedules the next dial for 50 ms. + await clock.advance(Duration.millis(40)); for (let i = 0; i < 3; i++) await settle(); + expect(plan.dials()).toBe(2); expect(reload.error.peek()).toBeUndefined(); clock.jump(Duration.millis(200)); for (let i = 0; i < 3; i++) await settle(); expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout); - expect(plan.dials()).toBe(1); + expect(plan.dials()).toBe(2); } finally { reload.close(); plan.restore(); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 5ff58fd5a4e..723386d9efd 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -40,14 +40,25 @@ function sameTokens(a: readonly string[], b: readonly string[]): boolean { return a.length === b.length && a.every((token, i) => token === b[i]); } +/** Whether `cause` is the peer refusing the credentials a session presented. */ +function refusal(cause: unknown): cause is SessionError { + return ( + cause instanceof SessionError && + (cause.code === SessionCode.Unauthorized || + cause.code === SessionCode.MalformedAuthToken || + cause.code === SessionCode.Expired) + ); +} + /** * Exponential backoff settings for the reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Every failure is retried, including an attempt that outlives - * {@link ConnectProps.timeout}, until {@link ReloadDelay.timeout} runs out; only an auth - * rejection or a refused GOAWAY redirect stops the current URL sooner. A new URL or a - * disable/re-enable starts another sequence. + * lockstep. Nothing retries until a session to the current URL has been established: the + * first attempt failing stops that URL with its error. After that, every failure is retried, + * including an attempt that outlives {@link ConnectProps.timeout}, until + * {@link ReloadDelay.timeout} runs out; only an auth rejection or a refused GOAWAY redirect + * stops the current URL sooner. A new URL or a disable/re-enable starts another sequence. * * A GOAWAY off a session that outlived {@link ReloadDelay.healthy} migrates at once, unless it * sends the connection back to an endpoint that already redirected it elsewhere and the @@ -79,7 +90,7 @@ export type ReloadDelay = { /** * Maximum total time to spend retrying the current URL before giving up - * (default: `null`, unlimited; zero gives up when the first attempt fails). Counts from the first dial, and restarts after a healthy session + * (default: `null`, unlimited; zero gives up at the first failure). Counts from the first dial, and restarts after a healthy session * outside a redirect loop, a URL change, or a disable/re-enable. It decides whether another * attempt starts and never cuts one that is running, so giving up can take up to one attempt * longer than the window; {@link ConnectProps.timeout} bounds each attempt. A peer that @@ -192,6 +203,11 @@ export type ReloadStatus = "connecting" | "connected" | "disconnected"; * The reconnect loop behind a Connection handle: connects, waits for the session to end, then * redials with exponential backoff. * + * The first attempt for a URL fails fast instead: until a session to it has been established, + * a failed attempt sets {@link Reload.error} rather than retrying, so a setup mistake such as an + * untrusted certificate or no version in common surfaces at once. A page opened before its + * relay is reachable therefore stays disconnected until a new URL or a disable/re-enable. + * * @internal */ export class Reload { @@ -215,7 +231,8 @@ export class Reload { /** * The failure that stopped retrying the current URL, or undefined while the loop is live. * - * Set on an auth rejection or when the retry window expires. Cleared when a new URL or a + * Set when the first attempt for a URL fails, on an auth rejection, or when the retry window + * expires. Cleared when a new URL or a * disable/re-enable starts another sequence, not on a page hide/show. Transient drops * that are still being retried leave this empty. */ @@ -367,6 +384,11 @@ export class Reload { // Whether the sequence stopped because the peer refused those tokens. #refused = false; + // The configured href a session has been established for. Until it matches, a failed attempt + // gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which + // resume a URL that worked. + #establishedHref: string | undefined; + // Increased by 1 each time to trigger a reload. #tick = new Signal(0); @@ -484,14 +506,7 @@ export class Reload { }); // New credentials redial a URL that stopped on an auth refusal of the old ones. - this.#signals.run((effect) => { - const tokens = effect.get(this.tokens); - if (!this.#refused || this.#givenUpHref === undefined || sameTokens(tokens, this.#dialedTokens)) return; - this.#refused = false; - this.#givenUpHref = undefined; - this.#error.set(undefined); - this.#tick.update((tick) => tick + 1); - }); + this.#signals.run((effect) => this.#redialRefused(effect.get(this.tokens))); this.#url = this.#signals.computed((effect) => effect.get(this.url)?.href); // Create a reactive root so cleanup is easier. @@ -616,6 +631,7 @@ export class Reload { this.#serve(connection); this.#setDialing(false); connected = now(); + this.#establishedHref = href; // A cancelled effect resolves undefined, so the sentinel tells the session // closing (null for clean, an Error otherwise) apart from this run being @@ -717,6 +733,18 @@ export class Reload { return; } + // The first attempt fails fast: a URL that never yielded a session is more likely + // misconfigured than restarting, and only the app can fix that. + if (this.#establishedHref !== href) { + console.warn("connection failed, not retrying:", err); + this.#giveUp(error(err)); + // A refused credential still redials once the tokens change, including a + // change that landed while this dial was in flight. + this.#refused = refusal(err); + this.#redialRefused(this.tokens.peek()); + return; + } + console.warn("connection error:", err); this.#retry(effect, connected, err); } @@ -895,13 +923,7 @@ export class Reload { // for good. // // Tokens replaced since that dial are new credentials, so those earn another dial. - if ( - cause instanceof SessionError && - (cause.code === SessionCode.Unauthorized || - cause.code === SessionCode.MalformedAuthToken || - cause.code === SessionCode.Expired) && - sameTokens(this.tokens.peek(), this.#dialedTokens) - ) { + if (refusal(cause) && sameTokens(this.tokens.peek(), this.#dialedTokens)) { console.warn("session rejected as unauthorized, not retrying"); this.#giveUp(cause); this.#refused = true; @@ -938,6 +960,15 @@ export class Reload { }, wait); } + // Redial a URL that stopped on an auth refusal, once `tokens` differ from the refused ones. + #redialRefused(tokens: readonly string[]): void { + if (!this.#refused || this.#givenUpHref === undefined || sameTokens(tokens, this.#dialedTokens)) return; + this.#refused = false; + this.#givenUpHref = undefined; + this.#error.set(undefined); + this.#tick.update((tick) => tick + 1); + } + #giveUp(cause: Error): void { this.#refused = false; this.#error.set(cause); diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 948c7424528..ec266e9c567 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -366,6 +366,8 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState /** * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. + * It applies only once a session has been established: a first dial that reaches no session ends + * the connection with its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 218369e4015..c6c5b3d289d 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -160,7 +160,7 @@ client = moq.Client( - `.cancel(code)`, `session.shutdown()`. Cancel immediately, or request a drain now and await `session.closed()` for its result, including a delivery timeout after one second. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - `.publish() → OriginProducer`, `.consume() → OriginConsumer`. The wired origin sides. - `.stats() → ConnectionStats`. Snapshot RTT, bandwidth estimates, and byte/packet counters. - - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; first-dial failures and reconnects share the same session. + - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while a first dial that reaches no session ends it with that error. - `.allocator() → Allocator`. Divide the send estimate between encoders and app-owned tracks. - `.viewers() → Viewers`. Divide the receive estimate between watched broadcasts: `.watch(max_bps) → Viewer` claims up to a ceiling, `Viewer.rate()` reads its slice (`None` while there is no estimate), and dropping the viewer releases it. The summed ceilings are the PROBE target; `.set_target(bps)` overrides it (`0` sends none, `None` follows the viewers again). - `.grants() → GrantConsumer`. On `moq-lite-07-wip` and on moq-transport draft-18+ with the MoQ Auth extension, `.grant()` reads the peer's `Grant` and `async for grant in session.grants()` follows it. diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index 615c65773a9..3627fd47f08 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -339,6 +339,9 @@ def _track_info(info: TrackInfo | None) -> MoqTrackInfo | None: class Backoff: """Retry pacing for the automatic reconnect. + It applies only once a session has been established: a first dial that reaches no + session ends the connection with its error rather than retrying. + The delay starts at ``initial``, multiplies by ``multiplier`` after each failed attempt, and caps at ``max``. After ``timeout`` of consecutive failures the connection gives up; ``timedelta(0)`` retries forever. ``None`` keeps each diff --git a/py/moq-rs/tests/test_server.py b/py/moq-rs/tests/test_server.py index e463c6f93c7..b94213b6b7f 100644 --- a/py/moq-rs/tests/test_server.py +++ b/py/moq-rs/tests/test_server.py @@ -172,10 +172,10 @@ async def accept_loop() -> None: async def test_session_state_names_a_failing_first_dial(): - """A first dial returns a session whose state reports each failed attempt.""" + """A first dial that reaches no session ends the session with its error rather than retrying.""" import socket - # A socket that never answers holds the port, so each dial times out. + # A socket that never answers holds the port, so the dial times out. with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as silent: silent.bind(("127.0.0.1", 0)) host, port = silent.getsockname() @@ -198,12 +198,12 @@ async def test_session_state_names_a_failing_first_dial(): state = session.state() assert isinstance(state.phase, moq.ConnectionPhase.CONNECTING) assert state.connects == 0 - for _ in range(2): - state = await asyncio.wait_for(session.state_changed(state.revision), timeout=10) - assert isinstance(state.phase, moq.ConnectionPhase.CONNECTING) - assert isinstance(state.phase.failure, moq.Error.Connect) - assert state.connects == 0 - session.cancel(0) + state = await asyncio.wait_for(session.state_changed(state.revision), timeout=10) + assert isinstance(state.phase, moq.ConnectionPhase.CLOSED) + assert isinstance(state.phase.failure, moq.Error.Connect) + assert state.connects == 0 + with pytest.raises(moq.Error.Connect): + await asyncio.wait_for(session.closed(), timeout=10) async def test_viewer_holds_without_an_estimate(): diff --git a/quest/m1/README.md b/quest/m1/README.md index bfcdf92eb96..72a218754e9 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -54,7 +54,6 @@ moves what remains of them once their claims merge. - [The recurring audits are fewer and carry plans, not results](/quest/m1/audit-merge.md) - m3's 13 audits become about five, each a Goal plus a short check-first list - [The CARLA bridge builds outside the root workspace](/quest/m1/carla-workspace.md) - workspace builds stop compiling the demo bridge - [Draft publishing keeps the upload and drops changelog rewriting](/quest/m1/drafts-publish-trim.md) - `sh/drafts/publish.sh` and its test shrink to the datatracker upload -- [A client's first dial fails fast](/quest/m1/reconnect-fail-fast.md) - a client that never got a session stops instead of retrying, later failures back off, and a version mismatch is logged on both sides - [Find out whether Cloudflare's relay really sends SOLICIT](/quest/m1/cf-solicit.md) - capture its SETUP, then close moq-dev/moq#8418 as their bug or fix our parsing ## Related diff --git a/quest/m1/reconnect-fail-fast.md b/quest/m1/reconnect-fail-fast.md deleted file mode 100644 index 84bab63543d..00000000000 --- a/quest/m1/reconnect-fail-fast.md +++ /dev/null @@ -1,62 +0,0 @@ -# [M] A client's first dial fails fast, and both sides log a version mismatch - -## Goal - -A client whose first walk of the peer's addresses establishes no MoQ session -stops and returns that walk's error instead of retrying. Once a session has -been established, the reconnect loop retries later failures with backoff, as -it does today: each failed handshake grows the delay, and a session that stays up -for `Backoff::healthy` (1 s by default) resets it, so relay restarts and -redeploys still recover. When the client and relay share no MoQ version, the -relay logs the versions offered against those it accepts, and the client -logs the refusal. Rust (`rs/moq-net/src/reconnect.rs`), JS -(`js/net/src/connection/reload.ts`), and moq-ffi's reconnect behave the same. - -## Plan - -Decided 2026-11-10 (maintainer, `/quest-import` of -[moq-dev/moq#8451](https://github.com/moq-dev/moq/issues/8451)). -In the maintainer's words: "any failure during the handshake increases the -backoff; completing the handshake (plus 1s) resets the backoff; the first -connection attempt never retries, so we fail fast." -✅ that rule (rejected: typed errors for version mismatch and untrusted -certificates; a moq2ts-style cap of N failed attempts before a first -session; retrying only when the previous attempt completed a handshake, -which ends every client whose first retry lands while its relay restarts). - -Why: the maintainer doesn't want retry decisions to depend on error kinds, -which miss cases as errors change. Failing the first dial ends the reported -loop (no common version), an untrusted certificate, and every other setup -mistake, with no error classification. - -Interpretation, recorded so it isn't asked again: "the first connection -attempt" is the first walk, so address fallback within that walk still runs. -A client started before its relay is reachable, such as a compose stack or -a phone launching offline, now exits, and its supervisor or app restarts it. -Say so in the reconnect docs and in moq-ffi's session docs. - -Version-mismatch logs: the relay's only line today is `failed to read -capsule e=UnexpectedEnd` from `moq_tokio::quic::web::session`, and the -client logs only `connecting` and `reconnecting after backoff`. Reproduce -with the issue's commands (`moq-relay --listen-version moq-lite-07-wip` -against a client on its default versions) before fixing. - -The error-kind matching in `Verdict::from(&Error)` is deleted by -[the follow-up](/quest/m2/reconnect-no-error-kinds.md), since `Verdict` is -public and its removal breaks. This quest leaves `Verdict` as it is, so after -a first session a `Refused` or `Fatal` verdict still ends the loop until -that quest lands. - -Run the `verify-reconnect` skill's TLA+ model of the connect policy, and -update the model with the new rule. - -Public API: no signature changes; `Reconnect` and `Reload` change behavior -on the first walk (name it in the PR's Impact section). Wire: none. - -## Closes - -- [#8451](https://github.com/moq-dev/moq/issues/8451) - with no MoQ version in common, a client retries forever and neither side logs why - -## Related - -- [moq-relay restarts without dropping sessions](/quest/m3/relay-restart.md) - the restarts this retry rule must keep surviving diff --git a/quest/m2/reconnect-no-error-kinds.md b/quest/m2/reconnect-no-error-kinds.md index cadabf84bec..2c35bc2b69f 100644 --- a/quest/m2/reconnect-no-error-kinds.md +++ b/quest/m2/reconnect-no-error-kinds.md @@ -21,7 +21,7 @@ for every address, stay. Decided 2026-11-10 (maintainer, `/quest-import` of [moq-dev/moq#8451](https://github.com/moq-dev/moq/issues/8451)): ✅ delete the error-kind matching (rejected: keep it). AGENTS.md says never -match on errors, and with [first-dial fail-fast](/quest/m1/reconnect-fail-fast.md) +match on errors, and with first-dial fail-fast (`Reconnect::established`) a setup mistake already ends the client before any session. If what remains of `Verdict` no longer distinguishes anything the walk @@ -41,7 +41,3 @@ update the model with the new rule. Public API: breaking (`Verdict` loses `Refused` and `Fatal`, and `From<&Error> for Verdict` goes). Wire: none. - -## Required - -- [A client's first dial fails fast](/quest/m1/reconnect-fail-fast.md) - the first-session rule that makes classifying errors unnecessary diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 3dc1eb6e124..15eba605e21 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -194,11 +194,12 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// -/// The delay starts at `initial`, multiplies by `multiplier` after each failed attempt, -/// and caps at `max`. After `timeout` of consecutive failures the connection gives up -/// for good; the window resets whenever a session stays up past 1s. Each null field uses -/// moq-tokio's default: 1s, x2, 5s, and no give-up. Zero means off: no wait, no growth, -/// no cap, or no give-up. +/// It applies only once a session has been established: a first dial that reaches no +/// session ends the connection with its error rather than retrying. The delay starts at +/// `initial`, multiplies by `multiplier` after each failed attempt, and caps at `max`. +/// After `timeout` of consecutive failures the connection gives up for good; the window +/// resets whenever a session stays up past 1s. Each null field uses moq-tokio's default: +/// 1s, x2, 5s, and no give-up. Zero means off: no wait, no growth, no cap, or no give-up. #[derive(Clone, Debug, Default, PartialEq, Eq, uniffi::Record)] pub struct MoqBackoff { /// Delay before the first reconnect attempt. 0 retries at once. @@ -590,11 +591,13 @@ impl MoqClient { /// Start a connection without waiting for its first dial to succeed. /// - /// The session automatically reconnects with backoff when the transport drops (unless - /// [`MoqClientConfig::once`] is set), and broadcasts consumed through it ride out the - /// gap. Read [`MoqSession::state`] - /// and wait with [`MoqSession::state_changed`] for first-dial failures and reconnects; - /// [`MoqSession::closed`] waits for the terminal result. + /// Once a session has been established, it automatically reconnects with backoff when + /// the transport drops (unless [`MoqClientConfig::once`] is set), and broadcasts + /// consumed through it ride out the gap. The first dial fails fast instead: if none of + /// the peer's addresses yields a session, [`MoqSession::closed`] resolves with that + /// error, so an app started before its relay is reachable must start the session again. + /// Read [`MoqSession::state`] and wait with [`MoqSession::state_changed`] for + /// reconnects; [`MoqSession::closed`] waits for the terminal result. /// /// Both origin sides are always accessible via [`MoqSession::publish`] and /// [`MoqSession::consume`], without the caller constructing a [`MoqOriginProducer`]. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index b727ca003cf..78914ac102e 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7243,12 +7243,13 @@ async fn wait_phase( Ok(state) } -/// The session exposes repeated first-dial failures even after its Client is released. +/// A failing first dial ends the session with its error, even after its Client is released, +/// rather than retrying: nothing has ever connected, so the cause is likely setup. #[tokio::test] async fn session_state_names_a_failing_first_dial() { let silent = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); let addr = silent.local_addr().unwrap(); - let client = MoqClient::new(MoqClientConfig { + let config = || MoqClientConfig { tls: insecure_tls(), bind: Some("127.0.0.1:0".into()), timeout: Some(Duration::from_millis(100)), @@ -7258,26 +7259,31 @@ async fn session_state_names_a_failing_first_dial() { }, backoff: fast_backoff(), ..Default::default() - }) - .unwrap(); + }; + let client = MoqClient::new(config()).unwrap(); let session = client.connect(format!("https://{addr}")).await.unwrap(); - let mut state = session.state(); + let state = session.state(); assert!(matches!(state.phase, MoqConnectionPhase::Connecting { .. })); assert_eq!(state.connects, 0); drop(client); - for _ in 0..2 { - state = kio::clock::timeout(TIMEOUT, session.state_changed(state.revision)) - .await - .unwrap() - .unwrap(); - assert!(matches!( + let state = kio::clock::timeout(TIMEOUT, session.state_changed(state.revision)) + .await + .unwrap() + .unwrap(); + assert!( + matches!( state.phase, - MoqConnectionPhase::Connecting { + MoqConnectionPhase::Closed { failure: Some(MoqError::Connect { .. }) } - )); - assert_eq!(state.connects, 0); - } + ), + "{:?}", + state.phase + ); + assert_eq!(state.connects, 0); + assert!(matches!(session.closed().await, Err(MoqError::Connect { .. }))); + + // The accepting side of a session reports its own lifecycle. drop(silent); let server = MoqServer::new(MoqServerConfig { bind: Some(addr.to_string()), @@ -7286,6 +7292,8 @@ async fn session_state_names_a_failing_first_dial() { }) .unwrap(); server.listen().await.unwrap(); + let client = MoqClient::new(config()).unwrap(); + let session = client.connect(format!("https://{addr}")).await.unwrap(); let request = server.accept().await.unwrap().unwrap(); let accepted = request.accept(None, None).await.unwrap(); let connected = kio::clock::timeout( diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index c0a14c0e104..dc738594463 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -618,6 +618,12 @@ impl Connect { /// The started session, or the error the handshake failed with. fn poll(&mut self, client: &Client, waiter: &kio::Waiter) -> Poll> { let res = ready!(self.poll_step(client, waiter)); + // A server that shares no version with us closes the session over our SETUP, and its + // close reason can lose the race to the stream's reset, so name the likely cause here. + if let (Err(err), Self::Exchange { negotiation, .. }) = (&res, &*self) { + let offered: Vec = negotiation.supported.iter().map(ToString::to_string).collect(); + tracing::warn!(%err, ?offered, "version negotiation failed; the server may accept none of the offered versions"); + } // Release what the handshake held, even on a failure. *self = Self::Done; Poll::Ready(res) diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index ac94e7e68a6..af03a9c6cbe 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -8,6 +8,13 @@ //! predecessor serving through its handover, and drops neither on the caller's behalf //! until it reports [`Action::Done`]. //! +//! The first walk of the peer's addresses fails fast: if no address establishes a +//! session, the policy ends with [`Action::Done`] carrying that walk's error rather than +//! retrying, so a setup mistake surfaces at once instead of looping. A client started +//! before its peer is reachable therefore exits, for its supervisor or app to restart. +//! Once a session has been established, a later failure backs off and retries under +//! [`Backoff`], so restarts and redeploys still recover. +//! //! A caller whose session landed on a fallback transport says so with //! [`Reconnect::fallback`]. The machine then asks for [`Action::Probe`]s of the preferred //! transport on a slow backoff and, once one gets through, moves to it through the same @@ -45,7 +52,9 @@ const PROBE_INITIAL: Duration = Duration::from_secs(10); /// The longest wait between probes of the preferred transport, before jitter. const PROBE_MAX: Duration = Duration::from_secs(300); -/// When to redial after a failure, and when to give up. +/// When to redial after a failure, and when to give up, once a session has been established. +/// +/// Before then nothing retries: the first walk ending without a session ends the policy. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter, @@ -78,7 +87,7 @@ pub struct Backoff { pub healthy: Duration, /// How long to keep retrying before giving up. `None` (the default) retries forever, so /// an outage of any length recovers on its own; a caller that can react to a persistent - /// failure sets one. Zero still makes the first walk, and gives up when it fails. + /// failure sets one. Zero gives up at the first failure. /// /// No new walk of the peer's addresses starts once it has passed, but one under way /// finishes: each address but the last gets up to 5 s, and the last runs to @@ -367,6 +376,8 @@ pub struct Reconnect { phase: Phase, /// The failures of the walk under way. walk: Walk, + /// Whether any walk established a session. Until one has, a failed walk ends the policy. + established: bool, /// An old session kept serving after a GOAWAY so its in-flight groups finish. predecessor: Option, queue: VecDeque>, @@ -500,6 +511,7 @@ impl + From> Reco redirects: Redirects::default(), phase: Phase::Start, walk: Walk::default(), + established: false, predecessor: None, queue: VecDeque::new(), }) @@ -644,6 +656,7 @@ impl + From> Reco // Its `Retired` stays, so the caller still drops the predecessor. self.queue.retain(|action| !matches!(action, Action::Offline(_))); self.walk = Walk::default(); + self.established = true; self.phase = Phase::Live(Box::new(Live { goaway: Some(session.draining()), session, @@ -872,7 +885,10 @@ impl + From> Reco Verdict::Refused => !self.changed(presented), Verdict::Retry | Verdict::Settled => false, }; - if terminal || self.config.once { + // The first walk fails fast: a peer that never answered with a session is more + // likely misconfigured (an unknown host, an untrusted certificate, no version in + // common) than restarting, and only the caller can fix that. + if terminal || self.config.once || !self.established { return self.done(Err(err)); } self.queue.push_back(Action::Retrying(err.clone())); diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 167b730488b..2391ad949f9 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -188,6 +188,12 @@ fn config(initial: Duration, timeout: Option) -> Config { config } +/// `machine` as it stands once a session has been established, so a failed walk retries. +fn established(mut machine: Reconnect) -> Reconnect { + machine.established = true; + machine +} + /// Check each gap between consecutive sessions against the jittered delay it should /// have waited: between half of `delay` and all of it, starting at `initial` and /// doubling up to `max`. @@ -310,13 +316,37 @@ async fn a_zero_initial_redials_at_once() { } } -/// A zero window is a real bound, not "unlimited": the first walk still dials, and its -/// failure gives up at once. A peer that keeps serving is unaffected. +/// The first walk never retries: with no session yet, its failure ends the policy at once +/// with that walk's error, however long the window. Its addresses still fall back in turn. #[kio::test] -async fn a_zero_timeout_gives_up_when_the_first_walk_fails() { +async fn the_first_walk_fails_fast() { + let targets = vec![url("tcp://a:1/"), url("tcp://b:1/")]; + let peers = [ + (targets[0].clone(), Peer::Fail(Verdict::Retry)), + (targets[1].clone(), Peer::Fail(Verdict::Retry)), + ]; + let started = kio::clock::now(); + let mut machine = Reconnect::new(config(INITIAL, None), targets.clone(), started).unwrap(); + let mut run = Run::default(); + let err = drive(&mut machine, &peers, &mut run, |_| false) + .await + .unwrap() + .expect_err("the first walk failed"); + assert!(matches!(err, TestError::Dial), "{err}"); + assert_eq!(run.dials, targets, "the walk tried every address once"); + assert!(run.events.is_empty(), "retried: {:?}", run.events); + assert_eq!(kio::clock::now(), started, "gave up without waiting"); +} + +/// A zero window is a real bound, not "unlimited": a retry's failure gives up at once. A +/// peer that keeps serving is unaffected. +#[kio::test] +async fn a_zero_timeout_gives_up_at_the_first_retry() { let peer = url("tcp://peer:1/"); let started = kio::clock::now(); - let mut machine = Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap(); + let mut machine = established( + Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap(), + ); let mut run = Run::default(); let err = drive( &mut machine, @@ -328,7 +358,7 @@ async fn a_zero_timeout_gives_up_when_the_first_walk_fails() { .unwrap() .expect_err("a failed walk kept retrying"); assert!(matches!(err, TestError::Policy(Failure::Exhausted { .. })), "{err}"); - assert_eq!(run.dials, std::slice::from_ref(&peer), "the first walk dialed once"); + assert_eq!(run.dials, std::slice::from_ref(&peer), "the walk dialed once"); assert_eq!(kio::clock::now(), started, "gave up without waiting"); let mut machine = Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap(); @@ -346,7 +376,8 @@ async fn a_zero_timeout_gives_up_when_the_first_walk_fails() { fn a_late_poll_past_the_window_gives_up() { let started = Instant::now(); let config = config(INITIAL, Some(Duration::from_secs(1))); - let mut machine = Reconnect::::new(config, vec![url("tcp://peer:1/")], started).unwrap(); + let mut machine = + established(Reconnect::::new(config, vec![url("tcp://peer:1/")], started).unwrap()); let waiter = kio::Waiter::noop(); assert!(matches!(machine.poll(started, &waiter), Action::Dial(_))); machine.failed(started, TestError::Dial, Verdict::Retry); @@ -394,7 +425,7 @@ async fn only_an_address_with_a_fallback_is_bounded() { let mut config = Config::default(); config.backoff.timeout = Some(Duration::from_secs(1)); let started = kio::clock::now(); - let mut machine = Reconnect::new(config, targets.clone(), started).unwrap(); + let mut machine = established(Reconnect::new(config, targets.clone(), started).unwrap()); let mut run = Run::default(); let err = drive(&mut machine, &peers, &mut run, |_| false) @@ -422,12 +453,14 @@ async fn a_settled_answer_stops_the_walk() { (targets[0].clone(), Peer::Fail(Verdict::Settled)), (targets[1].clone(), Peer::Serve), ]; - let mut machine = Reconnect::new( - config(INITIAL, Some(Duration::from_secs(1))), - targets.clone(), - kio::clock::now(), - ) - .unwrap(); + let mut machine = established( + Reconnect::new( + config(INITIAL, Some(Duration::from_secs(1))), + targets.clone(), + kio::clock::now(), + ) + .unwrap(), + ); let mut run = Run::default(); drive(&mut machine, &peers, &mut run, |run| run.dials.len() == 2).await; @@ -448,14 +481,14 @@ async fn an_unusable_address_ends_only_a_walk_without_hope() { (targets[0].clone(), Peer::Fail(Verdict::Retry)), (targets[1].clone(), Peer::Fail(Verdict::Unusable)), ]; - let mut machine = Reconnect::new(config(INITIAL, None), targets.clone(), kio::clock::now()).unwrap(); + let mut machine = established(Reconnect::new(config(INITIAL, None), targets.clone(), kio::clock::now()).unwrap()); let mut run = Run::default(); drive(&mut machine, &peers, &mut run, |run| run.dials.len() == 4).await; assert_eq!(run.events, ["retrying"], "an unlimited window keeps retrying"); let lone = url("tcp://c:1/"); let started = kio::clock::now(); - let mut machine = Reconnect::new(config(INITIAL, None), vec![lone.clone()], started).unwrap(); + let mut machine = established(Reconnect::new(config(INITIAL, None), vec![lone.clone()], started).unwrap()); let err = drive( &mut machine, &[(lone, Peer::Fail(Verdict::Unusable))], @@ -474,9 +507,11 @@ async fn an_unusable_address_ends_only_a_walk_without_hope() { async fn a_refusal_retries_only_new_tokens() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); - let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) - .unwrap() - .with_tokens(tokens.clone()); + let mut machine = established( + Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) + .unwrap() + .with_tokens(tokens.clone()), + ); let Action::Dial(_) = next(&mut machine).await else { panic!("expected a dial") @@ -1064,8 +1099,10 @@ fn dialing_never_logs_the_credential() { machine.failed(started.saturating_add(ATTEMPT), TestError::Dial, Verdict::Retry); loop { match machine.poll(started.saturating_add(ATTEMPT), &waiter) { - Action::Retrying(err) => reported.push(err.to_string()), - Action::Wait(_) => break, + Action::Done(Err(err)) => { + reported.push(err.to_string()); + break; + } action => panic!("unexpected {action:?}"), } } diff --git a/rs/moq-net/src/setup.rs b/rs/moq-net/src/setup.rs index 2e8294880b6..a59fef9d2e8 100644 --- a/rs/moq-net/src/setup.rs +++ b/rs/moq-net/src/setup.rs @@ -206,7 +206,18 @@ impl Client { .find(|v| supported.contains(v)); match version { Some(Version::Lite(version)) => Ok(version), - _ => Err(crate::Error::Version), + _ => { + // The client only learns that the session closed, so this is the one place + // that can name both sides of the mismatch. + let offered: Vec = self + .versions + .iter() + .map(|v| Version::try_from(*v).map_or_else(|_| format!("{v:?}"), |v| v.to_string())) + .collect(); + let accepted: Vec = supported.iter().map(ToString::to_string).collect(); + tracing::warn!(?offered, ?accepted, "client shares no MoQ version with this server"); + Err(crate::Error::Version) + } } } } diff --git a/rs/moq-net/verify/reconnect/Handover.cfg b/rs/moq-net/verify/reconnect/Handover.cfg index fab349d4bbe..7b1d6902c65 100644 --- a/rs/moq-net/verify/reconnect/Handover.cfg +++ b/rs/moq-net/verify/reconnect/Handover.cfg @@ -16,3 +16,4 @@ INVARIANTS StatusTruthful NoOrphan PresenceBalanced + FirstWalkFailsFast diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 0d6745e9514..7a299d55e14 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -19,7 +19,8 @@ (* Scope: one target, redirects between endpoints the policy accepts plus *) (* one it refuses, no retry window, no tokens, not connecting once, and no *) (* redirect-loop scoring. A session's age stands in for its tenure: 1 is *) -(* past `Backoff::healthy`. *) +(* past `Backoff::healthy`. Until a session has been established, a failed *) +(* dial ends the policy (`Reconnect::established`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -158,6 +159,12 @@ CallerSame == UNCHANGED callerVars (* The policy: one pass of `Reconnect::poll`'s loop. *) (***************************************************************************) +\* `Reconnect::established`. A dial that connects once the policy stopped +\* waiting on it still counts toward `used`, but that happens only after an +\* `Abandon`, which ends the policy until a session has been established. So +\* wherever the policy reads it, any session connected means one was established. +Established == used > 0 + \* `Reconnect::done`: the live session and the predecessor go with the policy. Done == /\ phase' = "done" @@ -228,11 +235,17 @@ Abandon == /\ ~PredDue /\ phase = "dialing" /\ timer["dial"] - /\ phase' = "backoff" - /\ timer' = [timer EXCEPT !["backoff"] = FALSE] /\ dial' = "none" /\ probing' = FALSE - /\ UNCHANGED <> + \* The first walk fails fast: `Action::Abandon`, then the `Done` it queued. + ELSE /\ Done + /\ queue' = <<"Done">> + /\ UNCHANGED timer + /\ UNCHANGED <> BackoffEnds == @@ -358,13 +371,15 @@ Connected == ELSE UNCHANGED <> /\ UNCHANGED <> -\* A dial failed, with a verdict a retry may repair or one it cannot. +\* A dial failed, with a verdict a retry may repair or one it cannot. Until a +\* session has been established, any failure ends the policy. Failed == /\ dial # "none" /\ dial' = "none" /\ IF phase # "dialing" THEN UNCHANGED <> - ELSE \/ /\ phase' = "backoff" + ELSE \/ /\ Established + /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] /\ UNCHANGED <> \/ /\ Done @@ -530,6 +545,10 @@ NoOrphan == PresenceBalanced == Parked => started - ended = IF lc = "live" THEN 1 ELSE 0 +\* The first walk fails fast: nothing backs off before a session has been +\* established, so a setup mistake ends the policy instead of looping. +FirstWalkFailsFast == phase = "backoff" => Established + \* A dial never goes elsewhere than where a session the policy still holds \* was redirected and the policy never acted on it: the peer is leaving, and \* the replacement lands back on the node that is going away. diff --git a/rs/moq-net/verify/reconnect/RestartWait.cfg b/rs/moq-net/verify/reconnect/RestartWait.cfg index efec725f99c..4b02f1049bd 100644 --- a/rs/moq-net/verify/reconnect/RestartWait.cfg +++ b/rs/moq-net/verify/reconnect/RestartWait.cfg @@ -15,3 +15,4 @@ INVARIANTS StatusTruthful NoOrphan PresenceBalanced + FirstWalkFailsFast diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 6e8bb8a79d5..b93675811a0 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -859,7 +859,8 @@ pub struct Snapshot { /// /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) -/// redials with exponential backoff. The read surface mirrors [`moq_net::Session`] +/// redials with exponential backoff. A first dial that reaches no session never retries: +/// the loop stops with its error, so a setup mistake surfaces at once. The read surface mirrors [`moq_net::Session`] /// so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), /// and [`recv_bandwidth`](Self::recv_bandwidth) track the live session and reset while disconnected. diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 1c6f74ba093..f27aabce50e 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -88,7 +88,9 @@ public typealias Compression = MoqFFI.MoqCompression public typealias ConnectionStats = MoqFFI.MoqConnectionStats /// Retry pacing for the automatic reconnect; see `Client(backoff:)`. A `nil` -/// field keeps its default. +/// field keeps its default. It applies only once a session has been established: +/// a first dial that reaches no session ends the connection with its error rather +/// than retrying. public typealias Backoff = MoqFFI.MoqBackoff /// Certificate trust and the mTLS identity for a `Client`. diff --git a/swift/Sources/Moq/Client.swift b/swift/Sources/Moq/Client.swift index 218e49d3645..69ee1ed63a7 100644 --- a/swift/Sources/Moq/Client.swift +++ b/swift/Sources/Moq/Client.swift @@ -65,6 +65,8 @@ public final class Client: Sendable { } /// Start a connection and return its session without waiting for the native first dial to succeed. + /// If that first dial reaches no session, the session closes with its error rather than retrying; + /// only an established session reconnects. public func connect(to url: String) async throws -> Session { Session(try await ffi.connect(url: url)) } From 8633e61a016c857bcd0cd18e7fa8bbe632b9fbff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:17:03 -0800 Subject: [PATCH 3/8] wip: wrapper tests and bindings --- c/moq/test/probe.c | 18 ++++++------------ dart/moq_ffi/lib/src/moq.dart | 4 +--- dart/moq_ffi/lib/src/uniffi_runtime.dart | 2 -- rs/moq-c/src/api.rs | 8 +++++--- rs/moq-c/src/test.rs | 13 +++++-------- rs/moq-net/src/reconnect/tests.rs | 16 +++++----------- 6 files changed, 22 insertions(+), 39 deletions(-) diff --git a/c/moq/test/probe.c b/c/moq/test/probe.c index e878a1db513..a151b7ad0f7 100644 --- a/c/moq/test/probe.c +++ b/c/moq/test/probe.c @@ -549,9 +549,8 @@ int main(void) moq_route_free(route); moq_origin_producer_free(origin); - /* A first dial that keeps failing still returns the session, whose state names each failed - * attempt while it retries, then ends closed once cancelled; the server is gone, so every - * dial fails. */ + /* A first dial that fails still returns the session, whose state ends closed with the + * dial's error rather than retrying; the server is gone, so the dial fails. */ moq_client_config retry_config = moq_client_config_default(); retry_config.timeout.has_value = true; retry_config.timeout.value = 100000000; /* 100ms */ @@ -561,20 +560,15 @@ int main(void) ok(moq_client_new(&retry_config, &retrying), "client new"); result dialing = {0}; task = moq_client_connect(retrying, url, on_session, &dialing); - moq_session *redialing = await(&dialing, "retrying connect"); + moq_session *redialing = await(&dialing, "failing connect"); moq_task_free(task); moq_connection_state *state = moq_session_state(redialing); - CHECK(state->phase.tag == MOQ_CONNECTION_PHASE_CONNECTING); - while (!state->phase.connecting.failure) { - state = next_state(redialing, state); - CHECK(state->phase.tag == MOQ_CONNECTION_PHASE_CONNECTING); - } - CHECK(state->phase.connecting.failure->tag == MOQ_ERROR_CONNECT); - CHECK(state->connects == 0); - moq_session_cancel(redialing, 0); while (state->phase.tag != MOQ_CONNECTION_PHASE_CLOSED) { + CHECK(state->phase.tag == MOQ_CONNECTION_PHASE_CONNECTING); state = next_state(redialing, state); } + CHECK(state->phase.closed.failure); + CHECK(state->phase.closed.failure->tag == MOQ_ERROR_CONNECT); CHECK(state->connects == 0); moq_connection_state_free(state); moq_session_free(redialing); diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index f82355639b5..c5d8ac6f818 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,9 +8,7 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; - import "package:ffi/ffi.dart"; - import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -25662,7 +25660,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqclient_cancel() != 29949) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 11393) { + if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 26263) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqsession_allocator() != 12118) { diff --git a/dart/moq_ffi/lib/src/uniffi_runtime.dart b/dart/moq_ffi/lib/src/uniffi_runtime.dart index 66b9e185973..3b7dd9cf392 100644 --- a/dart/moq_ffi/lib/src/uniffi_runtime.dart +++ b/dart/moq_ffi/lib/src/uniffi_runtime.dart @@ -8,7 +8,6 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; - import "package:ffi/ffi.dart"; class UniffiInternalError implements Exception { @@ -253,7 +252,6 @@ mixin FfiConverterPrimitive on FfiConverter { @override T lower(T value) => value; } - Uint8List createUint8ListFromInt(int value) { int length = value.bitLength ~/ 8 + 1; if (length != 4 && length != 8) { diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index d9d3a550257..02d5924d69c 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1744,8 +1744,9 @@ unsafe fn connect_session( /// Returns a non-zero handle to the session on success, or a negative code on (immediate) failure. /// You should call [moq_session_close], even on error, to free up resources. /// -/// The session reconnects automatically with exponential backoff if the connection drops. -/// Published broadcasts are re-announced and consumers re-subscribed on each reconnect, +/// Once connected, the session reconnects automatically with exponential backoff if the +/// connection drops. A first dial that reaches no session ends it instead, with that dial's +/// error as the terminal snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on each reconnect, /// since the origins outlive the underlying connection. /// /// `on_state` receives a [moq_connection_state] whenever the lifecycle changes: each @@ -1753,7 +1754,8 @@ unsafe fn connect_session( /// connect count. Changes that land together arrive as one snapshot. /// /// The last call has phase `MOQ_CONNECTION_PHASE_CLOSED`, with `data.closed.failure` zero after -/// [moq_session_close] or the negative code that made reconnecting give up. By then the +/// [moq_session_close], or the negative code of the failed first dial or of the failure that +/// made reconnecting give up. By then the /// session handle is already released, and `on_state` and `user_data` are never touched /// again, so that terminal snapshot is the point to release `user_data`. It fires even /// after [moq_session_close], so do not free `user_data` on the close call itself. diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 3677f8df996..d5af1140f87 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -7285,7 +7285,7 @@ fn session_connect_and_close() { ); } -/// Every failed dial reaches `on_state` while the loop retries, before any connect. +/// A failing first dial reaches `on_state` as the terminal snapshot, naming its failure. #[test] fn a_failing_dial_reports_its_failure() { // Hang up on every connection, so a WebSocket dial fails fast. Holding the listener @@ -7311,8 +7311,8 @@ fn a_failing_dial_reports_its_failure() { ) }); - let failed = cb.recv(); - assert_eq!(failed.phase, moq_connection_phase::MOQ_CONNECTION_PHASE_CONNECTING); + // The first dial never retries, so its failure is the terminal snapshot. + let failed = cb.recv_terminal(); assert_eq!(failed.connects, 0); assert!(failed.failure < 0, "the refused dial reports why: {failed:?}"); assert!( @@ -7320,11 +7320,8 @@ fn a_failing_dial_reports_its_failure() { "moq_error describes the failure during the callback" ); assert!(failed.version.is_empty(), "nothing is serving"); - - assert_eq!(moq_session_close(session), 0); - let terminal = cb.recv_terminal(); - assert_eq!(terminal.failure, 0, "a requested close is clean: {terminal:?}"); - assert_eq!(terminal.connects, 0); + // The terminal snapshot already released the handle. + assert!(moq_session_close(session) < 0); } /// Borrow a `&str` as the `moq_string` the list setters take. diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 2391ad949f9..c0a988aba47 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -344,9 +344,8 @@ async fn the_first_walk_fails_fast() { async fn a_zero_timeout_gives_up_at_the_first_retry() { let peer = url("tcp://peer:1/"); let started = kio::clock::now(); - let mut machine = established( - Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap(), - ); + let mut machine = + established(Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap()); let mut run = Run::default(); let err = drive( &mut machine, @@ -1097,14 +1096,9 @@ fn dialing_never_logs_the_credential() { Action::Dial(_) )); machine.failed(started.saturating_add(ATTEMPT), TestError::Dial, Verdict::Retry); - loop { - match machine.poll(started.saturating_add(ATTEMPT), &waiter) { - Action::Done(Err(err)) => { - reported.push(err.to_string()); - break; - } - action => panic!("unexpected {action:?}"), - } + match machine.poll(started.saturating_add(ATTEMPT), &waiter) { + Action::Done(Err(err)) => reported.push(err.to_string()), + action => panic!("unexpected {action:?}"), } }); From 4c5997fadd78bb48069b0e97df5aef58f77e720b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:27:52 -0800 Subject: [PATCH 4/8] wip: gst catalog test dial hangs --- rs/moq-gst/src/sink/imp.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index ead8e2a4118..1ef064ee545 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -1333,15 +1333,19 @@ mod tests { .build() } - struct CatalogRun(super::super::MoqSink); + /// A sink whose dial hangs for the whole test: a first dial that failed would end the + /// session, and the broadcast with it. + struct CatalogRun(super::super::MoqSink, std::net::TcpListener); impl CatalogRun { fn new() -> Self { gst::init().unwrap(); + // The kernel completes the TCP handshake, but nothing ever answers the session's. + let silent = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let sink = sink(); - sink.set_property("url", "tcp://127.0.0.1:1"); + sink.set_property("url", format!("tcp://{}", silent.local_addr().unwrap())); sink.set_property("broadcast", "catalog-test"); - Self(sink) + Self(sink, silent) } fn pad(&self, name: &str) -> gst::Pad { From 27650e5e41bd8bafc3c59b867d381b1b0955b693 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 19:00:57 -0800 Subject: [PATCH 5/8] wip: gst first connect errors --- doc/bin/gstreamer.md | 8 +++++--- rs/moq-gst/src/sink/imp.rs | 5 ++++- rs/moq-gst/tests/element.rs | 39 ++++++++++++------------------------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 4d89b85551b..42a34867408 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -91,8 +91,9 @@ codec config, a pad requested again, or a stream resumed after EOS publishes as `url`, `broadcast`, `linger`, `connect-version`, `tls-disable-verify`, `quic-idle-timeout`, `quic-keylog`, `quic-keep-alive`, and read-only `status`, `connected`, `moq-version`, and `estimated-send-rate`, `estimated-recv-rate`, `connection-stats`, and -`sessions`. The sink reconnects for as long as the pipeline runs and only -reports `failed` on an answer redialing can't change, such as a rejected token. +`sessions`. Once connected, the sink reconnects for as long as the pipeline runs and only +reports `failed` on an answer redialing can't change, such as a rejected token. A first +connect that reaches no relay reports `failed` and posts an error instead of retrying. Both elements accept the same connection properties. `connect-version` is a comma-separated list, matching the CLI's repeatable `--connect-version`; unknown versions fail startup. QUIC durations are milliseconds. Zero disables the local idle timeout; a zero @@ -168,7 +169,8 @@ A rendition that leaves the catalog gets EOS once its track ends. A broadcast that ends or loses its publisher holds its pads without EOS until it is announced again, so the pipeline does not end on its own. Losing the relay connection also holds the pads: `moqsrc` redials -without a retry deadline and resumes when the broadcast returns. A refused +without a retry deadline and resumes when the broadcast returns. A first connect +that reaches no relay posts an error rather than retrying. A refused connection, refused path or catalog, or malformed protocol input or catalog posts an error. Properties: `url`, `broadcast`, `max-delay`, `connect-version` (comma-separated protocol versions to offer, most diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index 1ef064ee545..ed600b1e40e 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -1335,7 +1335,10 @@ mod tests { /// A sink whose dial hangs for the whole test: a first dial that failed would end the /// session, and the broadcast with it. - struct CatalogRun(super::super::MoqSink, std::net::TcpListener); + struct CatalogRun( + super::super::MoqSink, + #[allow(dead_code, reason = "held so the dial hangs")] std::net::TcpListener, + ); impl CatalogRun { fn new() -> Self { diff --git a/rs/moq-gst/tests/element.rs b/rs/moq-gst/tests/element.rs index b8c1258c621..6effea0fedb 100644 --- a/rs/moq-gst/tests/element.rs +++ b/rs/moq-gst/tests/element.rs @@ -1502,14 +1502,12 @@ fn missing_url_fails_state_change() { let _ = sink.set_state(gst::State::Null); } -// A connect that cannot succeed does NOT post a fatal ERROR: the sink reconnects with backoff -// (issue #2212), so an unattended publisher survives a relay that is unreachable at startup or -// during an outage instead of tearing down the pipeline. It keeps retrying and stays disconnected. -// (A non-retryable failure, e.g. auth rejection, is still terminal; that path needs a live relay -// and is covered separately.) The "relay" accepts each dial and hangs up at once, so the test -// counts the retries rather than waiting out a window. +// A first connect that reaches no session posts a fatal ERROR instead of retrying, so a +// misconfigured relay URL, certificate, or version surfaces at once; a supervisor restarts the +// pipeline once the relay is reachable. Reconnecting with backoff is only for a session that +// once worked. The "relay" accepts each dial and hangs up at once. #[test] -fn connect_failure_retries_without_erroring() { +fn a_failed_first_connect_errors_without_retrying() { init(); let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); let addr = listener.local_addr().expect("local address"); @@ -1535,33 +1533,20 @@ fn connect_failure_retries_without_erroring() { pipeline.set_state(gst::State::Playing).is_ok(), "a valid url + broadcast must let the Ready->Playing change start (connect runs in the background)" ); - // The first dial and a retry after it failed. - for _ in 0..2 { - dials - .recv_timeout(std::time::Duration::from_secs(30)) - .expect("the sink stopped dialing"); - } let bus = pipeline.bus().expect("pipeline bus"); - let msg = bus.pop_filtered(&[gst::MessageType::Error]); + let msg = bus.timed_pop_filtered(gst::ClockTime::from_seconds(30), &[gst::MessageType::Error]); let connected = sink.property::("connected"); let status = sink.property::("status"); - let send_bitrate = sink.property::("estimated-send-rate"); - let recv_bitrate = sink.property::("estimated-recv-rate"); let _ = pipeline.set_state(gst::State::Null); + assert!(msg.is_some(), "a failed first connect must post an ERROR"); + assert!(!connected); + assert_eq!(status, gstmoq::ConnectionStatus::Failed); + dials.recv().expect("the sink dialed"); assert!( - msg.is_none(), - "a failed connect must NOT post an ERROR: the sink retries (issue #2212)" - ); - assert!( - !connected, - "a failed connect must leave connected = false while retrying" + dials.try_recv().is_err(), + "the sink redialed after its first connect failed" ); - // While retrying, status stays Disconnected (a transient retry, not the terminal Failed) and the - // bitrate estimates read 0. - assert_eq!(status, gstmoq::ConnectionStatus::Disconnected); - assert_eq!(send_bitrate, 0); - assert_eq!(recv_bitrate, 0); } // Both element factories expose the same connection syntax, and refuse unknown versions From ec2ea04769010b109408d759a3255a6a9b832932 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 19:18:03 -0800 Subject: [PATCH 6/8] wip: moq-tokio tests establish first --- rs/moq-tokio/src/connection.rs | 87 ++++++++++++++------ rs/moq-tokio/tests/integration/fresh_port.rs | 66 +++++++++------ 2 files changed, 101 insertions(+), 52 deletions(-) diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index b93675811a0..3a7c7fdaa46 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -47,7 +47,8 @@ fn verdict(err: &Error) -> Verdict { /// Three knobs, each with one job: `initial` and `multiplier` set the wait before /// each retry, `max` caps it, and `healthy` decides which sessions count as failed /// attempts. See [`moq_net::reconnect::Backoff`], which this resolves to, for how -/// the policy applies them. +/// the policy applies them. It applies only once a session has been established: a first +/// dial that reaches no session ends the connection instead of retrying. #[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -860,8 +861,8 @@ pub struct Snapshot { /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) /// redials with exponential backoff. A first dial that reaches no session never retries: -/// the loop stops with its error, so a setup mistake surfaces at once. The read surface mirrors [`moq_net::Session`] -/// so a caller can treat it like a session that transparently reconnects: +/// the loop stops with its error, so a setup mistake surfaces at once. The read surface +/// mirrors [`moq_net::Session`] so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), /// and [`recv_bandwidth`](Self::recv_bandwidth) track the live session and reset while disconnected. /// The extra toggle a plain session doesn't have is the connection lifecycle: [`established`](Self::established) @@ -2084,10 +2085,12 @@ mod tests { } } - /// A refused socket still spends the retry budget rather than stopping on its first error. + /// A refused first dial ends the loop with its error at once, spending none of an + /// unlimited retry window: a peer that never answered is more likely a setup mistake + /// than a restart. #[cfg(feature = "tcp")] #[tokio::test(start_paused = true)] - async fn a_refused_tcp_dial_keeps_retrying() { + async fn a_refused_first_dial_fails_at_once() { // Refusal is what's under test, so this can't use [`dead`]. Only Linux // refuses a SYN to a port that is bound but not listening, so only Linux // holds it: a released port can become another process's listener mid-test. @@ -2108,9 +2111,9 @@ mod tests { .to_string(); let mut config = crate::connect::Config::default(); - config.backoff.initial = Some((Duration::from_millis(50)).into()); - config.backoff.max = Some((Duration::from_millis(50)).into()); - config.backoff.timeout = Some((Duration::from_millis(100)).into()); + config.backoff.initial = Some((FAST).into()); + // Zero retries forever, so only the first walk failing fast ends this loop. + config.backoff.timeout = Some((Duration::ZERO).into()); // No timer may be pending while a dial is in flight, since the paused clock // jumps to it whenever the runtime idles. Linux and macOS refuse a loopback // connect inside the call, but Windows reports it later, after retrying the @@ -2122,17 +2125,37 @@ mod tests { let started = kio::clock::now(); let connection = client.connect(url(&format!("tcp://{addr}/"))); let err = connection.closed().await.expect_err("refused socket connected"); - assert!(matches!(err, Error::Reconnect(_)), "{err}"); assert!(err.to_string().contains(&refused), "{err}"); - assert!(started.elapsed() >= Duration::from_millis(100)); + // A retry would first wait out a jittered share of `initial`. + assert_eq!(started.elapsed(), Duration::ZERO, "the first walk backed off"); } - /// A redial that keeps failing reports why while it retries, and the connect that - /// finally lands clears it. Without this a caller only learns the reason once the - /// loop gives up, which with no retry window is never. + /// A connection to `host` that had a session and now redials a [`dead`] address. + /// + /// A first walk that fails ends the loop rather than retrying, so a test of the retry + /// loop starts from a session: this serves one, repoints `host` at a dead address, and + /// ends the session from the server. Returns the connection, already reporting + /// [`Status::Disconnected`], the live server's address and its later sessions, and the + /// server task. #[cfg(feature = "tcp")] - #[tokio::test] - async fn a_failing_redial_reports_why_until_it_connects() { + async fn redialing( + host: &str, + config: crate::connect::Config, + ) -> ( + Connection, + std::net::SocketAddr, + tokio::sync::mpsc::UnboundedReceiver, + tokio::task::JoinHandle<()>, + ) { + let (live, mut accepted, task) = serve(crate::origin::spawn()).await; + crate::resolve::hosts::point(host, [live]); + let mut connection = config + .init(Default::default()) + .unwrap() + .connect(url(&format!("tcp://{host}:1/"))); + assert_eq!(connection.status().await.unwrap(), Status::Connected); + let session = accepted.recv().await.unwrap(); + // Hold the failing listener until the runtime ends: a released port can // become another process's listener while the connection keeps retrying. let failing = dead().await.socket_addrs(|| None).unwrap(); @@ -2140,24 +2163,33 @@ mod tests { std::net::TcpListener::bind(failing[0]).is_err(), "the failing port stays reserved" ); - let (live, mut accepted, _task) = serve(crate::origin::spawn()).await; + crate::resolve::hosts::point(host, failing); + drop(session); + assert_eq!(connection.status().await.unwrap(), Status::Disconnected); + (connection, live, accepted, task) + } + /// A redial that keeps failing reports why while it retries, and the connect that + /// finally lands clears it. Without this a caller only learns the reason once the + /// loop gives up, which with no retry window is never. + #[cfg(feature = "tcp")] + #[tokio::test] + async fn a_failing_redial_reports_why_until_it_connects() { // Unique to this test: the table is process-wide. const HOST: &str = "failing-redial.test"; - crate::resolve::hosts::point(HOST, failing); let mut config = crate::connect::Config::default(); config.backoff.initial = Some((FAST).into()); config.backoff.max = Some((FAST).into()); config.backoff.timeout = Some((Duration::ZERO).into()); - let client = config.init(Default::default()).unwrap(); - let mut connection = client.connect(url(&format!("tcp://{HOST}:1/"))); + let (mut connection, live, mut accepted, _task) = redialing(HOST, config).await; let monitor = connection.monitor(); + // The session's end may be reported first; wait for a failed redial's reason. let failure = kio::wait(|waiter| { monitor.state.poll(waiter, |state| match state.lifecycle.failure() { - Some(err) => Poll::Ready(err.clone()), - None => Poll::Pending, + Some(err @ Error::Tcp(_)) => Poll::Ready(err.clone()), + _ => Poll::Pending, }) }) .await @@ -2186,14 +2218,14 @@ mod tests { #[cfg(feature = "tcp")] #[tokio::test] async fn closing_between_retries_clears_the_failure() { - // A held port, not a released one: another test's server could claim a - // released port, and the dial would connect instead of failing. + // Unique to this test: the table is process-wide. + const HOST: &str = "closing-between-retries.test"; + let mut config = crate::connect::Config::default(); config.backoff.initial = Some((FAST).into()); config.backoff.max = Some((FAST).into()); config.backoff.timeout = Some((Duration::ZERO).into()); - let client = config.init(Default::default()).unwrap(); - let connection = client.connect(dead().await); + let (connection, _live, _accepted, _task) = redialing(HOST, config).await; let mut monitor = connection.monitor(); kio::wait(|waiter| { @@ -2204,11 +2236,14 @@ mod tests { }) .await .unwrap_or_else(|_| panic!("the connection ended before reporting its retry reason")); + // The one session's connect and disconnect, both already counted. + let presence = monitor.presence_changed().await.unwrap(); + assert_eq!((presence.sessions_started, presence.sessions_ended), (1, 1)); connection.close().await.expect("close"); // Resolves with an error once the loop's state is final. let ended = monitor.presence_changed().await; - assert!(ended.is_err(), "nothing connected, so only the end moves presence"); + assert!(ended.is_err(), "nothing reconnected, so only the end moves presence"); assert!(monitor.failure().is_none(), "nothing is retrying"); } diff --git a/rs/moq-tokio/tests/integration/fresh_port.rs b/rs/moq-tokio/tests/integration/fresh_port.rs index 63f5ca01a78..a12929352e1 100644 --- a/rs/moq-tokio/tests/integration/fresh_port.rs +++ b/rs/moq-tokio/tests/integration/fresh_port.rs @@ -40,8 +40,8 @@ async fn peer() -> (SocketAddr, mpsc::UnboundedReceiver<(SocketAddr, moq_net::Se (addr, recv) } -/// The first source starts blocked; other sources get independent upstream sockets, -/// as in the relay's cluster_fresh_port test. Report each source only once. +/// Each source gets an independent upstream socket, as in the relay's cluster_fresh_port +/// test, and a blocked source is black-holed. Report each source only once. struct Proxy { addr: SocketAddr, sources: mpsc::UnboundedReceiver, @@ -61,9 +61,6 @@ async fn proxy(upstream: SocketAddr) -> Proxy { loop { let (len, source) = front.recv_from(&mut buf).await.unwrap(); if heard.insert(source) { - if heard.len() == 1 { - drop_sources.lock().unwrap().insert(source); - } send.send(source).unwrap(); } if drop_sources.lock().unwrap().contains(&source) { @@ -114,6 +111,18 @@ fn connect(client: &Client, addr: SocketAddr) -> Connection { client.connect(format!("moqt://{addr}/").parse::().unwrap()) } +/// Black-hole `source` and end the session it carried, waiting until `connection` sees it go. +/// +/// A first dial that fails ends the loop rather than retrying, so a test of how the loop +/// redials starts from an established session and drops its flow with this. +async fn drop_flow(proxy: &Proxy, source: SocketAddr, session: moq_net::Session, connection: &Connection) { + proxy.blocked.lock().unwrap().insert(source); + let monitor = connection.monitor(); + let revision = monitor.state().revision; + drop(session); + socket_step(monitor.state_changed(revision)).await.unwrap(); +} + /// Advance one failed dial and wait until the reconnect loop observes it. async fn fail(connection: &Connection) { let monitor = connection.monitor(); @@ -134,9 +143,11 @@ async fn a_dropped_flow_recovers_without_moving_a_healthy_connection() { let healthy = socket_step(connect(&client, peer).established()).await.unwrap(); let (healthy_addr, healthy_session) = socket_step(sessions.recv()).await.unwrap(); let mut proxy = proxy(peer).await; - let reconnect = connect(&client, proxy.addr); + let reconnect = socket_step(connect(&client, proxy.addr).established()).await.unwrap(); let first = socket_step(proxy.sources.recv()).await.unwrap(); assert_eq!(first, healthy_addr); + let (_, session) = socket_step(sessions.recv()).await.unwrap(); + drop_flow(&proxy, first, session, &reconnect).await; fail(&reconnect).await; assert!(proxy.sources.try_recv().is_err(), "one failed dial changed the port"); @@ -146,6 +157,7 @@ async fn a_dropped_flow_recovers_without_moving_a_healthy_connection() { let reconnect = socket_step(reconnect.established()).await.unwrap(); let (_, recovered_session) = socket_step(sessions.recv()).await.unwrap(); assert_eq!(healthy.connects(), 1); + assert_eq!(reconnect.connects(), 2); assert!(matches!( healthy.monitor().state().phase, moq_tokio::connection::Phase::Connected { .. } @@ -169,27 +181,25 @@ async fn a_successful_dial_resets_the_failure_count() { let (peer, mut sessions) = peer().await; let client = client("127.0.0.1:0".parse().unwrap()); let mut proxy = proxy(peer).await; - let reconnect = connect(&client, proxy.addr); + let mut reconnect = socket_step(connect(&client, proxy.addr).established()).await.unwrap(); let first = socket_step(proxy.sources.recv()).await.unwrap(); - fail(&reconnect).await; - proxy.blocked.lock().unwrap().remove(&first); - let reconnect = socket_step(reconnect.established()).await.unwrap(); - let (_, session) = socket_step(sessions.recv()).await.unwrap(); + let (_, mut session) = socket_step(sessions.recv()).await.unwrap(); - proxy.blocked.lock().unwrap().insert(first); - drop(session); - let monitor = reconnect.monitor(); - let revision = monitor.state().revision; - socket_step(monitor.state_changed(revision)).await.unwrap(); - fail(&reconnect).await; - proxy.blocked.lock().unwrap().remove(&first); - let reconnect = socket_step(reconnect.established()).await.unwrap(); - let (_, _session) = socket_step(sessions.recv()).await.unwrap(); + // One failed walk each time, so only a count the session in between failed to reset + // reaches the second failure that moves to a fresh port. + for _ in 0..2 { + drop_flow(&proxy, first, session, &reconnect).await; + fail(&reconnect).await; + proxy.blocked.lock().unwrap().remove(&first); + reconnect = socket_step(reconnect.established()).await.unwrap(); + (_, session) = socket_step(sessions.recv()).await.unwrap(); + } assert!( proxy.sources.try_recv().is_err(), "success did not reset the failed-dial count" ); - assert_eq!(reconnect.connects(), 2); + assert_eq!(reconnect.connects(), 3); + drop(session); socket_step(client.close()).await; } @@ -226,8 +236,10 @@ fn a_failed_socket_allocation_keeps_the_old_port_available_for_redial() { let (peer, mut sessions) = peer().await; let client = client("127.0.0.1:0".parse().unwrap()); let mut proxy = proxy(peer).await; - let reconnect = connect(&client, proxy.addr); + let reconnect = socket_step(connect(&client, proxy.addr).established()).await.unwrap(); let first = socket_step(proxy.sources.recv()).await.unwrap(); + let (_, session) = socket_step(sessions.recv()).await.unwrap(); + drop_flow(&proxy, first, session, &reconnect).await; let mut limit = libc::rlimit { rlim_cur: 0, @@ -247,26 +259,28 @@ fn a_failed_socket_allocation_keeps_the_old_port_available_for_redial() { let reconnect = socket_step(reconnect.established()).await.unwrap(); let (_, _session) = socket_step(sessions.recv()).await.unwrap(); assert!(proxy.sources.try_recv().is_err(), "the old socket was replaced"); - assert_eq!(reconnect.connects(), 1); + assert_eq!(reconnect.connects(), 2); socket_step(client.close()).await; }); } #[tokio::test(start_paused = true)] async fn a_fixed_bind_port_survives_consecutive_failed_dials() { - let (peer, _sessions) = peer().await; + let (peer, mut sessions) = peer().await; let reserved = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); let bind = reserved.local_addr().unwrap(); drop(reserved); let client = client(bind); let mut proxy = proxy(peer).await; - let reconnect = connect(&client, proxy.addr); + let reconnect = socket_step(connect(&client, proxy.addr).established()).await.unwrap(); assert_eq!(socket_step(proxy.sources.recv()).await.unwrap(), bind); + let (_, session) = socket_step(sessions.recv()).await.unwrap(); + drop_flow(&proxy, bind, session, &reconnect).await; for _ in 0..4 { fail(&reconnect).await; } assert!(proxy.sources.try_recv().is_err(), "a fixed bind port changed"); - assert_eq!(reconnect.connects(), 0); + assert_eq!(reconnect.connects(), 1); drop(reconnect); socket_step(client.close()).await; } From 4f4d52f902e83f0ba7dc699434e1124c4ffcee38 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 19:30:50 -0800 Subject: [PATCH 7/8] fix(net): a first walk refused for since-replaced tokens still redials --- .claude/skills/verify-reconnect/SKILL.md | 2 +- rs/moq-net/src/reconnect.rs | 14 +++++++----- rs/moq-net/src/reconnect/tests.rs | 27 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 2d3195631d3..513aad6eec2 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -18,7 +18,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"). | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | -| `FirstWalkFailsFast` | Nothing backs off before a session has been established: a failed first walk ends the policy. | A setup mistake (an untrusted certificate, no version in common) retries forever instead of surfacing. | +| `FirstWalkFailsFast` | Nothing backs off before a session has been established: a failed first walk ends the policy (a refusal of since-replaced tokens is the exception, and tokens are not modeled). | A setup mistake (an untrusted certificate, no version in common) retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index af03a9c6cbe..ec1d0a2cff8 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -10,7 +10,8 @@ //! //! The first walk of the peer's addresses fails fast: if no address establishes a //! session, the policy ends with [`Action::Done`] carrying that walk's error rather than -//! retrying, so a setup mistake surfaces at once instead of looping. A client started +//! retrying, so a setup mistake surfaces at once instead of looping. The one exception is +//! a refused credential the tokens have replaced since, which earns another dial. A client started //! before its peer is reachable therefore exits, for its supervisor or app to restart. //! Once a session has been established, a later failure backs off and retries under //! [`Backoff`], so restarts and redeploys still recover. @@ -880,15 +881,18 @@ impl + From> Reco fn walk_failed(&mut self, now: Instant, err: E, verdict: Verdict, presented: Option<&auth::Dial>) { self.walk = Walk::default(); + // Credentials replaced since the dial read them are the caller's fix, so they earn + // another dial even before a first session. + let renewed = verdict == Verdict::Refused && self.changed(presented); let terminal = match verdict { Verdict::Fatal | Verdict::Unusable => true, - Verdict::Refused => !self.changed(presented), + Verdict::Refused => !renewed, Verdict::Retry | Verdict::Settled => false, }; - // The first walk fails fast: a peer that never answered with a session is more - // likely misconfigured (an unknown host, an untrusted certificate, no version in + // Otherwise the first walk fails fast: a peer that never answered with a session is + // more likely misconfigured (an unknown host, an untrusted certificate, no version in // common) than restarting, and only the caller can fix that. - if terminal || self.config.once || !self.established { + if terminal || self.config.once || !(self.established || renewed) { return self.done(Err(err)); } self.queue.push_back(Action::Retrying(err.clone())); diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index c0a988aba47..2d55c7fabf2 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -529,6 +529,33 @@ async fn a_refusal_retries_only_new_tokens() { ); } +/// A first walk refused for tokens replaced since its dial still earns another dial: the +/// new tokens are the caller's fix. Unchanged tokens fail fast. +#[kio::test] +async fn a_first_refusal_retries_only_new_tokens() { + let peer = url("tcp://peer:1/"); + let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) + .unwrap() + .with_tokens(tokens.clone()); + + let Action::Dial(_) = next(&mut machine).await else { + panic!("expected a dial") + }; + tokens.set(vec!["new".parse().unwrap()]); + machine.failed(kio::clock::now(), TestError::Dial, Verdict::Refused); + assert!(matches!(next(&mut machine).await, Action::Retrying(_))); + + let Action::Dial(_) = next(&mut machine).await else { + panic!("expected a redial") + }; + machine.failed(kio::clock::now(), TestError::Dial, Verdict::Refused); + assert!( + matches!(next(&mut machine).await, Action::Done(Err(TestError::Dial))), + "an unchanged credential is refused for good" + ); +} + /// Ending the tokens' handle ends the policy at its next walk, rather than redialing /// sessions the handle refuses. #[kio::test] From c7200d9c2bd32639412c716ba7455c65ed201851 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 21:28:13 -0800 Subject: [PATCH 8/8] docs(ffi): note the replaced-credentials exception to first-dial fail-fast --- dart/moq_ffi/lib/src/moq.dart | 2 +- rs/moq-ffi/src/session.rs | 4 +++- rs/moq-net/src/reconnect.rs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index c5d8ac6f818..dc804d98339 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -25660,7 +25660,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqclient_cancel() != 29949) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 26263) { + if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 30086) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqsession_allocator() != 12118) { diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 15eba605e21..e5252fd622f 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -195,7 +195,8 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// /// It applies only once a session has been established: a first dial that reaches no -/// session ends the connection with its error rather than retrying. The delay starts at +/// session ends the connection with its error rather than retrying, unless the peer refused +/// credentials the tokens have replaced since, which earns another dial. The delay starts at /// `initial`, multiplies by `multiplier` after each failed attempt, and caps at `max`. /// After `timeout` of consecutive failures the connection gives up for good; the window /// resets whenever a session stays up past 1s. Each null field uses moq-tokio's default: @@ -596,6 +597,7 @@ impl MoqClient { /// consumed through it ride out the gap. The first dial fails fast instead: if none of /// the peer's addresses yields a session, [`MoqSession::closed`] resolves with that /// error, so an app started before its relay is reachable must start the session again. + /// A refusal of credentials replaced during that dial redials with the new ones instead. /// Read [`MoqSession::state`] and wait with [`MoqSession::state_changed`] for /// reconnects; [`MoqSession::closed`] waits for the terminal result. /// diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index ec1d0a2cff8..1b13b7ca336 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -55,7 +55,8 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// When to redial after a failure, and when to give up, once a session has been established. /// -/// Before then nothing retries: the first walk ending without a session ends the policy. +/// Before then nothing retries: the first walk ending without a session ends the policy, +/// unless the peer refused credentials the tokens have replaced since. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter,