From c7639ed0b538e39bb692284854a97a51d722febf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 14:46:36 -0700 Subject: [PATCH 1/9] chore(quest): claim quest/m1/hls-takeover-leak.md Quest-Claim: 36b780f3069bf436 From 3cce5e32ecbf57e880334e8a1646f5e7252069b2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 14:58:30 -0700 Subject: [PATCH 2/9] test(hls): reproduce epochless takeover establishment race Checkpoint both failing lite session reproductions and the confirmed late subscription identity race. Production behavior remains unchanged pending the identity and legacy refusal decision. Validation: just fix and quest check passed. just ci passed all selected non-Rust-test modules; Rust tests had 489 passes and the two expected takeover failures, with 7 additional cases unrun after fail-fast. Co-Authored-By: GPT-6 --- quest/m1/hls-takeover-leak.md | 267 +++++++----------------- rs/moq-hls/tests/integration/main.rs | 1 + rs/moq-hls/tests/integration/restart.rs | 145 +++++++++++++ 3 files changed, 218 insertions(+), 195 deletions(-) create mode 100644 rs/moq-hls/tests/integration/restart.rs diff --git a/quest/m1/hls-takeover-leak.md b/quest/m1/hls-takeover-leak.md index 34e728ab1b7..1fd31fd46e2 100644 --- a/quest/m1/hls-takeover-leak.md +++ b/quest/m1/hls-takeover-leak.md @@ -6,206 +6,83 @@ After a new publisher instance takes over a broadcast path, a recording cursor o replaced instance never yields the replacement's media. It drains the old run's own segments, then ends cleanly or fails with a typed error. It never stalls. -The `takeover` scenario below lands in `rs/moq-hls/tests/integration/restart.rs` with -the fix, as part of the session suite. - ## Plan -Found 2026-10-25 by the session suite. The takeover scenario fails on `main`. - -**Symptom.** The old run publishes three complete segments and leaves a GOP open at 12s. -A new instance takes over `live` and publishes its own groups from zero. The old run -finishes. The old cursor reads segments 1 and 2 with the old bytes. Segment 3, the old -run's final GOP (group 6), comes back with the new run's bytes. - -- If the new run has not completed its own group 6, the old cursor stalls on segment 3 - instead. -- The leak also happens when the old run finishes before the new one announces. -- It reproduces on moq-lite-06 (the default) and moq-lite-07-wip, with an epochless old - route and a new route with an epoch. -- Over HTTP, the old instance's URLs answer 404 once it is replaced, so only the cursor - leaks. - -**Where the fetch goes.** Temporary logging in `Rendition::fetch` -(`rs/moq-hls/src/export/rendition.rs`) showed the segment-3 fetch going through the -replaced broadcast: its epoch was `None` and `is_closed()` was true. The fetch reads -through the track that `Handle::capture` pinned with a `track.query()` before the -takeover, yet `fetch_group(6)` returned the replacement's group with the same sequence. - -**A bare moq-net reproduction did not leak.** The setup used lite-07, an epochless old -broadcast, and a held `track.query()` taken before the replacement, with: - -- a subscription on the old track, with its in-flight group either read or dropped unread; -- the replacement announced with an epoch and writing groups 0 to 2; -- the old group 1 finished after the takeover; -- in one variant, a concurrent subscription to the replacement. - -`fetch_group(1)` on the old query returned the old bytes. Groups the old subscription -never delivered answered `Unroutable`. So the leak needs something the export adds. - -**Where I looked:** - -- `rs/moq-hls/src/export/rendition.rs`: `Media`, `Handle::capture`, `poll_sync`, - `poll_track`, `Rendition::fetch`, and the free `fetch`. Pinning and window-ending on - close apply only to siblings (`self.sibling`). The catalog's own broadcast relies on - the source pin. -- `rs/moq-hls/src/export/segments.rs`: `Consumer::poll_feed`. The cursor's feed - subscribes through the binding (`poll_subscribe`, `FEED_MAX_DELAY`) and drops each - group unread. Its doc says a resumed subscription "only pulls while it is read". -- `rs/moq-mux/src/source.rs`: `Source::new` pins the catalog path to the served - instance; `Binding::poll_broadcast` returns the pin. -- `rs/moq-net/tests/integration/restart.rs`: `retired_fetch` covers a retired handle - whose query is taken after the restart, not one held before it. -- `rs/moq-net/src/lite/subscriber.rs`: `SubscriptionCleanup` tears subscriptions down in - `HashMap` order. That explains why the disconnect failure order varies, not this leak. - -**Leading hypothesis, unconfirmed:** the feed subscription, or a resubscribe from the -pinned query, resumes across the route change onto the replacement and fills the -retired track's cache for a group the old run had not delivered yet. Confirm by logging -which subscription delivers group 6 into the old track, then decide whether moq-net -refuses the resume or the export stops reading a replaced instance's tail. - -The takeover scenario uses the suite's `support` harness and needs a `contains` helper, -which was dropped from `support.rs` when the scenario moved here: - -```rust -//! A new publisher instance taking over a path is served under its own URLs, while the run it -//! replaced finishes its own recording and never serves the new run's bytes. - -use std::time::Duration; - -use axum::http::StatusCode; - -use crate::support::*; - -const OLD: u8 = 1; -const NEW: u8 = 2; - -/// The old run keeps publishing through the takeover and ends after it. Until a replaced run's -/// drain has a cutoff of its own, its cursor ends only once the old publisher does. -#[tokio::test(start_paused = true)] -async fn takeover() { - let pair = Pair::connect().await; - let mut old = Publisher::new(&pair.publisher, "live", Default::default()); - let server = pair.server(); - let app = server.router(); - let rendition = rendition(&server, "live").await; - let mut cursor = rendition.segments(); - - old.write(0, 13, OLD); - let old_media = media_uri(&app, "live").await; - listed(&app, &old_media, ".2.m4s").await; - let replaced = server.broadcaster("live").await.expect("announced"); - replaced.set_generation(Some("run-1")).expect("generation"); - let playlist = listed(&app, &old_media, "seg/run-1.").await; - let old_segments: Vec<_> = (0..3).map(|index| segment_uri(&old_media, &playlist, index)).collect(); - for uri in &old_segments { - assert_old(&app, uri).await; - } - // Reading, as a recorder does, so the cursor's subscription carries every group across. - let first = moq_clock::timeout(TIMEOUT, cursor.next()) - .await - .expect("the first segment") - .expect("records") - .expect("a segment"); - assert_eq!(first.segment, 0); - assert!(contains(&first.media, &[OLD; 8])); - - // The new instance numbers its groups and segments from zero again. - let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); - let mut new = Publisher::new(&pair.publisher, "live", route); - let deadline = moq_clock::now() + TIMEOUT; - let current = loop { - let current = server.broadcaster("live").await.expect("announced"); - if !std::sync::Arc::ptr_eq(¤t, &replaced) { - break current; - } - assert!(moq_clock::now() < deadline, "the new instance never took over"); - moq_clock::sleep(Duration::from_millis(50)).await; - }; - current.set_generation(Some("run-2")).expect("generation"); - new.write(0, 13, NEW); - - // The master lists the new instance, whose playlist lists its own generation. - let new_media = media_uri(&app, "live").await; - assert_ne!(new_media, old_media, "a new instance gets its own URLs"); - let playlist = listed(&app, &new_media, ".2.m4s").await; - assert!(!playlist.contains("run-1."), "{playlist}"); - let new_segment = segment_uri(&new_media, &playlist, 0); - assert!(new_segment.contains("/seg/run-2."), "{new_segment}"); - let response = get(&app, &new_segment).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(contains(response.body(), &[NEW; 8]) && !contains(response.body(), &[OLD; 8])); - // Nor does the new instance answer the old run's generation. - let crossed = new_segment.replace("/seg/run-2.", "/seg/run-1."); - assert_eq!(get(&app, &crossed).await.status(), StatusCode::NOT_FOUND); - - // The old run's URLs, under either generation, never answer with the new run's bytes. - for uri in &old_segments { - assert_old(&app, uri).await; - assert_old(&app, &uri.replace("/seg/run-1.", "/seg/run-2.")).await; - } - - // The old run ends while the new one carries on past the old run's last group. - old.finish(); - new.write(13, 21, NEW); - drop(old); - - // The old recording drains every segment of its own, its final GOP included, and nothing else. - let mut read = Vec::new(); - loop { - let next = moq_clock::timeout(TIMEOUT, cursor.next()) - .await - .expect("the replaced run's cursor never ended"); - let Some(segment) = next.expect("the replaced run finished cleanly") else { - break; - }; - assert!( - contains(&segment.media, &[OLD; 8]) && !contains(&segment.media, &[NEW; 8]), - "the replaced run's segment {} carries the new run's media", - segment.segment - ); - read.push(segment.segment); - } - assert_eq!(read, [1, 2, 3]); - let playlist = rendition.media_playlist(None).expect("finished").expect("listed"); - assert!(playlist.ends_with("#EXT-X-ENDLIST\n"), "{playlist}"); - - for uri in &old_segments { - assert_old(&app, uri).await; - } - assert_eq!(get(&app, &new_segment).await.status(), StatusCode::OK); - drop(new); -} - -/// `uri` answers the old run's bytes or nothing at all. -async fn assert_old(app: &axum::Router, uri: &str) { - let response = get(app, uri).await; - assert!(!contains(response.body(), &[NEW; 8]), "{uri} served the new run"); - match response.status() { - StatusCode::OK => assert!(contains(response.body(), &[OLD; 8]), "{uri}"), - _ => assert_missing(&response), - } -} - -/// The URI of the `index`th segment `playlist` lists, beside the media playlist at `media`. -fn segment_uri(media: &str, playlist: &str, index: usize) -> String { - let line = playlist - .lines() - .filter(|line| line.starts_with("seg/")) - .nth(index) - .unwrap_or_else(|| panic!("no segment {index} in {playlist}")); - media.replace("media.m3u8", line) -} - -/// Whether `haystack` contains `needle`. -fn contains(haystack: &[u8], needle: &[u8]) -> bool { - haystack.windows(needle.len()).any(|window| window == needle) -} +The failing session reproduction is checked in at +`rs/moq-hls/tests/integration/restart.rs`. Both `takeover_lite06` and +`takeover_lite07` fail on the replaced run's final segment carrying the new run's +payload. The reproduction uses the session suite's paused-clock `support` harness. +These are active failing tests in an unfinished draft, not a completed fix. + +### Confirmed cause + +The old run publishes three complete segments and leaves group 6 open at 12s. Its +recording cursor returns segment 0 from FETCHed groups while its live media +subscription is only queued. Returning that segment does not mean the upstream +SUBSCRIBE has opened or resolved. The new publisher then takes over the path. + +Temporary tracing produced this order for the old media subscription: + +```text +TAKEOVER LOCAL +SUB PREP live video0 id 2 +SUB ACTIVE live video0 id 2 +PUB DECODE subscribed live video0 +SELECTION live end +PUB RESOLVE subscribed live video0 ``` +`TrackServe::prepare_establish` registers id 2 against the old source's track. +`Establish::poll` checks that source's request validity, then on lite-05+ considers +the subscription active as soon as its request flushes. At this point the subscriber +has not yet received the old announcement's retirement, so its check succeeds. +The publisher's `RequestServe::poll_serve` resolves that epochless request through +`origin.request(path, None)`, which selects the replacement. The existing id 2 now +feeds new media into the old source's cache. No wire FETCH for group 6 is emitted: +`Rendition::fetch` finds that poisoned cached group and serves the new bytes under +the old row. The final segment contains two NEW-tagged frames, rather than the +old run's one OLD-tagged final frame. + +The source selection does retire, and the rendition's source and queried track +remain pinned. This is an establishment identity race, not a resume that crosses +two known publisher epochs. The earlier bare moq-net attempt already established +its subscription before takeover, so it did not exercise this ordering. + +Eagerly polling the HLS feed, or waiting for SUBSCRIBE_START before calling a +subscription active, changes this schedule but cannot prove which instance an +epochless request meant. Announcement and subscription streams are independently +ordered. The publisher may resolve the replacement and return its START before +the subscriber observes the old announcement's retirement. FETCH has the same +path-only addressing ambiguity. Neither a retry nor a new drain timeout fixes +that identity gap. + +### Decision needed + +Choose the identity and legacy behavior before implementing the fix: + +1. **Require addressable instance identity for safe tail reads (recommended).** + Use the intended publisher's epoch wherever the wire can name it, and define + a typed refusal for ambiguous epochless or legacy operations. This preserves + the no-cross-instance guarantee but may make a retired recording fail rather + than recover its last uncached GOP. Decide where the identity requirement is + enforced and which existing/public error represents the refusal. +2. **Keep full epochless tail recovery.** Design a wire association tying TRACK, + SUBSCRIBE, and FETCH to the same publisher instance, rather than resolving + each request's path independently. The unpublished wire can change; published + wires still need an explicit safe refusal policy. This is wider API/wire scope + and must include the JS implementation and draft. + +The separate drain-cutoff quest owns how long an already established old stream +may drain. Do not choose that policy or make a scheduling-only workaround here. + +After the decision, implement at the identity boundary and extend the session +coverage to an unfinished replacement group, the old run finishing before the +replacement announces, and established subscriptions that legitimately drain OLD +media. A missing old group may fail with the agreed typed error; it must neither +read NEW media nor wait for the replacement to finish a same-numbered group. + ## Related -- [HLS restart drain cutoff](/quest/m2/hls-restart-drain-cutoff.md) - changes what a replaced run's cursor drains; update the takeover scenario with it +- [HLS restart drain cutoff](/quest/m2/hls-restart-drain-cutoff.md) - owns how long a retired instance is allowed to drain - `export hls --output` (`rs/moq-cli/src/hls/record.rs`) - the recorder that would write the leaked segment - moq-hls's `moq-tokio` dev-dependency features (`tcp`, `quic`, `aws-lc-rs`) now serve only `origin::spawn`. Trimming them to `net` first needs moq-tokio's `Backoff::validate` fixed, since it is dead code under `net` alone. diff --git a/rs/moq-hls/tests/integration/main.rs b/rs/moq-hls/tests/integration/main.rs index 4016ac1a3b9..a0783574568 100644 --- a/rs/moq-hls/tests/integration/main.rs +++ b/rs/moq-hls/tests/integration/main.rs @@ -5,5 +5,6 @@ mod disconnect; mod finish; +mod restart; mod server; mod support; diff --git a/rs/moq-hls/tests/integration/restart.rs b/rs/moq-hls/tests/integration/restart.rs new file mode 100644 index 00000000000..84ce6606dd1 --- /dev/null +++ b/rs/moq-hls/tests/integration/restart.rs @@ -0,0 +1,145 @@ +//! A new publisher instance taking over a path is served under its own URLs, while the run it +//! replaced finishes its own recording and never serves the new run's bytes. + +use std::time::Duration; + +use axum::http::StatusCode; + +use crate::support::*; + +const OLD: u8 = 1; +const NEW: u8 = 2; + +/// The old run finishes after takeover, while its recording still needs the last open GOP. +#[tokio::test(start_paused = true)] +async fn takeover_lite06() { + takeover("moq-lite-06").await; +} + +#[tokio::test(start_paused = true)] +async fn takeover_lite07() { + takeover("moq-lite-07-wip").await; +} + +async fn takeover(version: &str) { + let pair = Pair::new(version.parse().unwrap(), Default::default()).await; + let mut old = Publisher::new(&pair.publisher, "live", Default::default()); + let server = pair.server(); + let app = server.router(); + let rendition = rendition(&server, "live").await; + let mut cursor = rendition.segments(); + + old.write(0, 13, OLD); + let old_media = media_uri(&app, "live").await; + listed(&app, &old_media, ".2.m4s").await; + let replaced = server.broadcaster("live").await.expect("announced"); + replaced.set_generation(Some("run-1")).expect("generation"); + let playlist = listed(&app, &old_media, "seg/run-1.").await; + let old_segments: Vec<_> = (0..3).map(|index| segment_uri(&old_media, &playlist, index)).collect(); + for uri in &old_segments { + assert_old(&app, uri).await; + } + // Read as a recorder does. A cached segment can return while the live feed is still opening. + let first = moq_clock::timeout(TIMEOUT, cursor.next()) + .await + .expect("the first segment") + .expect("records") + .expect("a segment"); + assert_eq!(first.segment, 0); + assert!(contains( + first.media.as_deref().expect("first segment media"), + &[OLD; 8] + )); + + // The new instance numbers its groups and segments from zero again. + let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + let mut new = Publisher::new(&pair.publisher, "live", route); + let deadline = moq_clock::now() + TIMEOUT; + let current = loop { + let current = server.broadcaster("live").await.expect("announced"); + if !std::sync::Arc::ptr_eq(¤t, &replaced) { + break current; + } + assert!(moq_clock::now() < deadline, "the new instance never took over"); + moq_clock::sleep(Duration::from_millis(50)).await; + }; + current.set_generation(Some("run-2")).expect("generation"); + new.write(0, 13, NEW); + + // The master lists the new instance, whose playlist lists its own generation. + let new_media = media_uri(&app, "live").await; + assert_ne!(new_media, old_media, "a new instance gets its own URLs"); + let playlist = listed(&app, &new_media, ".2.m4s").await; + assert!(!playlist.contains("run-1."), "{playlist}"); + let new_segment = segment_uri(&new_media, &playlist, 0); + assert!(new_segment.contains("/seg/run-2."), "{new_segment}"); + let response = get(&app, &new_segment).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(contains(response.body(), &[NEW; 8]) && !contains(response.body(), &[OLD; 8])); + // Nor does the new instance answer the old run's generation. + let crossed = new_segment.replace("/seg/run-2.", "/seg/run-1."); + assert_eq!(get(&app, &crossed).await.status(), StatusCode::NOT_FOUND); + + // The old run's URLs, under either generation, never answer with the new run's bytes. + for uri in &old_segments { + assert_old(&app, uri).await; + assert_old(&app, &uri.replace("/seg/run-1.", "/seg/run-2.")).await; + } + + // The old run ends while the new one carries on past the old run's last group. + old.finish(); + new.write(13, 21, NEW); + drop(old); + + // The old recording drains every segment of its own, its final GOP included, and nothing else. + let mut read = Vec::new(); + loop { + let next = moq_clock::timeout(TIMEOUT, cursor.next()) + .await + .expect("the replaced run's cursor never ended"); + let Some(segment) = next.expect("the replaced run finished cleanly") else { + break; + }; + let media = segment.media.as_deref().expect("segment media"); + assert!( + contains(media, &[OLD; 8]) && !contains(media, &[NEW; 8]), + "the replaced run's segment {} carries the new run's media", + segment.segment + ); + read.push(segment.segment); + } + assert_eq!(read, [1, 2, 3]); + let playlist = rendition.media_playlist(None).expect("finished").expect("listed"); + assert!(playlist.ends_with("#EXT-X-ENDLIST\n"), "{playlist}"); + + for uri in &old_segments { + assert_old(&app, uri).await; + } + assert_eq!(get(&app, &new_segment).await.status(), StatusCode::OK); + drop(new); +} + +/// `uri` answers the old run's bytes or nothing at all. +async fn assert_old(app: &axum::Router, uri: &str) { + let response = get(app, uri).await; + assert!(!contains(response.body(), &[NEW; 8]), "{uri} served the new run"); + match response.status() { + StatusCode::OK => assert!(contains(response.body(), &[OLD; 8]), "{uri}"), + _ => assert_missing(&response), + } +} + +/// The URI of the `index`th segment `playlist` lists, beside the media playlist at `media`. +fn segment_uri(media: &str, playlist: &str, index: usize) -> String { + let line = playlist + .lines() + .filter(|line| line.starts_with("seg/")) + .nth(index) + .unwrap_or_else(|| panic!("no segment {index} in {playlist}")); + media.replace("media.m3u8", line) +} + +/// Whether `haystack` contains `needle`. +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|window| window == needle) +} From 694e36c1524174693d9acde93c2395a2356c8810 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 23:11:58 -0700 Subject: [PATCH 3/9] fix(net): refuse an unanswered epochless request once its instance is replaced A moq-lite SUBSCRIBE or TRACKS without an epoch is resolved by whichever instance wins the path when it arrives. The subscriber stopped checking its source once the request was flushed, so a takeover landing between the flush and the publisher's resolution fed the replacement's media into the retired instance's cache, or left it waiting on the replacement's group. Keep the retirement check until the publisher first answers, in Rust and JS, and refuse with Unroutable as the moq-transport subscriber already does before SUBSCRIBE_OK. Answered subscriptions still drain the old instance. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 10 ++- js/net/src/lite/subscriber.test.ts | 111 ++++++++++++++++++++++- js/net/src/lite/subscriber.ts | 18 +++- quest/m1/README.md | 1 - quest/m1/hls-takeover-leak.md | 88 ------------------ rs/moq-hls/tests/integration/restart.rs | 115 ++++++++++++++++++------ rs/moq-net/src/lite/subscriber.rs | 66 +++++++++++--- rs/moq-net/tests/integration/restart.rs | 56 ++++++++++++ 8 files changed, 331 insertions(+), 134 deletions(-) delete mode 100644 quest/m1/hls-takeover-leak.md diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 605c8eb9050..383febaf2a4 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -116,10 +116,12 @@ authorization and hidden names work exactly as without one. without an epoch keeps its subscriptions until that route goes, and a track that fails ends instead of resuming, so request the path again. - **Another instance restarts.** A newer epoch, or, without one, another route - winning the path, is announced as a restart. Subscriptions already open stay - on the old instance until dropped or none of its routes remain, which ends - them with an error, while new requests get the new one, so a player follows - the restart by requesting the path again. + winning the path, is announced as a restart. Subscriptions already answered + stay on the old instance until dropped or none of its routes remain, which + ends them with an error, while new requests get the new one, so a player + follows the restart by requesting the path again. Without an epoch, a request + still unanswered at the restart ends the same way, since the replacement may + be the one answering it. Held handles can still read their old cached content while a subscription on the old instance stays open, but cannot open new epochless requests once the restart is delivered. Request the path again for diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index f49b06c9bbc..8a71c880bb4 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -24,7 +24,7 @@ import { type AnnounceBroadcast, AnnounceInit, AnnounceOk, encodeAnnounceBroadca import { Group } from "./group.ts"; import { Probe } from "./probe.ts"; import { StreamId } from "./stream.ts"; -import { encodeSubscribeResponse, SubscribeEnd } from "./subscribe.ts"; +import { encodeSubscribeResponse, SubscribeEnd, SubscribeStart } from "./subscribe.ts"; import { SUBSCRIBE_SETUP_TIMEOUT_MS, Subscriber } from "./subscriber.ts"; import { Track, TrackInfo } from "./track.ts"; import { Version } from "./version.ts"; @@ -1514,6 +1514,115 @@ test("a listing of an epochless handle retired while its token waits never reach subscriber.close(); }); +// Announce PUBLISHER_A's epochless `pool` on stream 0, and return how to replace it with PUBLISHER_B's. +async function epochlessPool(version: Version, streams: FakeStream[], announced: { next(): Promise }) { + const send = async (encode: (writer: Writer) => Promise) => { + const writer = new Writer( + new WritableStream({ + write: (chunk) => streams[0].inbound.enqueue(new Uint8Array(chunk)), + }), + version, + ); + await encode(writer); + }; + const path = Path.from("pool"); + await send((writer) => new AnnounceOk(PEER, 0).encode(writer, version)); + await send((writer) => + encodeAnnounceBroadcast(writer, { status: "active", suffix: path, hops: [PUBLISHER_A] }, version), + ); + await announced.next(); + return async () => { + await send((writer) => + encodeAnnounceBroadcast( + writer, + version === Version.DRAFT_05 ? { status: "ended", suffix: path } : { status: "endedId", id: 0n }, + version, + ), + ); + await announced.next(); + await send((writer) => + encodeAnnounceBroadcast(writer, { status: "active", suffix: path, hops: [PUBLISHER_B] }, version), + ); + await announced.next(); + }; +} + +// A SUBSCRIBE without an epoch resolves whichever instance won the path when it arrived, so one +// unanswered when its instance is replaced is refused. One answered first stays on its instance. +test.each([ + ["unanswered on lite-05 is refused", Version.DRAFT_05, false], + ["unanswered on lite-06 is refused", Version.DRAFT_06, false], + ["unanswered on lite-07 is refused", Version.DRAFT_07, false], + ["answered on lite-07 stays", Version.DRAFT_07, true], +] as const)("an epochless SUBSCRIBE %s at a takeover", async (_, version, answered) => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const announced = subscriber.announced(); + try { + await opened(0); + const replace = await epochlessPool(version, streams, announced); + const held = subscriber.consume(Path.from("pool/job")); + const track = held.track("video").subscribe(); + await answerTrackInfo(await opened(1), version); + const subscribe = await opened(2); + await subscribe.reading; + if (answered) { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + await encodeSubscribeResponse(writer, { start: new SubscribeStart(0) }, version); + for (const chunk of chunks) subscribe.inbound.enqueue(chunk); + await drainReactions(); + } + + await replace(); + if (answered) { + await drainReactions(); + expect(track.closed.peek()).toBeUndefined(); + } else { + const err = await track.closed; + expect(err).toBeInstanceOf(StreamError); + expect((err as StreamError).code).toBe(StreamCode.Unroutable); + await subscribe.aborted; + } + track.close(); + held.close(); + } finally { + announced.close(); + subscriber.close(); + warn.mockRestore(); + } +}); + +// A TRACKS request without an epoch lists whichever instance won the path when it arrived. +test("a listing of an epochless handle retired before its answer is refused", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const announced = subscriber.announced(); + await opened(0); + const replace = await epochlessPool(version, streams, announced); + const held = subscriber.consume(Path.from("pool/job")); + const tracks = held.tracks(); + expect(await tracks.next()).toEqual({ kind: "synced" }); + const event = tracks.next().catch((err: unknown) => err); + const listing = await opened(1); + await listing.reading; + + await replace(); + const err = await event; + expect(err).toBeInstanceOf(StreamError); + expect((err as StreamError).code).toBe(StreamCode.Unroutable); + await listing.aborted; + tracks.close(); + held.close(); + announced.close(); + subscriber.close(); +}); + test("a listing whose readers leave as its token settles never reaches the wire", async () => { const { quic, streams } = fakeSession(); let release!: () => void; diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 245b45d4e59..ca9cb1ac763 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -140,6 +140,9 @@ interface SubscribeEntry { // the SUBSCRIBE has its first response, so the publisher's demand never lapses between // the two. held?: Stream; + // Called at the SUBSCRIBE's first response (lite-05+). Until then a SUBSCRIBE without an + // epoch may be answered by whichever instance won the path, so its retirement refuses it. + answered?: () => void; } /** Subscribe to the peer and own its receive-side protocol tasks. @@ -689,8 +692,8 @@ export class Subscriber { // fails, our grant stops covering the broadcast, or nobody lists the broadcast any more. // Returns without answering when setup was cut short, for the caller to look again. async #listTracks(broadcast: Path.Valid, consumed: Consumed, lister: Lister): Promise { - // Until the request is out, the consumed instance must still be the selected one: a TRACKS - // request without an epoch would list its replacement. + // Until the peer answers, the consumed instance must still be the selected one: a TRACKS + // request without an epoch lists whichever instance won the path when it arrived. const valid = consumed.valid; if (valid?.aborted) { lister.abort(error(valid.reason)); @@ -748,10 +751,12 @@ export class Subscriber { await stream.writer.u53(StreamId.Tracks); await new TracksMessage(broadcast, consumed.epoch).encode(stream.writer, this.version); let synced = false; + let unanswered = cut; for (;;) { const pause = listingBudget.poll(); if (pause !== undefined) await untilAborted(pause, stop); - const event = await untilAborted(decodeTracksEventMaybe(stream.reader, this.version), stop); + const event = await untilAborted(decodeTracksEventMaybe(stream.reader, this.version), unanswered); + unanswered = stop; if (!event) { // The peer's broadcast ended, whether or not its retraction has arrived: end the // listing and let go of what it listed. FIN our half too, so the stream is done. @@ -921,8 +926,10 @@ export class Subscriber { return; } - valid?.removeEventListener("abort", retire); const { stream, entry } = opened; + // Older drafts already waited for SUBSCRIBE_OK. + if (hasTrackStream(this.version)) entry.answered = () => valid?.removeEventListener("abort", retire); + else valid?.removeEventListener("abort", retire); const producer = entry.track; try { // Watch for subscription changes and send SUBSCRIBE_UPDATE. Lite01/Lite02 @@ -980,6 +987,7 @@ export class Subscriber { console.warn(`subscribe error: id=${id} broadcast=${broadcast} track=${request.name} error=${reason(e)}`); stream.abort(e); } finally { + valid?.removeEventListener("abort", retire); entry.held?.close(); this.#subscribes.delete(id); } @@ -1394,6 +1402,8 @@ export class Subscriber { // The publisher answered, so its demand stands on the subscription from here. entry.held?.close(); entry.held = undefined; + entry.answered?.(); + entry.answered = undefined; if (!resp) { if (entry.end === undefined && entry.track.closed.peek() === undefined) throw new ProtocolViolation("subscribe stream ended without SUBSCRIBE_END"); diff --git a/quest/m1/README.md b/quest/m1/README.md index 3c30e051e2e..ccdf01b9f4d 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -155,7 +155,6 @@ surface and zero-copy texture import, land before the cut. - [Generated JS API reference](/quest/m1/js-api-reference.md) - the doc site renders an API reference for every `@moq/*` package from API Extractor's doc model - [Interop coverage](/quest/m1/interop-coverage/README.md) - the riskiest moq-transport features moq-net exposes get cross-implementation tests over drafts 18 and 22 - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - A/A and deliberately slowed Nightly runs set the paired comparison's threshold, so its regression alerts can be trusted -- [HLS takeover leak](/quest/m1/hls-takeover-leak.md) - a replaced instance's recording cursor never yields the replacement's media, and ends or fails instead of stalling - [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - nightly ffprobe checks that advertised timing, sync flags, and bytes match the media served - [HLS import log redaction](/quest/m1/hls-log-redaction.md) - no HLS import log line carries an unredacted playlist URL - [App-limited BBR3 estimate ages](/quest/m1/bbr-app-limited.md) - an app-limited BBR3 estimate falls within seconds after the path drops diff --git a/quest/m1/hls-takeover-leak.md b/quest/m1/hls-takeover-leak.md deleted file mode 100644 index 1fd31fd46e2..00000000000 --- a/quest/m1/hls-takeover-leak.md +++ /dev/null @@ -1,88 +0,0 @@ -# [M] HLS takeover leaks the replacement's media - -## Goal - -After a new publisher instance takes over a broadcast path, a recording cursor on the -replaced instance never yields the replacement's media. It drains the old run's own -segments, then ends cleanly or fails with a typed error. It never stalls. - -## Plan - -The failing session reproduction is checked in at -`rs/moq-hls/tests/integration/restart.rs`. Both `takeover_lite06` and -`takeover_lite07` fail on the replaced run's final segment carrying the new run's -payload. The reproduction uses the session suite's paused-clock `support` harness. -These are active failing tests in an unfinished draft, not a completed fix. - -### Confirmed cause - -The old run publishes three complete segments and leaves group 6 open at 12s. Its -recording cursor returns segment 0 from FETCHed groups while its live media -subscription is only queued. Returning that segment does not mean the upstream -SUBSCRIBE has opened or resolved. The new publisher then takes over the path. - -Temporary tracing produced this order for the old media subscription: - -```text -TAKEOVER LOCAL -SUB PREP live video0 id 2 -SUB ACTIVE live video0 id 2 -PUB DECODE subscribed live video0 -SELECTION live end -PUB RESOLVE subscribed live video0 -``` - -`TrackServe::prepare_establish` registers id 2 against the old source's track. -`Establish::poll` checks that source's request validity, then on lite-05+ considers -the subscription active as soon as its request flushes. At this point the subscriber -has not yet received the old announcement's retirement, so its check succeeds. -The publisher's `RequestServe::poll_serve` resolves that epochless request through -`origin.request(path, None)`, which selects the replacement. The existing id 2 now -feeds new media into the old source's cache. No wire FETCH for group 6 is emitted: -`Rendition::fetch` finds that poisoned cached group and serves the new bytes under -the old row. The final segment contains two NEW-tagged frames, rather than the -old run's one OLD-tagged final frame. - -The source selection does retire, and the rendition's source and queried track -remain pinned. This is an establishment identity race, not a resume that crosses -two known publisher epochs. The earlier bare moq-net attempt already established -its subscription before takeover, so it did not exercise this ordering. - -Eagerly polling the HLS feed, or waiting for SUBSCRIBE_START before calling a -subscription active, changes this schedule but cannot prove which instance an -epochless request meant. Announcement and subscription streams are independently -ordered. The publisher may resolve the replacement and return its START before -the subscriber observes the old announcement's retirement. FETCH has the same -path-only addressing ambiguity. Neither a retry nor a new drain timeout fixes -that identity gap. - -### Decision needed - -Choose the identity and legacy behavior before implementing the fix: - -1. **Require addressable instance identity for safe tail reads (recommended).** - Use the intended publisher's epoch wherever the wire can name it, and define - a typed refusal for ambiguous epochless or legacy operations. This preserves - the no-cross-instance guarantee but may make a retired recording fail rather - than recover its last uncached GOP. Decide where the identity requirement is - enforced and which existing/public error represents the refusal. -2. **Keep full epochless tail recovery.** Design a wire association tying TRACK, - SUBSCRIBE, and FETCH to the same publisher instance, rather than resolving - each request's path independently. The unpublished wire can change; published - wires still need an explicit safe refusal policy. This is wider API/wire scope - and must include the JS implementation and draft. - -The separate drain-cutoff quest owns how long an already established old stream -may drain. Do not choose that policy or make a scheduling-only workaround here. - -After the decision, implement at the identity boundary and extend the session -coverage to an unfinished replacement group, the old run finishing before the -replacement announces, and established subscriptions that legitimately drain OLD -media. A missing old group may fail with the agreed typed error; it must neither -read NEW media nor wait for the replacement to finish a same-numbered group. - -## Related - -- [HLS restart drain cutoff](/quest/m2/hls-restart-drain-cutoff.md) - owns how long a retired instance is allowed to drain -- `export hls --output` (`rs/moq-cli/src/hls/record.rs`) - the recorder that would write the leaked segment -- moq-hls's `moq-tokio` dev-dependency features (`tcp`, `quic`, `aws-lc-rs`) now serve only `origin::spawn`. Trimming them to `net` first needs moq-tokio's `Backoff::validate` fixed, since it is dead code under `net` alone. diff --git a/rs/moq-hls/tests/integration/restart.rs b/rs/moq-hls/tests/integration/restart.rs index 84ce6606dd1..a88288bae85 100644 --- a/rs/moq-hls/tests/integration/restart.rs +++ b/rs/moq-hls/tests/integration/restart.rs @@ -4,24 +4,32 @@ use std::time::Duration; use axum::http::StatusCode; +use moq_hls::export::segments; use crate::support::*; const OLD: u8 = 1; const NEW: u8 = 2; -/// The old run finishes after takeover, while its recording still needs the last open GOP. -#[tokio::test(start_paused = true)] -async fn takeover_lite06() { - takeover("moq-lite-06").await; +/// What the replaced run's recording holds when the new instance takes over. +#[derive(Clone, Copy, Debug)] +enum Case { + /// A cached segment returned while the cursor's live SUBSCRIBE was still unanswered. + /// Without an epoch nothing says which instance answers it, so it is refused and the + /// recording loses the old run's last uncached GOP. + Unanswered, + /// As [`Self::Unanswered`], while the replacement's own group of the same number is + /// still open, which the old cursor once waited on forever. + UnansweredUnfinished, + /// The live subscription was answered before the takeover, so it stays on the old run + /// and drains every segment of its own. + Answered, + /// As [`Self::Unanswered`], with the old run finished just before the new instance + /// announces. + FinishedFirst, } -#[tokio::test(start_paused = true)] -async fn takeover_lite07() { - takeover("moq-lite-07-wip").await; -} - -async fn takeover(version: &str) { +async fn takeover(version: &str, case: Case) { let pair = Pair::new(version.parse().unwrap(), Default::default()).await; let mut old = Publisher::new(&pair.publisher, "live", Default::default()); let server = pair.server(); @@ -50,6 +58,12 @@ async fn takeover(version: &str) { first.media.as_deref().expect("first segment media"), &[OLD; 8] )); + match case { + // Paused, the clock only moves once every task is idle, so the feed has its answer. + Case::Answered => moq_clock::sleep(Duration::from_millis(500)).await, + Case::FinishedFirst => old.finish(), + Case::Unanswered | Case::UnansweredUnfinished => {} + } // The new instance numbers its groups and segments from zero again. let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); @@ -86,19 +100,77 @@ async fn takeover(version: &str) { assert_old(&app, &uri.replace("/seg/run-1.", "/seg/run-2.")).await; } - // The old run ends while the new one carries on past the old run's last group. - old.finish(); - new.write(13, 21, NEW); + // The old run ends, while the new one carries on past the old run's last group unless it + // leaves that group open. + if !matches!(case, Case::FinishedFirst) { + old.finish(); + } + if !matches!(case, Case::UnansweredUnfinished) { + new.write(13, 21, NEW); + } drop(old); - // The old recording drains every segment of its own, its final GOP included, and nothing else. + // The old recording reads only its own segments, and either completes or fails at the GOP + // it lost. + let (read, end) = drain_old(&mut cursor).await; + match case { + Case::Answered => { + end.unwrap_or_else(|err| panic!("{case:?}: {err}")); + assert_eq!(read, [1, 2, 3], "{case:?}"); + let playlist = rendition.media_playlist(None).expect("finished").expect("listed"); + assert!(playlist.ends_with("#EXT-X-ENDLIST\n"), "{case:?}: {playlist}"); + } + Case::Unanswered | Case::UnansweredUnfinished | Case::FinishedFirst => { + assert!( + matches!(end, Err(moq_hls::Error::SegmentUnavailable { segment: 3 })), + "{case:?}: {end:?}" + ); + assert_eq!(read, [1, 2], "{case:?}"); + } + } + + for uri in &old_segments { + assert_old(&app, uri).await; + } + drop(new); +} + +#[tokio::test(start_paused = true)] +async fn unanswered_lite06() { + takeover("moq-lite-06", Case::Unanswered).await; +} + +#[tokio::test(start_paused = true)] +async fn unanswered_lite07() { + takeover("moq-lite-07-wip", Case::Unanswered).await; +} + +#[tokio::test(start_paused = true)] +async fn unanswered_unfinished() { + takeover("moq-lite-07-wip", Case::UnansweredUnfinished).await; +} + +#[tokio::test(start_paused = true)] +async fn answered() { + takeover("moq-lite-07-wip", Case::Answered).await; +} + +#[tokio::test(start_paused = true)] +async fn finished_first() { + takeover("moq-lite-07-wip", Case::FinishedFirst).await; +} + +/// Read `cursor` until it ends, asserting each segment carries only the old run's media. +async fn drain_old(cursor: &mut segments::Consumer) -> (Vec, moq_hls::Result<()>) { let mut read = Vec::new(); loop { let next = moq_clock::timeout(TIMEOUT, cursor.next()) .await - .expect("the replaced run's cursor never ended"); - let Some(segment) = next.expect("the replaced run finished cleanly") else { - break; + .expect("the replaced run's cursor stalled"); + let segment = match next { + Ok(Some(segment)) => segment, + Ok(None) => return (read, Ok(())), + Err(err) => return (read, Err(err)), }; let media = segment.media.as_deref().expect("segment media"); assert!( @@ -108,15 +180,6 @@ async fn takeover(version: &str) { ); read.push(segment.segment); } - assert_eq!(read, [1, 2, 3]); - let playlist = rendition.media_playlist(None).expect("finished").expect("listed"); - assert!(playlist.ends_with("#EXT-X-ENDLIST\n"), "{playlist}"); - - for uri in &old_segments { - assert_old(&app, uri).await; - } - assert_eq!(get(&app, &new_segment).await.status(), StatusCode::OK); - drop(new); } /// `uri` answers the old run's bytes or nothing at all. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 5593cb0ab9f..ddf7212de94 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1885,16 +1885,16 @@ struct TracksRecv { gate: crate::auth::Gate, /// The gate let the listing through, and this hop synced its readers. admitted: bool, - /// Until the request is out, the source must still be the selected instance: a TRACKS - /// request without an epoch would list its replacement. - validity: crate::model::broadcast::RequestValidity, + /// Until the peer answers, the source must still be the selected instance: a TRACKS + /// request without an epoch lists whichever instance won the path when it arrived. + unanswered: Option, } impl TracksRecv { /// Apply the peer's events until it ends the listing, or the error that ends it. fn poll(&mut self, source: &SourceServe, waiter: &kio::Waiter) -> Poll> { - if self.stream.is_none() { - self.validity.check(waiter)?; + if let Some(validity) = &self.unanswered { + validity.check(waiter)?; } ready!(self.gate.poll_ready(waiter)); if self.gate.poll_denied(waiter).is_ready() { @@ -1923,7 +1923,9 @@ impl TracksRecv { ready!(stream.writer.poll_flush(&mut cx))?; loop { ready!(self.budget.poll_yield(waiter)); - let Some(event) = ready!(stream.reader.poll_decode_maybe::(&mut cx))? else { + let event = ready!(stream.reader.poll_decode_maybe::(&mut cx))?; + self.unanswered = None; + let Some(event) = event else { return Poll::Ready(Ok(())); }; match event { @@ -2007,7 +2009,7 @@ impl SourceServe { crate::auth::Direction::Subscribe, ), admitted: false, - validity: self.dynamic.validity().start(), + unanswered: Some(self.dynamic.validity().start()), }); } Some(_) if self.lister.poll_unused(waiter).is_ready() => { @@ -2138,6 +2140,7 @@ mod tests { served: None, end: None, wanted: None, + unanswered: None, }) } @@ -2252,6 +2255,7 @@ mod tests { served: None, end: None, wanted: None, + unanswered: None, }); if update { h.serve @@ -2712,6 +2716,7 @@ mod tests { served: None, end: Some(lite::SubscribeEnd { group: 10 }), wanted: None, + unanswered: None, }; assert_eq!( sub.owed(None), @@ -4057,6 +4062,30 @@ mod tests { assert_eq!(session.log.bi_opens(), 0); } + /// An instance retired while its listing is on the wire but unanswered lists nothing: + /// the peer may list the replacement that won the path. + #[moq_clock::test] + async fn an_unanswered_listing_of_a_retired_instance_is_refused() { + let open = kio::Producer::new(true); + let session = SinkSession::gated_bi(open.consume()); + let selected = kio::Producer::new(()); + let validity = crate::model::broadcast::RequestValidity::Route(selected.consume()); + let (mut serve, consumer, _route) = listed_source(session.clone(), crate::auth::Link::new(false), validity); + let mut tracks = consumer.tracks().unwrap(); + + assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_pending()); + assert_eq!(session.log.bi_opens(), 1); + assert!(matches!( + tracks.next().now_or_never(), + Some(Ok(Some(crate::broadcast::TracksEvent::Synced))) + )); + + drop(selected); + let _ = kio::Task::poll(&mut serve, &kio::Waiter::noop()); + assert!(matches!(tracks.next().now_or_never(), Some(Err(Error::Unroutable)))); + assert_eq!(session.log.resets(), vec![crate::StreamError::Unroutable.to_code()]); + } + /// A listing outside the session's scope is refused before it reaches the wire. #[moq_clock::test] async fn a_listing_outside_the_scope_is_refused() { @@ -5946,6 +5975,10 @@ struct SubStream { end: Option, /// The refresh join's wanted group, restated in every SUBSCRIBE_UPDATE. wanted: Option, + /// Until the publisher's first response, what the source must still hold: a SUBSCRIBE + /// without an epoch resolves whichever instance wins the path when it arrives, so one + /// still unanswered once the source is replaced may be answered by the replacement. + unanswered: Option, } impl SubStream { @@ -6626,7 +6659,7 @@ impl Establish { self.state = EstablishState::WaitOk { stream }; continue; } - return Poll::Ready(Ok(self.activate(stream))); + return Poll::Ready(Ok(self.activate(stream, false))); } EstablishState::WaitOk { stream } => { if let Poll::Ready(err) = self.closed.poll_closed(&mut cx) { @@ -6640,7 +6673,7 @@ impl Establish { else { unreachable!() }; - return Poll::Ready(Ok(self.activate(stream))); + return Poll::Ready(Ok(self.activate(stream, true))); } } } @@ -6654,8 +6687,10 @@ impl Establish { } } - fn activate(&self, stream: Stream) -> SubStream { + /// The live subscription, `answered` once the publisher's first response arrived. + fn activate(&self, stream: Stream, answered: bool) -> SubStream { SubStream { + unanswered: (!answered).then(|| self.validity.clone()), stream, id: self.id, max_delay: wire_max_delay(self.subscription.max_delay), @@ -7479,6 +7514,7 @@ impl ServeLoop { // subscription now. A query still held once that subscription ends // stays local: nothing reopens the TRACK for it. self.held = None; + active.unanswered = None; match res { Ok(Some(msg)) => { match &msg { @@ -7600,6 +7636,16 @@ impl ServeLoop { } } + // A SUBSCRIBE still unanswered once its source is replaced may have resolved + // the replacement: refuse it. An answer already read above wins. + if let Sub::Active(active) = &self.sub + && let Some(validity) = &active.unanswered + && let Err(err) = validity.check(waiter) + { + tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "unanswered subscribe refused: instance replaced"); + return Poll::Ready(ServeEnd::GiveBack(err)); + } + // (5) The session died: hand the track back for another route, with // the session's error in case none takes it. if let Poll::Ready(err) = self.closed.poll_closed(&mut cx) { diff --git a/rs/moq-net/tests/integration/restart.rs b/rs/moq-net/tests/integration/restart.rs index 5fcfe0bdbc4..fcfcbd79781 100644 --- a/rs/moq-net/tests/integration/restart.rs +++ b/rs/moq-net/tests/integration/restart.rs @@ -528,6 +528,62 @@ retired_fetch_tests! { retired_fetch_ietf22: "moq-transport-22", } +/// An epochless SUBSCRIBE names no instance, so until the publisher answers it, it may have +/// resolved whichever instance won the path. Unanswered when its instance is replaced, it is +/// refused, even if the publisher had resolved it on the old one, rather than risk filling +/// the old instance from the replacement. +async fn unanswered_takeover(version: &str) { + let version: Version = version.parse().unwrap(); + let publisher = produce_origin(1); + let viewer = produce_origin(2); + let _session = link(version, &publisher, &viewer).await; + let old = publisher.publish("live", origin::Route::default()).unwrap(); + // Quiet, so the publisher has nothing to answer the SUBSCRIBE with yet. + let old_track = old.create_track("video", timed()).unwrap(); + let mut announced = viewer.consume().announced(); + assert!(matches!(next_event(&mut announced).await, announce::Event::Start(_))); + let (_held, mut subscription) = subscribe(&viewer, "live").await; + settle().await; + assert!( + old_track.subscription().is_some(), + "{version}: the SUBSCRIBE reached the old instance" + ); + + let replacement = publisher.publish("live", origin::Route::default()).unwrap(); + let replacement_track = replacement.create_track("video", timed()).unwrap(); + write(&replacement_track, "new:0"); + expect_restart(version, &mut announced, "live").await; + write(&old_track, "old:0"); + + let next = moq_clock::timeout(Duration::from_secs(10), subscription.recv_group()) + .await + .expect("subscription hung"); + match next { + Err(err) => assert!(matches!(err, moq_net::Error::Unroutable), "{version}: {err}"), + Ok(group) => panic!( + "{version}: the replaced instance's SUBSCRIBE went on: {:?}", + group.map(|group| group.sequence) + ), + } +} + +macro_rules! unanswered_takeover_tests { + ($($name:ident: $version:literal,)*) => { $( + #[moq_clock::test] + async fn $name() { + moq_clock::timeout(TEST_TIMEOUT, unanswered_takeover($version)).await.expect("timed out"); + } + )* }; +} + +// moq-transport answers a SUBSCRIBE at once, before any group, and its subscriber already +// waits for that answer. +unanswered_takeover_tests! { + unanswered_takeover_lite05: "moq-lite-05", + unanswered_takeover_lite06: "moq-lite-06", + unanswered_takeover_lite07: "moq-lite-07-wip", +} + #[moq_clock::test] async fn retired_fetch_explicit_epoch_control() { moq_clock::timeout(TEST_TIMEOUT, retired_fetch("moq-lite-07-wip", true)) From 94378c7f298f4d0ec7124d3893e7e95ba94cb717 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 23:27:56 -0700 Subject: [PATCH 4/9] test(net): answer the SUBSCRIBE a drain test holds open Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/session_close.rs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/tests/integration/session_close.rs b/rs/moq-net/tests/integration/session_close.rs index bc08b6a9f3a..9f9bb045c30 100644 --- a/rs/moq-net/tests/integration/session_close.rs +++ b/rs/moq-net/tests/integration/session_close.rs @@ -103,6 +103,17 @@ async fn setup_with_gate(version: &str, gate: Option>) -> Se } } +/// Open a group the caller holds unfinished, so the publisher answers the SUBSCRIBE: the +/// withdrawal a close starts with refuses an unanswered SUBSCRIBE without an epoch, which +/// would leave nothing to drain. +async fn answer(track: &moq_net::track::Producer) -> moq_net::group::Producer { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); + // Paused, the clock only moves once every task is idle, so the answer has landed. + moq_clock::sleep(Duration::from_millis(100)).await; + group +} + /// TRACK_INFO creates demand before any SUBSCRIBE, so demand alone cannot gate a drain test. #[moq_clock::test] async fn track_info_interest_precedes_a_subscription() { @@ -181,9 +192,8 @@ async fn ietf_close_gives_up_on_a_live_track() { } async fn close_gives_up_on_a_live_track_for(version: &str) { - let Setup { - pair, track: _track, .. - } = setup(version).await; + let Setup { pair, track, .. } = setup(version).await; + let _live = answer(&track).await; let started = moq_clock::now(); let res = moq_clock::timeout(TIMEOUT, pair.client.close()) @@ -205,9 +215,8 @@ async fn ietf_abort_cuts_a_drain_short() { } async fn abort_cuts_a_drain_short_for(version: &str) { - let Setup { - pair, track: _track, .. - } = setup(version).await; + let Setup { pair, track, .. } = setup(version).await; + let _live = answer(&track).await; let other = pair.client.clone(); let started = moq_clock::now(); From 10d5b4e67a7534b4caf740f635912b12ed566d03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 23:34:55 -0700 Subject: [PATCH 5/9] fix(net): expect the large subscription variant on wasm too Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index ddf7212de94..11a24166312 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -6000,12 +6000,9 @@ impl SubStream { } } -#[cfg_attr( - target_pointer_width = "64", - expect( - clippy::large_enum_variant, - reason = "one per track copy, which holds a live stream while anyone reads it, so boxing it would add an allocation per upstream subscription without saving memory" - ) +#[expect( + clippy::large_enum_variant, + reason = "one per track copy, which holds a live stream while anyone reads it, so boxing it would add an allocation per upstream subscription without saving memory" )] enum Sub { None, From f39b7a49ec69840a29a15d9c3d3d86f7c150484d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 23:47:15 -0700 Subject: [PATCH 6/9] test(tokio): answer the SUBSCRIBE a handover test holds open Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/tests/integration/backend.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index c4d0d1de92d..8332a1968b8 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -1078,7 +1078,6 @@ async fn quic_client_close_keeps_predecessor_handover() { .parse() .unwrap(); - let (subscribed_tx, subscribed_rx) = tokio::sync::oneshot::channel::<()>(); let client = std::thread::spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -1107,7 +1106,12 @@ async fn quic_client_close_keeps_predecessor_handover() { while track.subscription().is_none() { track.subscription_changed().await.expect("track closed"); } - subscribed_tx.send(()).unwrap(); + // An unfinished group answers the SUBSCRIBE: one without an epoch still unanswered + // at the handover is refused, which would leave nothing to drain. + let mut group = track.append_group().expect("failed to append group"); + group + .write_frame(moq_tokio::moq_net::Timestamp::ZERO, b"live".as_slice()) + .expect("failed to write frame"); // Close while the old session is still inside its handover window. while connection.status().await.expect("connection stopped") != moq_tokio::Status::Migrating {} @@ -1144,12 +1148,12 @@ async fn quic_client_close_keeps_predecessor_handover() { .await .expect("subscribe timed out") .expect("subscribe failed"); - // Receiving is what sends the subscription, so read in the background. - let _received = tokio::spawn(async move { track.recv_group().await.map(|group| group.is_some()) }); - tokio::time::timeout(TIMEOUT, subscribed_rx) + // Receiving is what sends the subscription. + let _group = tokio::time::timeout(TIMEOUT, track.recv_group()) .await - .expect("the subscription never reached the client") - .expect("client thread panicked"); + .expect("the subscription was never answered") + .expect("subscribe failed") + .expect("track ended"); first .drain() From 46d5210f361a91316541c1c4a20c751a713b9f13 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 26 Oct 2026 10:19:10 -0700 Subject: [PATCH 7/9] fix(net): refuse an unanswered SUBSCRIBE before reading a co-ready answer WIP: breaks finished_broadcast_mock::a_finished_broadcast_delivers_its_finished_track. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 83 +++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 10 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 11a24166312..9c06741557a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -3464,6 +3464,67 @@ mod tests { } } + /// A SUBSCRIBE_START that is only read after the source was replaced may be the + /// replacement's answer, so the retirement refuses the subscription instead. + #[moq_clock::test] + async fn an_answer_read_after_the_retirement_is_refused() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let selected = kio::Producer::new(()); + let serve = TrackServe { + validity: crate::model::broadcast::RequestValidity::Route(selected.consume()), + route: None, + subscriber, + path: Path::new("room").to_owned(), + epoch: None, + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let request = broadcast.reserve_track("video").unwrap(); + let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); + let mut reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let gate = serve.gate(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Serve(serving), + gate, + }; + assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); + + let mut start = Vec::new(); + lite::SubscribeResponse::Start(lite::SubscribeStart { + group: 0, + largest: None, + }) + .encode(&mut crate::coding::Encoder::new(&mut start, version.into()), version) + .unwrap(); + session.push(&start); + drop(selected); + let _ = kio::Task::poll(&mut running, &kio::Waiter::noop()); + + let res = reader.recv_group().now_or_never().map(|res| res.map(|_| ())); + assert!(matches!(res, Some(Err(Error::Unroutable))), "{version:?}: {res:?}"); + } + } + /// Lite-07's SUBSCRIBE_START says where the track is, so a reader knows its largest /// frame before any group arrives. The FIN with no SUBSCRIBE_END then aborts the track, /// and the stream ends with it. @@ -7400,6 +7461,18 @@ impl ServeLoop { ServeMode::Select => { let mut cx = waiter.context(); + // A SUBSCRIBE still unanswered once its source is replaced may have resolved + // the replacement: refuse it. Checked before reading the stream, so an answer + // that only arrived alongside the replacement cannot claim it, and before the + // flush, so an update parked on flow control cannot hold the refusal back. + if let Sub::Active(active) = &self.sub + && let Some(validity) = &active.unanswered + && let Err(err) = validity.check(waiter) + { + tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "unanswered subscribe refused: instance replaced"); + return Poll::Ready(ServeEnd::GiveBack(err)); + } + // Deliver any buffered SUBSCRIBE_UPDATE before selecting, so the // demand that produced it is on the wire. if let Sub::Active(active) = &mut self.sub { @@ -7633,16 +7706,6 @@ impl ServeLoop { } } - // A SUBSCRIBE still unanswered once its source is replaced may have resolved - // the replacement: refuse it. An answer already read above wins. - if let Sub::Active(active) = &self.sub - && let Some(validity) = &active.unanswered - && let Err(err) = validity.check(waiter) - { - tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "unanswered subscribe refused: instance replaced"); - return Poll::Ready(ServeEnd::GiveBack(err)); - } - // (5) The session died: hand the track back for another route, with // the session's error in case none takes it. if let Poll::Ready(err) = self.closed.poll_closed(&mut cx) { From aa7dc2d40f812a213268b585a747b6c3ddecdfe9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 26 Oct 2026 10:47:09 -0700 Subject: [PATCH 8/9] fix(net): refuse an unanswered epochless request only on a replacement A withdrawal ended the source's selection period too, so a SUBSCRIBE or TRACKS still unanswered when its broadcast was withdrawn was refused, and a track that finished as its broadcast ended lost its content. moq-lite has the publisher conclude requests in flight after a retraction. The selection now keeps an answering lease beside its period: closed once another instance of the peer takes the path (a withdrawal followed by a new announcement included), left open by a withdrawal alone. A request already sent checks that lease until its first answer; the period still gates a request before it goes out. JS mirrors it with Validity.answering. Restores the session_close and moq-tokio handover tests to main, since a withdrawal no longer refuses the SUBSCRIBE they hold open. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 5 +- js/net/src/consume.ts | 26 ++- js/net/src/lite/subscriber.test.ts | 73 ++++--- js/net/src/lite/subscriber.ts | 41 ++-- rs/moq-net/src/fuzz.rs | 2 +- rs/moq-net/src/lite/subscriber.rs | 179 +++++++++++------- rs/moq-net/src/model/broadcast.rs | 28 ++- rs/moq-net/src/model/mod.rs | 2 +- rs/moq-net/src/model/origin.rs | 130 ++++++++++++- rs/moq-net/tests/integration/session_close.rs | 21 +- rs/moq-tokio/tests/integration/backend.rs | 18 +- 11 files changed, 369 insertions(+), 156 deletions(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 383febaf2a4..04cc11013c4 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -120,8 +120,9 @@ authorization and hidden names work exactly as without one. stay on the old instance until dropped or none of its routes remain, which ends them with an error, while new requests get the new one, so a player follows the restart by requesting the path again. Without an epoch, a request - still unanswered at the restart ends the same way, since the replacement may - be the one answering it. + still unanswered when another instance takes the path, even after a + withdrawal, ends the same way, since that instance may be the one answering + it. A withdrawal alone leaves it to its answer. Held handles can still read their old cached content while a subscription on the old instance stays open, but cannot open new epochless requests once the restart is delivered. Request the path again for diff --git a/js/net/src/consume.ts b/js/net/src/consume.ts index d3c90ac1fde..3f45e311b4a 100644 --- a/js/net/src/consume.ts +++ b/js/net/src/consume.ts @@ -6,20 +6,35 @@ import { compareRouteCandidates, coveringPrefixes } from "./internal.ts"; import type * as Path from "./path.ts"; import { type Instance, sameInstance } from "./wire.ts"; +// Unroutable, as in Rust: the instance going away is an end, not a failure. +function retired(): StreamError { + return new StreamError(StreamCode.Unroutable, { message: "broadcast instance retired" }); +} + class Validity { #current = new AbortController(); + #answering = new AbortController(); get signal(): AbortSignal { return this.#current.signal; } - update(allowed: boolean): void { + /** + * Aborted once another instance takes the path, which may answer a request still unanswered. + * A withdrawal alone leaves it, and the instance winning again keeps it. + */ + get answering(): AbortSignal { + return this.#answering.signal; + } + + update(allowed: boolean, replaced = false): void { if (allowed) { if (this.#current.signal.aborted) this.#current = new AbortController(); - } else if (!this.#current.signal.aborted) { - // Unroutable, as in Rust: the instance going away is an end, not a failure. - this.#current.abort(new StreamError(StreamCode.Unroutable, { message: "broadcast instance retired" })); + if (this.#answering.signal.aborted) this.#answering = new AbortController(); + return; } + if (!this.#current.signal.aborted) this.#current.abort(retired()); + if (replaced && !this.#answering.signal.aborted) this.#answering.abort(retired()); } } @@ -159,7 +174,8 @@ export class BroadcastCache { for (const path of this.#paths.get(prefix) ?? []) { const selected = this.instance(path); for (const entry of this.#cache.get(path)?.values() ?? []) { - entry.selection.validity?.update(sameInstance(entry.selection, selected)); + const same = sameInstance(entry.selection, selected); + entry.selection.validity?.update(same, !same && !(selected instanceof Blind)); } } } diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 8a71c880bb4..50730acd9f7 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -24,7 +24,7 @@ import { type AnnounceBroadcast, AnnounceInit, AnnounceOk, encodeAnnounceBroadca import { Group } from "./group.ts"; import { Probe } from "./probe.ts"; import { StreamId } from "./stream.ts"; -import { encodeSubscribeResponse, SubscribeEnd, SubscribeStart } from "./subscribe.ts"; +import { encodeSubscribeResponse, SubscribeEnd, type SubscribeResponse, SubscribeStart } from "./subscribe.ts"; import { SUBSCRIBE_SETUP_TIMEOUT_MS, Subscriber } from "./subscriber.ts"; import { Track, TrackInfo } from "./track.ts"; import { Version } from "./version.ts"; @@ -1514,7 +1514,8 @@ test("a listing of an epochless handle retired while its token waits never reach subscriber.close(); }); -// Announce PUBLISHER_A's epochless `pool` on stream 0, and return how to replace it with PUBLISHER_B's. +// Announce PUBLISHER_A's epochless `pool` on stream 0, and return how to withdraw it and how to +// announce PUBLISHER_B's in its place. async function epochlessPool(version: Version, streams: FakeStream[], announced: { next(): Promise }) { const send = async (encode: (writer: Writer) => Promise) => { const writer = new Writer( @@ -1531,7 +1532,7 @@ async function epochlessPool(version: Version, streams: FakeStream[], announced: encodeAnnounceBroadcast(writer, { status: "active", suffix: path, hops: [PUBLISHER_A] }, version), ); await announced.next(); - return async () => { + const withdraw = async () => { await send((writer) => encodeAnnounceBroadcast( writer, @@ -1540,49 +1541,68 @@ async function epochlessPool(version: Version, streams: FakeStream[], announced: ), ); await announced.next(); + }; + const announce = async () => { await send((writer) => encodeAnnounceBroadcast(writer, { status: "active", suffix: path, hops: [PUBLISHER_B] }, version), ); await announced.next(); }; + return { withdraw, replace: () => withdraw().then(announce) }; +} + +// Write the subscribe responses to the subscriber, FINishing the stream when asked. +async function respond(stream: FakeStream, version: Version, responses: SubscribeResponse[], fin = false) { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + for (const response of responses) await encodeSubscribeResponse(writer, response, version); + for (const chunk of chunks) stream.inbound.enqueue(chunk); + if (fin) stream.inbound.close(); } // A SUBSCRIBE without an epoch resolves whichever instance won the path when it arrived, so one -// unanswered when its instance is replaced is refused. One answered first stays on its instance. +// unanswered when another instance takes the path is refused, a withdrawal followed by a new +// announcement included. One answered first stays on its instance, and a withdrawal alone leaves +// it to its answer, so a track that finished as its broadcast ended still arrives. test.each([ - ["unanswered on lite-05 is refused", Version.DRAFT_05, false], - ["unanswered on lite-06 is refused", Version.DRAFT_06, false], - ["unanswered on lite-07 is refused", Version.DRAFT_07, false], - ["answered on lite-07 stays", Version.DRAFT_07, true], -] as const)("an epochless SUBSCRIBE %s at a takeover", async (_, version, answered) => { + ["unanswered on lite-05 is refused", Version.DRAFT_05, "replaced"], + ["unanswered on lite-06 is refused", Version.DRAFT_06, "replaced"], + ["unanswered on lite-07 is refused", Version.DRAFT_07, "replaced"], + ["answered on lite-07 stays", Version.DRAFT_07, "answered"], + ["unanswered on lite-05 finishes", Version.DRAFT_05, "withdrawn"], + ["unanswered on lite-06 finishes", Version.DRAFT_06, "withdrawn"], + ["unanswered on lite-07 finishes", Version.DRAFT_07, "withdrawn"], +] as const)("an epochless SUBSCRIBE %s at a takeover or withdrawal", async (_, version, outcome) => { const warn = spyOn(console, "warn").mockImplementation(() => {}); const { quic, streams, opened } = fakeSession(); const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); const announced = subscriber.announced(); try { await opened(0); - const replace = await epochlessPool(version, streams, announced); + const pool = await epochlessPool(version, streams, announced); const held = subscriber.consume(Path.from("pool/job")); const track = held.track("video").subscribe(); await answerTrackInfo(await opened(1), version); const subscribe = await opened(2); await subscribe.reading; - if (answered) { - const chunks: Uint8Array[] = []; - const writer = new Writer( - new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), - version, - ); - await encodeSubscribeResponse(writer, { start: new SubscribeStart(0) }, version); - for (const chunk of chunks) subscribe.inbound.enqueue(chunk); - await drainReactions(); - } - await replace(); - if (answered) { + if (outcome === "answered") { + await respond(subscribe, version, [{ start: new SubscribeStart(0) }]); + await drainReactions(); + await pool.replace(); await drainReactions(); expect(track.closed.peek()).toBeUndefined(); + } else if (outcome === "withdrawn") { + await pool.withdraw(); + await drainReactions(); + expect(track.closed.peek()).toBeUndefined(); + await respond(subscribe, version, [{ start: new SubscribeStart(0) }, { end: new SubscribeEnd(0) }], true); + expect(await track.closed).toBeNull(); } else { + await pool.replace(); const err = await track.closed; expect(err).toBeInstanceOf(StreamError); expect((err as StreamError).code).toBe(StreamCode.Unroutable); @@ -1597,14 +1617,15 @@ test.each([ } }); -// A TRACKS request without an epoch lists whichever instance won the path when it arrived. -test("a listing of an epochless handle retired before its answer is refused", async () => { +// A TRACKS request without an epoch lists whichever instance won the path when it arrived, so +// another instance taking the path refuses one still unanswered. +test("a listing of an epochless handle replaced before its answer is refused", async () => { const version = Version.DRAFT_07; const { quic, streams, opened } = fakeSession(); const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); const announced = subscriber.announced(); await opened(0); - const replace = await epochlessPool(version, streams, announced); + const pool = await epochlessPool(version, streams, announced); const held = subscriber.consume(Path.from("pool/job")); const tracks = held.tracks(); expect(await tracks.next()).toEqual({ kind: "synced" }); @@ -1612,7 +1633,7 @@ test("a listing of an epochless handle retired before its answer is refused", as const listing = await opened(1); await listing.reading; - await replace(); + await pool.replace(); const err = await event; expect(err).toBeInstanceOf(StreamError); expect((err as StreamError).code).toBe(StreamCode.Unroutable); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index ca9cb1ac763..c4b07185998 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -114,6 +114,9 @@ interface Consumed { readonly epoch?: Epoch.Valid; readonly id?: number; readonly valid?: AbortSignal; + // Aborted once another instance takes the path, which may answer a request without an epoch + // still unanswered. A withdrawal alone leaves the peer to answer it. + readonly answering?: AbortSignal; } interface SubscribeEntry { @@ -141,7 +144,7 @@ interface SubscribeEntry { // the two. held?: Stream; // Called at the SUBSCRIBE's first response (lite-05+). Until then a SUBSCRIBE without an - // epoch may be answered by whichever instance won the path, so its retirement refuses it. + // epoch may be answered by whichever instance won the path, so a replacement refuses it. answered?: () => void; } @@ -648,6 +651,9 @@ export class Subscriber { get valid() { return instance.validity?.signal; }, + get answering() { + return instance.validity?.answering; + }, }; const consumer = new ConsumeBroadcast(this, path, consumed); // Registered before anyone can list the broadcast, so no reader syncs without the peer. @@ -692,8 +698,9 @@ export class Subscriber { // fails, our grant stops covering the broadcast, or nobody lists the broadcast any more. // Returns without answering when setup was cut short, for the caller to look again. async #listTracks(broadcast: Path.Valid, consumed: Consumed, lister: Lister): Promise { - // Until the peer answers, the consumed instance must still be the selected one: a TRACKS - // request without an epoch lists whichever instance won the path when it arrived. + // The consumed instance must still be the selected one until the request is out, then + // not replaced until the peer answers: a TRACKS request without an epoch lists whichever + // instance won the path when it arrived. const valid = consumed.valid; if (valid?.aborted) { lister.abort(error(valid.reason)); @@ -751,7 +758,8 @@ export class Subscriber { await stream.writer.u53(StreamId.Tracks); await new TracksMessage(broadcast, consumed.epoch).encode(stream.writer, this.version); let synced = false; - let unanswered = cut; + const answering = consumed.answering; + let unanswered = answering ? AbortSignal.any([stop, answering]) : stop; for (;;) { const pause = listingBudget.poll(); if (pause !== undefined) await untilAborted(pause, stop); @@ -863,7 +871,7 @@ export class Subscriber { request: track.Request, state: SubscribeSetup, ) { - const { epoch, valid } = consumed; + const { epoch, valid, answering } = consumed; const subscription = request.subscription; // `timescale` stays undefined until TRACK_INFO (or, on older drafts, @@ -889,11 +897,12 @@ export class Subscriber { // so the timeout path can close them if they finish opening after the deadline, // and `cancel` ends a setup still running once the deadline passed, resetting // its TRACK stream so a peer that never answers can't hold one per attempt. - const retire = () => { - state.cancel.abort(valid?.reason); - state.producer?.close(error(valid?.reason)); - state.stream?.abort(error(valid?.reason)); + const retirer = (signal?: AbortSignal) => () => { + state.cancel.abort(signal?.reason); + state.producer?.close(error(signal?.reason)); + state.stream?.abort(error(signal?.reason)); }; + const retire = retirer(valid); valid?.addEventListener("abort", retire); if (valid?.aborted) retire(); const deadline = new Deadline(); @@ -927,9 +936,15 @@ export class Subscriber { } const { stream, entry } = opened; - // Older drafts already waited for SUBSCRIBE_OK. - if (hasTrackStream(this.version)) entry.answered = () => valid?.removeEventListener("abort", retire); - else valid?.removeEventListener("abort", retire); + valid?.removeEventListener("abort", retire); + // Sent but unanswered (lite-05+; older drafts already waited for SUBSCRIBE_OK), only a + // replacement refuses it, since that instance may be the one answering. + const replaced = retirer(answering); + if (hasTrackStream(this.version) && answering) { + answering.addEventListener("abort", replaced); + entry.answered = () => answering.removeEventListener("abort", replaced); + if (answering.aborted) replaced(); + } const producer = entry.track; try { // Watch for subscription changes and send SUBSCRIBE_UPDATE. Lite01/Lite02 @@ -987,7 +1002,7 @@ export class Subscriber { console.warn(`subscribe error: id=${id} broadcast=${broadcast} track=${request.name} error=${reason(e)}`); stream.abort(e); } finally { - valid?.removeEventListener("abort", retire); + answering?.removeEventListener("abort", replaced); entry.held?.close(); this.#subscribes.delete(id); } diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index e9c82848035..602c78a6034 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1452,7 +1452,7 @@ impl RequestValidity { /// Capture the lifetime of one pending request. pub fn start(&self) -> kio::Consumer<()> { match self.0.start() { - crate::model::broadcast::RequestValidity::Period(route, _) => route, + crate::model::broadcast::RequestValidity::Period(route, ..) => route, _ => unreachable!("a source watch always captures a route lease"), } } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 9c06741557a..a6b7f29b4ca 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1885,16 +1885,19 @@ struct TracksRecv { gate: crate::auth::Gate, /// The gate let the listing through, and this hop synced its readers. admitted: bool, - /// Until the peer answers, the source must still be the selected instance: a TRACKS - /// request without an epoch lists whichever instance won the path when it arrived. + /// Until the peer answers, what the source must still hold: the selected instance + /// until the request is out, then no replacement, since a TRACKS request without an + /// epoch lists whichever instance won the path when it arrived. unanswered: Option, } impl TracksRecv { /// Apply the peer's events until it ends the listing, or the error that ends it. fn poll(&mut self, source: &SourceServe, waiter: &kio::Waiter) -> Poll> { - if let Some(validity) = &self.unanswered { - validity.check(waiter)?; + match (&self.unanswered, &self.stream) { + (Some(validity), None) => validity.check(waiter)?, + (Some(validity), Some(_)) => validity.check_unanswered(waiter)?, + (None, _) => {} } ready!(self.gate.poll_ready(waiter)); if self.gate.poll_denied(waiter).is_ready() { @@ -3464,67 +3467,100 @@ mod tests { } } - /// A SUBSCRIBE_START that is only read after the source was replaced may be the - /// replacement's answer, so the retirement refuses the subscription instead. - #[moq_clock::test] - async fn an_answer_read_after_the_retirement_is_refused() { - for version in [Version::Lite05, Version::Lite06, Version::Lite07] { - let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); - let subscriber = Subscriber::new(SubscriberConfig { - runtime: crate::time::sim(), - session: crate::driver::Transport::new(session.clone()), - origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), - recv_bandwidth: None, - version, - peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), - peer_hop: None, - cost: None, - going_away: Default::default(), - auth: crate::auth::Link::new(false), - }); - let selected = kio::Producer::new(()); - let serve = TrackServe { - validity: crate::model::broadcast::RequestValidity::Route(selected.consume()), - route: None, - subscriber, - path: Path::new("room").to_owned(), - epoch: None, - name: "video".to_string(), - }; - let broadcast = crate::broadcast::Info::new().produce(); - let request = broadcast.reserve_track("video").unwrap(); - let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); - let mut reader = broadcast - .consume() - .track("video") - .unwrap() - .subscribe(None) - .await - .unwrap(); - let gate = serve.gate(); - let mut running = TrackServeRun { - serve, - state: TrackRunState::Serve(serving), - gate, - }; - assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); + /// Answer a SUBSCRIBE without an epoch with a finished, empty track, but only read the + /// answer after `retire` changed its source's selection, then return what the reader got. + async fn answer_after( + version: Version, + retire: impl FnOnce(&mut crate::model::TestSelection), + ) -> Option, Error>> { + let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let mut selection = crate::model::TestSelection::new(); + let serve = TrackServe { + validity: selection.validity(), + route: None, + subscriber, + path: Path::new("room").to_owned(), + epoch: None, + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let request = broadcast.reserve_track("video").unwrap(); + let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); + let mut reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let gate = serve.gate(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Serve(serving), + gate, + }; + assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); - let mut start = Vec::new(); - lite::SubscribeResponse::Start(lite::SubscribeStart { - group: 0, - largest: None, - }) - .encode(&mut crate::coding::Encoder::new(&mut start, version.into()), version) + let mut answer = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut answer, version.into()); + lite::SubscribeResponse::Start(lite::SubscribeStart { + group: 0, + largest: None, + }) + .encode(&mut encoder, version) + .unwrap(); + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0 }) + .encode(&mut encoder, version) .unwrap(); - session.push(&start); - drop(selected); - let _ = kio::Task::poll(&mut running, &kio::Waiter::noop()); + session.push(&answer); + retire(&mut selection); + let _ = kio::Task::poll(&mut running, &kio::Waiter::noop()); + + reader + .recv_group() + .now_or_never() + .map(|res| res.map(|group| group.map(|group| group.sequence))) + } - let res = reader.recv_group().now_or_never().map(|res| res.map(|_| ())); + /// An answer read only after another instance took the path may be that instance's, + /// so the replacement refuses the subscription instead, as does a withdrawal followed + /// by a new announcement. + #[moq_clock::test] + async fn an_answer_read_after_a_replacement_is_refused() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let res = answer_after(version, |selection| selection.replace()).await; + assert!(matches!(res, Some(Err(Error::Unroutable))), "{version:?}: {res:?}"); + let res = answer_after(version, |selection| { + selection.withdraw(); + selection.replace(); + }) + .await; assert!(matches!(res, Some(Err(Error::Unroutable))), "{version:?}: {res:?}"); } } + /// A withdrawal alone leaves the SUBSCRIBE to its answer: the publisher concludes a + /// subscription in flight, so a track that finished as its broadcast ended arrives whole. + #[moq_clock::test] + async fn an_answer_read_after_a_withdrawal_is_kept() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let res = answer_after(version, |selection| selection.withdraw()).await; + assert!(matches!(res, Some(Ok(None))), "{version:?}: {res:?}"); + } + } + /// Lite-07's SUBSCRIBE_START says where the track is, so a reader knows its largest /// frame before any group arrives. The FIN with no SUBSCRIBE_END then aborts the track, /// and the stream ends with it. @@ -4123,15 +4159,15 @@ mod tests { assert_eq!(session.log.bi_opens(), 0); } - /// An instance retired while its listing is on the wire but unanswered lists nothing: - /// the peer may list the replacement that won the path. + /// A listing on the wire but unanswered when another instance takes the path lists + /// nothing: the peer may list that instance. #[moq_clock::test] - async fn an_unanswered_listing_of_a_retired_instance_is_refused() { + async fn an_unanswered_listing_of_a_replaced_instance_is_refused() { let open = kio::Producer::new(true); let session = SinkSession::gated_bi(open.consume()); - let selected = kio::Producer::new(()); - let validity = crate::model::broadcast::RequestValidity::Route(selected.consume()); - let (mut serve, consumer, _route) = listed_source(session.clone(), crate::auth::Link::new(false), validity); + let mut selection = crate::model::TestSelection::new(); + let (mut serve, consumer, _route) = + listed_source(session.clone(), crate::auth::Link::new(false), selection.validity()); let mut tracks = consumer.tracks().unwrap(); assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_pending()); @@ -4141,7 +4177,7 @@ mod tests { Some(Ok(Some(crate::broadcast::TracksEvent::Synced))) )); - drop(selected); + selection.replace(); let _ = kio::Task::poll(&mut serve, &kio::Waiter::noop()); assert!(matches!(tracks.next().now_or_never(), Some(Err(Error::Unroutable)))); assert_eq!(session.log.resets(), vec![crate::StreamError::Unroutable.to_code()]); @@ -6038,7 +6074,7 @@ struct SubStream { wanted: Option, /// Until the publisher's first response, what the source must still hold: a SUBSCRIBE /// without an epoch resolves whichever instance wins the path when it arrives, so one - /// still unanswered once the source is replaced may be answered by the replacement. + /// still unanswered once another instance takes the path may be answered by it. unanswered: Option, } @@ -7461,13 +7497,14 @@ impl ServeLoop { ServeMode::Select => { let mut cx = waiter.context(); - // A SUBSCRIBE still unanswered once its source is replaced may have resolved - // the replacement: refuse it. Checked before reading the stream, so an answer - // that only arrived alongside the replacement cannot claim it, and before the - // flush, so an update parked on flow control cannot hold the refusal back. + // A SUBSCRIBE still unanswered once another instance takes the path may have + // resolved it: refuse it. A withdrawal alone leaves it to its answer. Checked + // before reading the stream, so an answer that only arrived alongside the + // replacement cannot claim it, and before the flush, so an update parked on + // flow control cannot hold the refusal back. if let Sub::Active(active) = &self.sub && let Some(validity) = &active.unanswered - && let Err(err) = validity.check(waiter) + && let Err(err) = validity.check_unanswered(waiter) { tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "unanswered subscribe refused: instance replaced"); return Poll::Ready(ServeEnd::GiveBack(err)); diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 12474e48c46..19024e4d983 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -571,8 +571,12 @@ pub(crate) enum RequestValidity { Unrestricted, Route(kio::Consumer<()>), Selected(std::sync::Arc), - /// One period of a selection, captured by [`Self::start`]. - Period(kio::Consumer<()>, std::sync::Arc), + /// One period of a selection and its answering lease, captured by [`Self::start`]. + Period( + kio::Consumer<()>, + kio::Consumer<()>, + std::sync::Arc, + ), } impl RequestValidity { @@ -592,7 +596,7 @@ impl RequestValidity { /// Capture one selection period so a pending request never revives after a replacement. pub(crate) fn start(&self) -> Self { match self { - Self::Selected(selected) => Self::Period(selected.current(), selected.clone()), + Self::Selected(selected) => Self::Period(selected.current(), selected.answering(), selected.clone()), other => other.clone(), } } @@ -601,7 +605,7 @@ impl RequestValidity { match self { Self::Route(route) if route.poll_closed(waiter).is_ready() => Err(crate::Error::Unroutable), // A replacement retires the period only once its held `Restart` is released. - Self::Period(route, selected) + Self::Period(route, _, selected) if route.poll_closed(waiter).is_ready() || selected.poll_retired(waiter).is_ready() => { Err(crate::Error::Unroutable) @@ -610,6 +614,22 @@ impl RequestValidity { _ => Ok(()), } } + + /// [`Self::check`] for a request already sent but not yet answered: only another + /// instance taking the path refuses it, since without an epoch that instance may be + /// the one answering. A withdrawal leaves it to its answer, as moq-lite has the + /// publisher conclude in-flight requests. + pub(crate) fn check_unanswered(&self, waiter: &kio::Waiter) -> Result<(), crate::Error> { + match self { + Self::Period(_, answering, selected) + if answering.poll_closed(waiter).is_ready() || selected.poll_retired(waiter).is_ready() => + { + Err(crate::Error::Unroutable) + } + Self::Selected(_) => self.start().check_unanswered(waiter), + _ => Ok(()), + } + } } #[cfg(test)] diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 5a842b590e7..ea6c3debb60 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -82,4 +82,4 @@ pub(crate) use origin_impl::{InterestChange, Interests}; pub(crate) use origin_impl::AnnounceProducer; #[cfg(all(test, feature = "async"))] -pub(crate) use origin_impl::ProduceTest; +pub(crate) use origin_impl::{ProduceTest, TestSelection}; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 96393b568c7..dc19c268fe2 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4142,6 +4142,10 @@ struct Selected { prefix: PathOwned, /// The selection period, closed when the peer stops selecting the instance. current: Option>, + /// Open until another instance of the peer takes the path, which may answer a + /// request still unanswered. A withdrawal alone leaves it open, and a period the + /// instance wins again keeps it. + answering: Option>, /// The [`Release`] that ends `current`: the peer replaced the instance with a /// route without an epoch, whose `Restart` is held until then. retiring: Option, @@ -4157,34 +4161,49 @@ enum Change { Retire(Release), /// The peer no longer selects the instance, effective at once. End, + /// Another instance of the peer took the path, effective at once. + Replace, } impl Selected { /// How the period changes now that the peer's best route is `best`, if it does. fn change(&self, best: Option<&RouteEntry>, hold: &Hold) -> Option { - if best.is_some_and(|entry| entry.instance() == self.instance) { + let replaced = self.retiring.as_ref().is_some_and(Release::released); + let Some(best) = best else { + // A released replacement already took the path before this withdrawal. + self.current.as_ref()?; + return Some(if replaced { Change::Replace } else { Change::End }); + }; + if best.instance() == self.instance { return match (&self.current, &self.retiring) { (Some(_), None) => None, - (Some(_), Some(release)) if !release.released() => Some(Change::Continue), + (Some(_), Some(_)) if !replaced => Some(Change::Continue), _ => Some(Change::Start), }; } - self.current.as_ref()?; + if self.current.is_none() { + // Withdrawn earlier: the instance taking the path now may answer what is left. + return self.answering.is_some().then_some(Change::Replace); + } // Announce cursors hold a restart at the same prefix onto a route without an // epoch, so the period lasts as long. Anything else they deliver at once. - let held = best.is_some_and(|entry| entry.epoch.is_none() && entry.prefix == self.prefix); + let held = best.epoch.is_none() && best.prefix == self.prefix; match (held, &self.retiring) { (true, Some(_)) => None, - (true, None) => Some(hold.release().map_or(Change::End, Change::Retire)), - (false, _) => Some(Change::End), + (true, None) => Some(hold.release().map_or(Change::Replace, Change::Retire)), + (false, _) => Some(Change::Replace), } } fn apply(&mut self, change: Change) { match change { Change::Start => { + // Past a released replacement, the instance that answered in between + // was the replacement. + if self.retiring.take().is_some() || self.answering.is_none() { + self.answering = Some(kio::Producer::default()); + } self.current = Some(kio::Producer::default()); - self.retiring = None; } Change::Continue => self.retiring = None, Change::Retire(release) => self.retiring = Some(release), @@ -4192,6 +4211,11 @@ impl Selected { self.current = None; self.retiring = None; } + Change::Replace => { + self.current = None; + self.answering = None; + self.retiring = None; + } } } } @@ -4216,6 +4240,17 @@ impl Selection { .unwrap_or_else(|| kio::Producer::new(()).consume()) } + /// Closed once another instance takes the path, for a request already sent. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(super) fn answering(&self) -> kio::Consumer<()> { + self.signal + .read() + .answering + .as_ref() + .map(kio::Producer::consume) + .unwrap_or_else(|| kio::Producer::new(()).consume()) + } + /// `Ready` once the current period's replacement is released, registering /// `waiter` until then. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -5415,6 +5450,7 @@ impl Dynamic { instance, prefix, current: active.then(kio::Producer::default), + answering: active.then(kio::Producer::default), retiring: None, }); let signal = producer.consume(); @@ -7163,6 +7199,49 @@ impl ProduceTest for Hop { } } +/// Test-only: one peer's route selected for `room/a`, which a test withdraws or +/// replaces to drive the [`broadcast::RequestValidity`](crate::model::broadcast::RequestValidity) +/// it hands out. +#[cfg(all(test, feature = "async"))] +pub(crate) struct TestSelection { + origin: Producer, + route: Option, + replacement: Option, +} + +#[cfg(all(test, feature = "async"))] +impl TestSelection { + const PEER: u64 = 2; + + pub(crate) fn new() -> Self { + let origin = Hop::new(1).unwrap().produce(); + let route = origin + .dynamic_route("room", Route::default().with_via(Hop::new(Self::PEER).unwrap())) + .unwrap(); + Self { + origin, + route: Some(route), + replacement: None, + } + } + + /// What a request for `room/a` through the selected route must still hold. + pub(crate) fn validity(&self) -> crate::model::broadcast::RequestValidity { + crate::model::broadcast::RequestValidity::for_source(self.route.as_ref().unwrap(), &Path::new("room/a")) + } + + /// The peer withdraws the route. + pub(crate) fn withdraw(&mut self) { + self.route = None; + } + + /// Another instance of the peer takes the path. + pub(crate) fn replace(&mut self) { + let route = Route::default().with_via(Hop::new(Self::PEER).unwrap()); + self.replacement = Some(self.origin.dynamic_route("room/a", route).unwrap()); + } +} + #[cfg(test)] mod tests { use super::*; @@ -13035,6 +13114,43 @@ mod tests { assert!(fresh.is_closed()); } + /// The answering lease outlives a withdrawal, since the peer still answers for the + /// instance it retracted, and ends once another instance takes the path. + #[moq_clock::test] + async fn source_selection_answering_ends_only_on_a_replacement() { + let producer = origin(1).produce(); + let path = Path::new("room/a"); + let first = producer + .dynamic_route("room", Route::default().with_via(origin(2))) + .unwrap(); + let selected = first.request_validity(&path).unwrap(); + let answering = selected.answering(); + // A route that loses to the selected one replaces nothing. + let worse = producer + .dynamic_route("room", Route::default().with_via(origin(2)).with_cost(Cost::from(7))) + .unwrap(); + assert!(!answering.is_closed()); + drop(worse); + + let replacement = producer + .dynamic_route("room/a", Route::default().with_via(origin(2))) + .unwrap(); + assert!(answering.is_closed()); + // Winning again opens a new lease, which a withdrawal leaves open. + drop(replacement); + let answering = selected.answering(); + assert!(!answering.is_closed()); + drop(first); + assert!(selected.current().is_closed()); + assert!(!answering.is_closed()); + + // Announced again, the path has a new instance: a replacement. + let _again = producer + .dynamic_route("room", Route::default().with_via(origin(2))) + .unwrap(); + assert!(answering.is_closed()); + } + #[moq_clock::test] async fn source_selection_unregisters_without_retaining_the_source() { let producer = origin(1).produce(); diff --git a/rs/moq-net/tests/integration/session_close.rs b/rs/moq-net/tests/integration/session_close.rs index 9f9bb045c30..bc08b6a9f3a 100644 --- a/rs/moq-net/tests/integration/session_close.rs +++ b/rs/moq-net/tests/integration/session_close.rs @@ -103,17 +103,6 @@ async fn setup_with_gate(version: &str, gate: Option>) -> Se } } -/// Open a group the caller holds unfinished, so the publisher answers the SUBSCRIBE: the -/// withdrawal a close starts with refuses an unanswered SUBSCRIBE without an epoch, which -/// would leave nothing to drain. -async fn answer(track: &moq_net::track::Producer) -> moq_net::group::Producer { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); - // Paused, the clock only moves once every task is idle, so the answer has landed. - moq_clock::sleep(Duration::from_millis(100)).await; - group -} - /// TRACK_INFO creates demand before any SUBSCRIBE, so demand alone cannot gate a drain test. #[moq_clock::test] async fn track_info_interest_precedes_a_subscription() { @@ -192,8 +181,9 @@ async fn ietf_close_gives_up_on_a_live_track() { } async fn close_gives_up_on_a_live_track_for(version: &str) { - let Setup { pair, track, .. } = setup(version).await; - let _live = answer(&track).await; + let Setup { + pair, track: _track, .. + } = setup(version).await; let started = moq_clock::now(); let res = moq_clock::timeout(TIMEOUT, pair.client.close()) @@ -215,8 +205,9 @@ async fn ietf_abort_cuts_a_drain_short() { } async fn abort_cuts_a_drain_short_for(version: &str) { - let Setup { pair, track, .. } = setup(version).await; - let _live = answer(&track).await; + let Setup { + pair, track: _track, .. + } = setup(version).await; let other = pair.client.clone(); let started = moq_clock::now(); diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index 8332a1968b8..c4d0d1de92d 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -1078,6 +1078,7 @@ async fn quic_client_close_keeps_predecessor_handover() { .parse() .unwrap(); + let (subscribed_tx, subscribed_rx) = tokio::sync::oneshot::channel::<()>(); let client = std::thread::spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -1106,12 +1107,7 @@ async fn quic_client_close_keeps_predecessor_handover() { while track.subscription().is_none() { track.subscription_changed().await.expect("track closed"); } - // An unfinished group answers the SUBSCRIBE: one without an epoch still unanswered - // at the handover is refused, which would leave nothing to drain. - let mut group = track.append_group().expect("failed to append group"); - group - .write_frame(moq_tokio::moq_net::Timestamp::ZERO, b"live".as_slice()) - .expect("failed to write frame"); + subscribed_tx.send(()).unwrap(); // Close while the old session is still inside its handover window. while connection.status().await.expect("connection stopped") != moq_tokio::Status::Migrating {} @@ -1148,12 +1144,12 @@ async fn quic_client_close_keeps_predecessor_handover() { .await .expect("subscribe timed out") .expect("subscribe failed"); - // Receiving is what sends the subscription. - let _group = tokio::time::timeout(TIMEOUT, track.recv_group()) + // Receiving is what sends the subscription, so read in the background. + let _received = tokio::spawn(async move { track.recv_group().await.map(|group| group.is_some()) }); + tokio::time::timeout(TIMEOUT, subscribed_rx) .await - .expect("the subscription was never answered") - .expect("subscribe failed") - .expect("track ended"); + .expect("the subscription never reached the client") + .expect("client thread panicked"); first .drain() From 9ddd9650ff0e59d7065769ee7cdf856c21805955 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 26 Oct 2026 11:14:00 -0700 Subject: [PATCH 9/9] fix(net): allow check_unanswered without the async feature Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/broadcast.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 19024e4d983..76922008349 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -619,6 +619,7 @@ impl RequestValidity { /// instance taking the path refuses it, since without an epoch that instance may be /// the one answering. A withdrawal leaves it to its answer, as moq-lite has the /// publisher conclude in-flight requests. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn check_unanswered(&self, waiter: &kio::Waiter) -> Result<(), crate::Error> { match self { Self::Period(_, answering, selected)