diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 01448beab0b..4efe6557b10 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -116,10 +116,13 @@ authorization and hidden names work exactly as without one. without an epoch keeps its subscriptions until that route goes, and a track that fails ends instead of resuming, so request the path again. - **Another instance restarts.** A newer epoch, or, without one, another route - winning the path, is announced as a restart. Subscriptions already open stay - on the old instance until dropped or none of its routes remain, which ends - them with an error, while new requests get the new one, so a player follows - the restart by requesting the path again. + winning the path, is announced as a restart. Subscriptions already answered + stay on the old instance until dropped or none of its routes remain, which + ends them with an error, while new requests get the new one, so a player + follows the restart by requesting the path again. Without an epoch, a request + still unanswered when another instance takes the path, even after a + withdrawal, ends the same way, since that instance may be the one answering + it. A withdrawal alone leaves it to its answer. Held handles can still read their old cached content while a subscription on the old instance stays open, but cannot open new epochless requests once the restart is delivered. Request the path again for diff --git a/js/net/src/consume.ts b/js/net/src/consume.ts index d3c90ac1fde..3f45e311b4a 100644 --- a/js/net/src/consume.ts +++ b/js/net/src/consume.ts @@ -6,20 +6,35 @@ import { compareRouteCandidates, coveringPrefixes } from "./internal.ts"; import type * as Path from "./path.ts"; import { type Instance, sameInstance } from "./wire.ts"; +// Unroutable, as in Rust: the instance going away is an end, not a failure. +function retired(): StreamError { + return new StreamError(StreamCode.Unroutable, { message: "broadcast instance retired" }); +} + class Validity { #current = new AbortController(); + #answering = new AbortController(); get signal(): AbortSignal { return this.#current.signal; } - update(allowed: boolean): void { + /** + * Aborted once another instance takes the path, which may answer a request still unanswered. + * A withdrawal alone leaves it, and the instance winning again keeps it. + */ + get answering(): AbortSignal { + return this.#answering.signal; + } + + update(allowed: boolean, replaced = false): void { if (allowed) { if (this.#current.signal.aborted) this.#current = new AbortController(); - } else if (!this.#current.signal.aborted) { - // Unroutable, as in Rust: the instance going away is an end, not a failure. - this.#current.abort(new StreamError(StreamCode.Unroutable, { message: "broadcast instance retired" })); + if (this.#answering.signal.aborted) this.#answering = new AbortController(); + return; } + if (!this.#current.signal.aborted) this.#current.abort(retired()); + if (replaced && !this.#answering.signal.aborted) this.#answering.abort(retired()); } } @@ -159,7 +174,8 @@ export class BroadcastCache { for (const path of this.#paths.get(prefix) ?? []) { const selected = this.instance(path); for (const entry of this.#cache.get(path)?.values() ?? []) { - entry.selection.validity?.update(sameInstance(entry.selection, selected)); + const same = sameInstance(entry.selection, selected); + entry.selection.validity?.update(same, !same && !(selected instanceof Blind)); } } } diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index ecd9f48a22a..21928323ace 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -25,7 +25,13 @@ import { type AnnounceBroadcast, AnnounceInit, AnnounceOk, encodeAnnounceBroadca import { Group } from "./group.ts"; import { Probe } from "./probe.ts"; import { StreamId } from "./stream.ts"; -import { encodeSubscribeResponse, SubscribeDrop, SubscribeEnd, SubscribeStart } from "./subscribe.ts"; +import { + encodeSubscribeResponse, + SubscribeDrop, + SubscribeEnd, + type SubscribeResponse, + SubscribeStart, +} from "./subscribe.ts"; import { SUBSCRIBE_SETUP_TIMEOUT_MS, Subscriber } from "./subscriber.ts"; import { Track, TrackInfo } from "./track.ts"; import { Version } from "./version.ts"; @@ -1515,6 +1521,136 @@ test("a listing of an epochless handle retired while its token waits never reach subscriber.close(); }); +// Announce PUBLISHER_A's epochless `pool` on stream 0, and return how to withdraw it and how to +// announce PUBLISHER_B's in its place. +async function epochlessPool(version: Version, streams: FakeStream[], announced: { next(): Promise }) { + const send = async (encode: (writer: Writer) => Promise) => { + const writer = new Writer( + new WritableStream({ + write: (chunk) => streams[0].inbound.enqueue(new Uint8Array(chunk)), + }), + version, + ); + await encode(writer); + }; + const path = Path.from("pool"); + await send((writer) => new AnnounceOk(PEER, 0).encode(writer, version)); + await send((writer) => + encodeAnnounceBroadcast(writer, { status: "active", suffix: path, hops: [PUBLISHER_A] }, version), + ); + await announced.next(); + const withdraw = async () => { + await send((writer) => + encodeAnnounceBroadcast( + writer, + version === Version.DRAFT_05 ? { status: "ended", suffix: path } : { status: "endedId", id: 0n }, + version, + ), + ); + await announced.next(); + }; + const announce = async () => { + await send((writer) => + encodeAnnounceBroadcast(writer, { status: "active", suffix: path, hops: [PUBLISHER_B] }, version), + ); + await announced.next(); + }; + return { withdraw, replace: () => withdraw().then(announce) }; +} + +// Write the subscribe responses to the subscriber, FINishing the stream when asked. +async function respond(stream: FakeStream, version: Version, responses: SubscribeResponse[], fin = false) { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + for (const response of responses) await encodeSubscribeResponse(writer, response, version); + for (const chunk of chunks) stream.inbound.enqueue(chunk); + if (fin) stream.inbound.close(); +} + +// A SUBSCRIBE without an epoch resolves whichever instance won the path when it arrived, so one +// unanswered when another instance takes the path is refused, a withdrawal followed by a new +// announcement included. One answered first stays on its instance, and a withdrawal alone leaves +// it to its answer, so a track that finished as its broadcast ended still arrives. +test.each([ + ["unanswered on lite-05 is refused", Version.DRAFT_05, "replaced"], + ["unanswered on lite-06 is refused", Version.DRAFT_06, "replaced"], + ["unanswered on lite-07 is refused", Version.DRAFT_07, "replaced"], + ["answered on lite-07 stays", Version.DRAFT_07, "answered"], + ["unanswered on lite-05 finishes", Version.DRAFT_05, "withdrawn"], + ["unanswered on lite-06 finishes", Version.DRAFT_06, "withdrawn"], + ["unanswered on lite-07 finishes", Version.DRAFT_07, "withdrawn"], +] as const)("an epochless SUBSCRIBE %s at a takeover or withdrawal", async (_, version, outcome) => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const announced = subscriber.announced(); + try { + await opened(0); + const pool = await epochlessPool(version, streams, announced); + const held = subscriber.consume(Path.from("pool/job")); + const track = held.track("video").subscribe(); + await answerTrackInfo(await opened(1), version); + const subscribe = await opened(2); + await subscribe.reading; + + if (outcome === "answered") { + await respond(subscribe, version, [{ start: new SubscribeStart(0) }]); + await drainReactions(); + await pool.replace(); + await drainReactions(); + expect(track.closed.peek()).toBeUndefined(); + } else if (outcome === "withdrawn") { + await pool.withdraw(); + await drainReactions(); + expect(track.closed.peek()).toBeUndefined(); + await respond(subscribe, version, [{ start: new SubscribeStart(0) }, { end: new SubscribeEnd(0) }], true); + expect(await track.closed).toBeNull(); + } else { + await pool.replace(); + const err = await track.closed; + expect(err).toBeInstanceOf(StreamError); + expect((err as StreamError).code).toBe(StreamCode.Unroutable); + await subscribe.aborted; + } + track.close(); + held.close(); + } finally { + announced.close(); + subscriber.close(); + warn.mockRestore(); + } +}); + +// A TRACKS request without an epoch lists whichever instance won the path when it arrived, so +// another instance taking the path refuses one still unanswered. +test("a listing of an epochless handle replaced before its answer is refused", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const announced = subscriber.announced(); + await opened(0); + const pool = await epochlessPool(version, streams, announced); + const held = subscriber.consume(Path.from("pool/job")); + const tracks = held.tracks(); + expect(await tracks.next()).toEqual({ kind: "synced" }); + const event = tracks.next().catch((err: unknown) => err); + const listing = await opened(1); + await listing.reading; + + await pool.replace(); + const err = await event; + expect(err).toBeInstanceOf(StreamError); + expect((err as StreamError).code).toBe(StreamCode.Unroutable); + await listing.aborted; + tracks.close(); + held.close(); + announced.close(); + subscriber.close(); +}); + test("a listing whose readers leave as its token settles never reaches the wire", async () => { const { quic, streams } = fakeSession(); let release!: () => void; diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 84f8ef5441a..c45ed3a4762 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -115,6 +115,9 @@ interface Consumed { readonly epoch?: Epoch.Valid; readonly id?: number; readonly valid?: AbortSignal; + // Aborted once another instance takes the path, which may answer a request without an epoch + // still unanswered. A withdrawal alone leaves the peer to answer it. + readonly answering?: AbortSignal; } interface SubscribeEntry { @@ -147,6 +150,9 @@ interface SubscribeEntry { // the SUBSCRIBE has its first response, so the publisher's demand never lapses between // the two. held?: Stream; + // Called at the SUBSCRIBE's first response (lite-05+). Until then a SUBSCRIBE without an + // epoch may be answered by whichever instance won the path, so a replacement refuses it. + answered?: () => void; } /** Subscribe to the peer and own its receive-side protocol tasks. @@ -652,6 +658,9 @@ export class Subscriber { get valid() { return instance.validity?.signal; }, + get answering() { + return instance.validity?.answering; + }, }; const consumer = new ConsumeBroadcast(this, path, consumed); // Registered before anyone can list the broadcast, so no reader syncs without the peer. @@ -696,8 +705,9 @@ export class Subscriber { // fails, our grant stops covering the broadcast, or nobody lists the broadcast any more. // Returns without answering when setup was cut short, for the caller to look again. async #listTracks(broadcast: Path.Valid, consumed: Consumed, lister: Lister): Promise { - // Until the request is out, the consumed instance must still be the selected one: a TRACKS - // request without an epoch would list its replacement. + // The consumed instance must still be the selected one until the request is out, then + // not replaced until the peer answers: a TRACKS request without an epoch lists whichever + // instance won the path when it arrived. const valid = consumed.valid; if (valid?.aborted) { lister.abort(error(valid.reason)); @@ -755,10 +765,13 @@ export class Subscriber { await stream.writer.u53(StreamId.Tracks); await new TracksMessage(broadcast, consumed.epoch).encode(stream.writer, this.version); let synced = false; + const answering = consumed.answering; + let unanswered = answering ? AbortSignal.any([stop, answering]) : stop; for (;;) { const pause = listingBudget.poll(); if (pause !== undefined) await untilAborted(pause, stop); - const event = await untilAborted(decodeTracksEventMaybe(stream.reader, this.version), stop); + const event = await untilAborted(decodeTracksEventMaybe(stream.reader, this.version), unanswered); + unanswered = stop; if (!event) { // The peer's broadcast ended, whether or not its retraction has arrived: end the // listing and let go of what it listed. FIN our half too, so the stream is done. @@ -865,7 +878,7 @@ export class Subscriber { request: track.Request, state: SubscribeSetup, ) { - const { epoch, valid } = consumed; + const { epoch, valid, answering } = consumed; const subscription = request.subscription; // `timescale` stays undefined until TRACK_INFO (or, on older drafts, @@ -891,11 +904,12 @@ export class Subscriber { // so the timeout path can close them if they finish opening after the deadline, // and `cancel` ends a setup still running once the deadline passed, resetting // its TRACK stream so a peer that never answers can't hold one per attempt. - const retire = () => { - state.cancel.abort(valid?.reason); - state.producer?.close(error(valid?.reason)); - state.stream?.abort(error(valid?.reason)); + const retirer = (signal?: AbortSignal) => () => { + state.cancel.abort(signal?.reason); + state.producer?.close(error(signal?.reason)); + state.stream?.abort(error(signal?.reason)); }; + const retire = retirer(valid); valid?.addEventListener("abort", retire); if (valid?.aborted) retire(); const deadline = new Deadline(); @@ -928,8 +942,16 @@ export class Subscriber { return; } - valid?.removeEventListener("abort", retire); const { stream, entry } = opened; + valid?.removeEventListener("abort", retire); + // Sent but unanswered (lite-05+; older drafts already waited for SUBSCRIBE_OK), only a + // replacement refuses it, since that instance may be the one answering. + const replaced = retirer(answering); + if (hasTrackStream(this.version) && answering) { + answering.addEventListener("abort", replaced); + entry.answered = () => answering.removeEventListener("abort", replaced); + if (answering.aborted) replaced(); + } const producer = entry.track; try { // Watch for subscription changes and send SUBSCRIBE_UPDATE. Lite01/Lite02 @@ -991,6 +1013,7 @@ export class Subscriber { console.warn(`subscribe error: id=${id} broadcast=${broadcast} track=${request.name} error=${reason(e)}`); stream.abort(e); } finally { + answering?.removeEventListener("abort", replaced); entry.held?.close(); this.#subscribes.delete(id); } @@ -1408,6 +1431,8 @@ export class Subscriber { // The publisher answered, so its demand stands on the subscription from here. entry.held?.close(); entry.held = undefined; + entry.answered?.(); + entry.answered = undefined; if (!resp) { // A bounded range may have widened since, in an update the publisher never read. const covered = hasRanges(this.version) && entry.bounded; diff --git a/quest/m1/README.md b/quest/m1/README.md index 4f079f97864..431aacf59b1 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -159,7 +159,6 @@ surface and zero-copy texture import, land before the cut. - [Generated JS API reference](/quest/m1/js-api-reference.md) - the doc site renders an API reference for every `@moq/*` package from API Extractor's doc model - [Interop coverage](/quest/m1/interop-coverage/README.md) - the riskiest moq-transport features moq-net exposes get cross-implementation tests over drafts 18 and 22 - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - A/A and deliberately slowed Nightly runs set the paired comparison's threshold, so its regression alerts can be trusted -- [HLS takeover leak](/quest/m1/hls-takeover-leak.md) - a replaced instance's recording cursor never yields the replacement's media, and ends or fails instead of stalling - [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - nightly ffprobe checks that advertised timing, sync flags, and bytes match the media served - [App-limited BBR3 estimate ages](/quest/m1/bbr-app-limited.md) - an app-limited BBR3 estimate falls within seconds after the path drops - [Seeded BBR3 in tests](/quest/m1/bbr-test-seed.md) - unseeded `Bbr3Config::default()` tests replay the same randomness diff --git a/quest/m1/hls-takeover-leak.md b/quest/m1/hls-takeover-leak.md deleted file mode 100644 index 0296d31b336..00000000000 --- a/quest/m1/hls-takeover-leak.md +++ /dev/null @@ -1,211 +0,0 @@ -# [M] HLS takeover leaks the replacement's media - -## Goal - -After a new publisher instance takes over a broadcast path, a recording cursor on the -replaced instance never yields the replacement's media. It drains the old run's own -segments, then ends cleanly or fails with a typed error. It never stalls. - -The `takeover` scenario below lands in `rs/moq-hls/tests/integration/restart.rs` with -the fix, as part of the session suite. - -## Plan - -Found 2026-10-25 by the session suite. The takeover scenario fails on `main`. - -**Symptom.** The old run publishes three complete segments and leaves a GOP open at 12s. -A new instance takes over `live` and publishes its own groups from zero. The old run -finishes. The old cursor reads segments 1 and 2 with the old bytes. Segment 3, the old -run's final GOP (group 6), comes back with the new run's bytes. - -- If the new run has not completed its own group 6, the old cursor stalls on segment 3 - instead. -- The leak also happens when the old run finishes before the new one announces. -- It reproduces on moq-lite-06 (the default) and moq-lite-07-wip, with an epochless old - route and a new route with an epoch. -- Over HTTP, the old instance's URLs answer 404 once it is replaced, so only the cursor - leaks. - -**Where the fetch goes.** Temporary logging in `Rendition::fetch` -(`rs/moq-hls/src/export/rendition.rs`) showed the segment-3 fetch going through the -replaced broadcast: its epoch was `None` and `is_closed()` was true. The fetch reads -through the track that `Handle::capture` pinned with a `track.query()` before the -takeover, yet `fetch_group(6)` returned the replacement's group with the same sequence. - -**A bare moq-net reproduction did not leak.** The setup used lite-07, an epochless old -broadcast, and a held `track.query()` taken before the replacement, with: - -- a subscription on the old track, with its in-flight group either read or dropped unread; -- the replacement announced with an epoch and writing groups 0 to 2; -- the old group 1 finished after the takeover; -- in one variant, a concurrent subscription to the replacement. - -`fetch_group(1)` on the old query returned the old bytes. Groups the old subscription -never delivered answered `Unroutable`. So the leak needs something the export adds. - -**Where I looked:** - -- `rs/moq-hls/src/export/rendition.rs`: `Media`, `Handle::capture`, `poll_sync`, - `poll_track`, `Rendition::fetch`, and the free `fetch`. Pinning and window-ending on - close apply only to siblings (`self.sibling`). The catalog's own broadcast relies on - the source pin. -- `rs/moq-hls/src/export/segments.rs`: `Consumer::poll_feed`. The cursor's feed - subscribes through the binding (`poll_subscribe`, `FEED_MAX_DELAY`) and drops each - group unread. Its doc says a resumed subscription "only pulls while it is read". -- `rs/moq-mux/src/source.rs`: `Source::new` pins the catalog path to the served - instance; `Binding::poll_broadcast` returns the pin. -- `rs/moq-net-tests/tests/integration/restart.rs`: `retired_fetch` covers a retired handle - whose query is taken after the restart, not one held before it. -- `rs/moq-net/src/lite/subscriber.rs`: `SubscriptionCleanup` tears subscriptions down in - `HashMap` order. That explains why the disconnect failure order varies, not this leak. - -**Leading hypothesis, unconfirmed:** the feed subscription, or a resubscribe from the -pinned query, resumes across the route change onto the replacement and fills the -retired track's cache for a group the old run had not delivered yet. Confirm by logging -which subscription delivers group 6 into the old track, then decide whether moq-net -refuses the resume or the export stops reading a replaced instance's tail. - -The takeover scenario uses the suite's `support` harness and needs a `contains` helper, -which was dropped from `support.rs` when the scenario moved here: - -```rust -//! A new publisher instance taking over a path is served under its own URLs, while the run it -//! replaced finishes its own recording and never serves the new run's bytes. - -use std::time::Duration; - -use axum::http::StatusCode; - -use crate::support::*; - -const OLD: u8 = 1; -const NEW: u8 = 2; - -/// The old run keeps publishing through the takeover and ends after it. Until a replaced run's -/// drain has a cutoff of its own, its cursor ends only once the old publisher does. -#[tokio::test(start_paused = true)] -async fn takeover() { - let pair = Pair::connect().await; - let mut old = Publisher::new(&pair.publisher, "live", Default::default()); - let server = pair.server(); - let app = server.router(); - let rendition = rendition(&server, "live").await; - let mut cursor = rendition.segments(); - - old.write(0, 13, OLD); - let old_media = media_uri(&app, "live").await; - listed(&app, &old_media, ".2.m4s").await; - let replaced = server.broadcaster("live").await.expect("announced"); - replaced.set_generation(Some("run-1")).expect("generation"); - let playlist = listed(&app, &old_media, "seg/run-1.").await; - let old_segments: Vec<_> = (0..3).map(|index| segment_uri(&old_media, &playlist, index)).collect(); - for uri in &old_segments { - assert_old(&app, uri).await; - } - // Reading, as a recorder does, so the cursor's subscription carries every group across. - let first = moq_clock::timeout(TIMEOUT, cursor.next()) - .await - .expect("the first segment") - .expect("records") - .expect("a segment"); - assert_eq!(first.segment, 0); - assert!(contains(&first.media, &[OLD; 8])); - - // The new instance numbers its groups and segments from zero again. - let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); - let mut new = Publisher::new(&pair.publisher, "live", route); - let deadline = moq_clock::now() + TIMEOUT; - let current = loop { - let current = server.broadcaster("live").await.expect("announced"); - if !std::sync::Arc::ptr_eq(¤t, &replaced) { - break current; - } - assert!(moq_clock::now() < deadline, "the new instance never took over"); - moq_clock::sleep(Duration::from_millis(50)).await; - }; - current.set_generation(Some("run-2")).expect("generation"); - new.write(0, 13, NEW); - - // The master lists the new instance, whose playlist lists its own generation. - let new_media = media_uri(&app, "live").await; - assert_ne!(new_media, old_media, "a new instance gets its own URLs"); - let playlist = listed(&app, &new_media, ".2.m4s").await; - assert!(!playlist.contains("run-1."), "{playlist}"); - let new_segment = segment_uri(&new_media, &playlist, 0); - assert!(new_segment.contains("/seg/run-2."), "{new_segment}"); - let response = get(&app, &new_segment).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(contains(response.body(), &[NEW; 8]) && !contains(response.body(), &[OLD; 8])); - // Nor does the new instance answer the old run's generation. - let crossed = new_segment.replace("/seg/run-2.", "/seg/run-1."); - assert_eq!(get(&app, &crossed).await.status(), StatusCode::NOT_FOUND); - - // The old run's URLs, under either generation, never answer with the new run's bytes. - for uri in &old_segments { - assert_old(&app, uri).await; - assert_old(&app, &uri.replace("/seg/run-1.", "/seg/run-2.")).await; - } - - // The old run ends while the new one carries on past the old run's last group. - old.finish(); - new.write(13, 21, NEW); - drop(old); - - // The old recording drains every segment of its own, its final GOP included, and nothing else. - let mut read = Vec::new(); - loop { - let next = moq_clock::timeout(TIMEOUT, cursor.next()) - .await - .expect("the replaced run's cursor never ended"); - let Some(segment) = next.expect("the replaced run finished cleanly") else { - break; - }; - assert!( - contains(&segment.media, &[OLD; 8]) && !contains(&segment.media, &[NEW; 8]), - "the replaced run's segment {} carries the new run's media", - segment.segment - ); - read.push(segment.segment); - } - assert_eq!(read, [1, 2, 3]); - let playlist = rendition.media_playlist(None).expect("finished").expect("listed"); - assert!(playlist.ends_with("#EXT-X-ENDLIST\n"), "{playlist}"); - - for uri in &old_segments { - assert_old(&app, uri).await; - } - assert_eq!(get(&app, &new_segment).await.status(), StatusCode::OK); - drop(new); -} - -/// `uri` answers the old run's bytes or nothing at all. -async fn assert_old(app: &axum::Router, uri: &str) { - let response = get(app, uri).await; - assert!(!contains(response.body(), &[NEW; 8]), "{uri} served the new run"); - match response.status() { - StatusCode::OK => assert!(contains(response.body(), &[OLD; 8]), "{uri}"), - _ => assert_missing(&response), - } -} - -/// The URI of the `index`th segment `playlist` lists, beside the media playlist at `media`. -fn segment_uri(media: &str, playlist: &str, index: usize) -> String { - let line = playlist - .lines() - .filter(|line| line.starts_with("seg/")) - .nth(index) - .unwrap_or_else(|| panic!("no segment {index} in {playlist}")); - media.replace("media.m3u8", line) -} - -/// Whether `haystack` contains `needle`. -fn contains(haystack: &[u8], needle: &[u8]) -> bool { - haystack.windows(needle.len()).any(|window| window == needle) -} -``` - -## Related - -- [HLS restart drain cutoff](/quest/m2/hls-restart-drain-cutoff.md) - changes what a replaced run's cursor drains; update the takeover scenario with it -- `export hls --output` (`rs/moq-cli/src/hls/record.rs`) - the recorder that would write the leaked segment -- moq-hls's `moq-tokio` dev-dependency features (`tcp`, `quic`, `aws-lc-rs`) now serve only `origin::spawn`. Trimming them to `net` first needs moq-tokio's `Backoff::validate` fixed, since it is dead code under `net` alone. diff --git a/rs/moq-hls/tests/integration/main.rs b/rs/moq-hls/tests/integration/main.rs index 4016ac1a3b9..a0783574568 100644 --- a/rs/moq-hls/tests/integration/main.rs +++ b/rs/moq-hls/tests/integration/main.rs @@ -5,5 +5,6 @@ mod disconnect; mod finish; +mod restart; mod server; mod support; diff --git a/rs/moq-hls/tests/integration/restart.rs b/rs/moq-hls/tests/integration/restart.rs new file mode 100644 index 00000000000..a88288bae85 --- /dev/null +++ b/rs/moq-hls/tests/integration/restart.rs @@ -0,0 +1,208 @@ +//! A new publisher instance taking over a path is served under its own URLs, while the run it +//! replaced finishes its own recording and never serves the new run's bytes. + +use std::time::Duration; + +use axum::http::StatusCode; +use moq_hls::export::segments; + +use crate::support::*; + +const OLD: u8 = 1; +const NEW: u8 = 2; + +/// What the replaced run's recording holds when the new instance takes over. +#[derive(Clone, Copy, Debug)] +enum Case { + /// A cached segment returned while the cursor's live SUBSCRIBE was still unanswered. + /// Without an epoch nothing says which instance answers it, so it is refused and the + /// recording loses the old run's last uncached GOP. + Unanswered, + /// As [`Self::Unanswered`], while the replacement's own group of the same number is + /// still open, which the old cursor once waited on forever. + UnansweredUnfinished, + /// The live subscription was answered before the takeover, so it stays on the old run + /// and drains every segment of its own. + Answered, + /// As [`Self::Unanswered`], with the old run finished just before the new instance + /// announces. + FinishedFirst, +} + +async fn takeover(version: &str, case: Case) { + let pair = Pair::new(version.parse().unwrap(), Default::default()).await; + let mut old = Publisher::new(&pair.publisher, "live", Default::default()); + let server = pair.server(); + let app = server.router(); + let rendition = rendition(&server, "live").await; + let mut cursor = rendition.segments(); + + old.write(0, 13, OLD); + let old_media = media_uri(&app, "live").await; + listed(&app, &old_media, ".2.m4s").await; + let replaced = server.broadcaster("live").await.expect("announced"); + replaced.set_generation(Some("run-1")).expect("generation"); + let playlist = listed(&app, &old_media, "seg/run-1.").await; + let old_segments: Vec<_> = (0..3).map(|index| segment_uri(&old_media, &playlist, index)).collect(); + for uri in &old_segments { + assert_old(&app, uri).await; + } + // Read as a recorder does. A cached segment can return while the live feed is still opening. + let first = moq_clock::timeout(TIMEOUT, cursor.next()) + .await + .expect("the first segment") + .expect("records") + .expect("a segment"); + assert_eq!(first.segment, 0); + assert!(contains( + first.media.as_deref().expect("first segment media"), + &[OLD; 8] + )); + match case { + // Paused, the clock only moves once every task is idle, so the feed has its answer. + Case::Answered => moq_clock::sleep(Duration::from_millis(500)).await, + Case::FinishedFirst => old.finish(), + Case::Unanswered | Case::UnansweredUnfinished => {} + } + + // The new instance numbers its groups and segments from zero again. + let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + let mut new = Publisher::new(&pair.publisher, "live", route); + let deadline = moq_clock::now() + TIMEOUT; + let current = loop { + let current = server.broadcaster("live").await.expect("announced"); + if !std::sync::Arc::ptr_eq(¤t, &replaced) { + break current; + } + assert!(moq_clock::now() < deadline, "the new instance never took over"); + moq_clock::sleep(Duration::from_millis(50)).await; + }; + current.set_generation(Some("run-2")).expect("generation"); + new.write(0, 13, NEW); + + // The master lists the new instance, whose playlist lists its own generation. + let new_media = media_uri(&app, "live").await; + assert_ne!(new_media, old_media, "a new instance gets its own URLs"); + let playlist = listed(&app, &new_media, ".2.m4s").await; + assert!(!playlist.contains("run-1."), "{playlist}"); + let new_segment = segment_uri(&new_media, &playlist, 0); + assert!(new_segment.contains("/seg/run-2."), "{new_segment}"); + let response = get(&app, &new_segment).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(contains(response.body(), &[NEW; 8]) && !contains(response.body(), &[OLD; 8])); + // Nor does the new instance answer the old run's generation. + let crossed = new_segment.replace("/seg/run-2.", "/seg/run-1."); + assert_eq!(get(&app, &crossed).await.status(), StatusCode::NOT_FOUND); + + // The old run's URLs, under either generation, never answer with the new run's bytes. + for uri in &old_segments { + assert_old(&app, uri).await; + assert_old(&app, &uri.replace("/seg/run-1.", "/seg/run-2.")).await; + } + + // The old run ends, while the new one carries on past the old run's last group unless it + // leaves that group open. + if !matches!(case, Case::FinishedFirst) { + old.finish(); + } + if !matches!(case, Case::UnansweredUnfinished) { + new.write(13, 21, NEW); + } + drop(old); + + // The old recording reads only its own segments, and either completes or fails at the GOP + // it lost. + let (read, end) = drain_old(&mut cursor).await; + match case { + Case::Answered => { + end.unwrap_or_else(|err| panic!("{case:?}: {err}")); + assert_eq!(read, [1, 2, 3], "{case:?}"); + let playlist = rendition.media_playlist(None).expect("finished").expect("listed"); + assert!(playlist.ends_with("#EXT-X-ENDLIST\n"), "{case:?}: {playlist}"); + } + Case::Unanswered | Case::UnansweredUnfinished | Case::FinishedFirst => { + assert!( + matches!(end, Err(moq_hls::Error::SegmentUnavailable { segment: 3 })), + "{case:?}: {end:?}" + ); + assert_eq!(read, [1, 2], "{case:?}"); + } + } + + for uri in &old_segments { + assert_old(&app, uri).await; + } + drop(new); +} + +#[tokio::test(start_paused = true)] +async fn unanswered_lite06() { + takeover("moq-lite-06", Case::Unanswered).await; +} + +#[tokio::test(start_paused = true)] +async fn unanswered_lite07() { + takeover("moq-lite-07-wip", Case::Unanswered).await; +} + +#[tokio::test(start_paused = true)] +async fn unanswered_unfinished() { + takeover("moq-lite-07-wip", Case::UnansweredUnfinished).await; +} + +#[tokio::test(start_paused = true)] +async fn answered() { + takeover("moq-lite-07-wip", Case::Answered).await; +} + +#[tokio::test(start_paused = true)] +async fn finished_first() { + takeover("moq-lite-07-wip", Case::FinishedFirst).await; +} + +/// Read `cursor` until it ends, asserting each segment carries only the old run's media. +async fn drain_old(cursor: &mut segments::Consumer) -> (Vec, moq_hls::Result<()>) { + let mut read = Vec::new(); + loop { + let next = moq_clock::timeout(TIMEOUT, cursor.next()) + .await + .expect("the replaced run's cursor stalled"); + let segment = match next { + Ok(Some(segment)) => segment, + Ok(None) => return (read, Ok(())), + Err(err) => return (read, Err(err)), + }; + let media = segment.media.as_deref().expect("segment media"); + assert!( + contains(media, &[OLD; 8]) && !contains(media, &[NEW; 8]), + "the replaced run's segment {} carries the new run's media", + segment.segment + ); + read.push(segment.segment); + } +} + +/// `uri` answers the old run's bytes or nothing at all. +async fn assert_old(app: &axum::Router, uri: &str) { + let response = get(app, uri).await; + assert!(!contains(response.body(), &[NEW; 8]), "{uri} served the new run"); + match response.status() { + StatusCode::OK => assert!(contains(response.body(), &[OLD; 8]), "{uri}"), + _ => assert_missing(&response), + } +} + +/// The URI of the `index`th segment `playlist` lists, beside the media playlist at `media`. +fn segment_uri(media: &str, playlist: &str, index: usize) -> String { + let line = playlist + .lines() + .filter(|line| line.starts_with("seg/")) + .nth(index) + .unwrap_or_else(|| panic!("no segment {index} in {playlist}")); + media.replace("media.m3u8", line) +} + +/// Whether `haystack` contains `needle`. +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|window| window == needle) +} diff --git a/rs/moq-net-tests/tests/integration/restart.rs b/rs/moq-net-tests/tests/integration/restart.rs index 8ff98774d9d..f41bfe7bf22 100644 --- a/rs/moq-net-tests/tests/integration/restart.rs +++ b/rs/moq-net-tests/tests/integration/restart.rs @@ -526,6 +526,62 @@ retired_fetch_tests! { retired_fetch_ietf22: "moq-transport-22", } +/// An epochless SUBSCRIBE names no instance, so until the publisher answers it, it may have +/// resolved whichever instance won the path. Unanswered when its instance is replaced, it is +/// refused, even if the publisher had resolved it on the old one, rather than risk filling +/// the old instance from the replacement. +async fn unanswered_takeover(version: &str) { + let version: Version = version.parse().unwrap(); + let publisher = produce_origin(1); + let viewer = produce_origin(2); + let _session = link(version, &publisher, &viewer).await; + let old = publisher.publish("live", origin::Route::default()).unwrap(); + // Quiet, so the publisher has nothing to answer the SUBSCRIBE with yet. + let old_track = old.create_track("video", timed()).unwrap(); + let mut announced = viewer.consume().announced(); + assert!(matches!(next_event(&mut announced).await, announce::Event::Start(_))); + let (_held, mut subscription) = subscribe(&viewer, "live").await; + settle().await; + assert!( + old_track.subscription().is_some(), + "{version}: the SUBSCRIBE reached the old instance" + ); + + let replacement = publisher.publish("live", origin::Route::default()).unwrap(); + let replacement_track = replacement.create_track("video", timed()).unwrap(); + write(&replacement_track, "new:0"); + expect_restart(version, &mut announced, "live").await; + write(&old_track, "old:0"); + + let next = moq_clock::timeout(Duration::from_secs(10), subscription.recv_group()) + .await + .expect("subscription hung"); + match next { + Err(err) => assert!(matches!(err, moq_net::Error::Unroutable), "{version}: {err}"), + Ok(group) => panic!( + "{version}: the replaced instance's SUBSCRIBE went on: {:?}", + group.map(|group| group.sequence) + ), + } +} + +macro_rules! unanswered_takeover_tests { + ($($name:ident: $version:literal,)*) => { $( + #[moq_clock::test] + async fn $name() { + moq_clock::timeout(TEST_TIMEOUT, unanswered_takeover($version)).await.expect("timed out"); + } + )* }; +} + +// moq-transport answers a SUBSCRIBE at once, before any group, and its subscriber already +// waits for that answer. +unanswered_takeover_tests! { + unanswered_takeover_lite05: "moq-lite-05", + unanswered_takeover_lite06: "moq-lite-06", + unanswered_takeover_lite07: "moq-lite-07-wip", +} + #[moq_clock::test] async fn retired_fetch_explicit_epoch_control() { moq_clock::timeout(TEST_TIMEOUT, retired_fetch("moq-lite-07-wip", true)) diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index e9c82848035..602c78a6034 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1452,7 +1452,7 @@ impl RequestValidity { /// Capture the lifetime of one pending request. pub fn start(&self) -> kio::Consumer<()> { match self.0.start() { - crate::model::broadcast::RequestValidity::Period(route, _) => route, + crate::model::broadcast::RequestValidity::Period(route, ..) => route, _ => unreachable!("a source watch always captures a route lease"), } } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 6c5403da9fd..3ccbdf5c7ae 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1885,16 +1885,19 @@ struct TracksRecv { gate: crate::auth::Gate, /// The gate let the listing through, and this hop synced its readers. admitted: bool, - /// Until the request is out, the source must still be the selected instance: a TRACKS - /// request without an epoch would list its replacement. - validity: crate::model::broadcast::RequestValidity, + /// Until the peer answers, what the source must still hold: the selected instance + /// until the request is out, then no replacement, since a TRACKS request without an + /// epoch lists whichever instance won the path when it arrived. + unanswered: Option, } impl TracksRecv { /// Apply the peer's events until it ends the listing, or the error that ends it. fn poll(&mut self, source: &SourceServe, waiter: &kio::Waiter) -> Poll> { - if self.stream.is_none() { - self.validity.check(waiter)?; + match (&self.unanswered, &self.stream) { + (Some(validity), None) => validity.check(waiter)?, + (Some(validity), Some(_)) => validity.check_unanswered(waiter)?, + (None, _) => {} } ready!(self.gate.poll_ready(waiter)); if self.gate.poll_denied(waiter).is_ready() { @@ -1923,7 +1926,9 @@ impl TracksRecv { ready!(stream.writer.poll_flush(&mut cx))?; loop { ready!(self.budget.poll_yield(waiter)); - let Some(event) = ready!(stream.reader.poll_decode_maybe::(&mut cx))? else { + let event = ready!(stream.reader.poll_decode_maybe::(&mut cx))?; + self.unanswered = None; + let Some(event) = event else { return Poll::Ready(Ok(())); }; match event { @@ -2007,7 +2012,7 @@ impl SourceServe { crate::auth::Direction::Subscribe, ), admitted: false, - validity: self.dynamic.validity().start(), + unanswered: Some(self.dynamic.validity().start()), }); } Some(_) if self.lister.poll_unused(waiter).is_ready() => { @@ -2138,6 +2143,7 @@ mod tests { served: None, end: None, wanted: None, + unanswered: None, bounded: false, updated: false, }) @@ -2272,6 +2278,7 @@ mod tests { served: None, end: None, wanted: None, + unanswered: None, bounded: false, updated: false, }); @@ -2734,6 +2741,7 @@ mod tests { served: None, end: Some(lite::SubscribeEnd { group: 10 }), wanted: None, + unanswered: None, bounded: false, updated: false, }; @@ -3483,6 +3491,100 @@ mod tests { } } + /// Answer a SUBSCRIBE without an epoch with a finished, empty track, but only read the + /// answer after `retire` changed its source's selection, then return what the reader got. + async fn answer_after( + version: Version, + retire: impl FnOnce(&mut crate::model::TestSelection), + ) -> Option, Error>> { + let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let mut selection = crate::model::TestSelection::new(); + let serve = TrackServe { + validity: selection.validity(), + route: None, + subscriber, + path: Path::new("room").to_owned(), + epoch: None, + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let request = broadcast.reserve_track("video").unwrap(); + let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); + let mut reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let gate = serve.gate(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Serve(serving), + gate, + }; + assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); + + let mut answer = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut answer, version.into()); + lite::SubscribeResponse::Start(lite::SubscribeStart { + group: 0, + largest: None, + }) + .encode(&mut encoder, version) + .unwrap(); + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0 }) + .encode(&mut encoder, version) + .unwrap(); + session.push(&answer); + retire(&mut selection); + let _ = kio::Task::poll(&mut running, &kio::Waiter::noop()); + + reader + .recv_group() + .now_or_never() + .map(|res| res.map(|group| group.map(|group| group.sequence))) + } + + /// An answer read only after another instance took the path may be that instance's, + /// so the replacement refuses the subscription instead, as does a withdrawal followed + /// by a new announcement. + #[moq_clock::test] + async fn an_answer_read_after_a_replacement_is_refused() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let res = answer_after(version, |selection| selection.replace()).await; + assert!(matches!(res, Some(Err(Error::Unroutable))), "{version:?}: {res:?}"); + let res = answer_after(version, |selection| { + selection.withdraw(); + selection.replace(); + }) + .await; + assert!(matches!(res, Some(Err(Error::Unroutable))), "{version:?}: {res:?}"); + } + } + + /// A withdrawal alone leaves the SUBSCRIBE to its answer: the publisher concludes a + /// subscription in flight, so a track that finished as its broadcast ended arrives whole. + #[moq_clock::test] + async fn an_answer_read_after_a_withdrawal_is_kept() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let res = answer_after(version, |selection| selection.withdraw()).await; + assert!(matches!(res, Some(Ok(None))), "{version:?}: {res:?}"); + } + } + /// Lite-07's SUBSCRIBE_START says where the track is, so a reader knows its largest /// frame before any group arrives. The FIN with no SUBSCRIBE_END then aborts the track, /// and the stream ends with it. @@ -4094,6 +4196,30 @@ mod tests { assert_eq!(session.log.bi_opens(), 0); } + /// A listing on the wire but unanswered when another instance takes the path lists + /// nothing: the peer may list that instance. + #[moq_clock::test] + async fn an_unanswered_listing_of_a_replaced_instance_is_refused() { + let open = kio::Producer::new(true); + let session = SinkSession::gated_bi(open.consume()); + let mut selection = crate::model::TestSelection::new(); + let (mut serve, consumer, _route) = + listed_source(session.clone(), crate::auth::Link::new(false), selection.validity()); + let mut tracks = consumer.tracks().unwrap(); + + assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_pending()); + assert_eq!(session.log.bi_opens(), 1); + assert!(matches!( + tracks.next().now_or_never(), + Some(Ok(Some(crate::broadcast::TracksEvent::Synced))) + )); + + selection.replace(); + let _ = kio::Task::poll(&mut serve, &kio::Waiter::noop()); + assert!(matches!(tracks.next().now_or_never(), Some(Err(Error::Unroutable)))); + assert_eq!(session.log.resets(), vec![crate::StreamError::Unroutable.to_code()]); + } + /// A listing outside the session's scope is refused before it reaches the wire. #[moq_clock::test] async fn a_listing_outside_the_scope_is_refused() { @@ -6037,6 +6163,10 @@ struct SubStream { end: Option, /// The refresh join's wanted group, restated in every SUBSCRIBE_UPDATE. wanted: Option, + /// Until the publisher's first response, what the source must still hold: a SUBSCRIBE + /// without an epoch resolves whichever instance wins the path when it arrives, so one + /// still unanswered once another instance takes the path may be answered by it. + unanswered: Option, /// Whether this stream ever asked for bounded ranges alone, which a lite-07 publisher /// FINs without SUBSCRIBE_END once it covers them. bounded: bool, @@ -6093,12 +6223,9 @@ impl SubStream { } } -#[cfg_attr( - target_pointer_width = "64", - expect( - clippy::large_enum_variant, - reason = "one per track copy, which holds a live stream while anyone reads it, so boxing it would add an allocation per upstream subscription without saving memory" - ) +#[expect( + clippy::large_enum_variant, + reason = "one per track copy, which holds a live stream while anyone reads it, so boxing it would add an allocation per upstream subscription without saving memory" )] enum Sub { None, @@ -6754,7 +6881,7 @@ impl Establish { self.state = EstablishState::WaitOk { stream }; continue; } - return Poll::Ready(Ok(self.activate(stream))); + return Poll::Ready(Ok(self.activate(stream, false))); } EstablishState::WaitOk { stream } => { if let Poll::Ready(err) = self.closed.poll_closed(&mut cx) { @@ -6768,7 +6895,7 @@ impl Establish { else { unreachable!() }; - return Poll::Ready(Ok(self.activate(stream))); + return Poll::Ready(Ok(self.activate(stream, true))); } } } @@ -6782,8 +6909,10 @@ impl Establish { } } - fn activate(&self, stream: Stream) -> SubStream { + /// The live subscription, `answered` once the publisher's first response arrived. + fn activate(&self, stream: Stream, answered: bool) -> SubStream { SubStream { + unanswered: (!answered).then(|| self.validity.clone()), stream, id: self.id, max_delay: wire_max_delay(self.subscription.max_delay), @@ -7567,6 +7696,19 @@ impl ServeLoop { self.closing = None; } + // A SUBSCRIBE still unanswered once another instance takes the path may have + // resolved it: refuse it. A withdrawal alone leaves it to its answer. Checked + // before reading the stream, so an answer that only arrived alongside the + // replacement cannot claim it, and before the flush, so an update parked on + // flow control cannot hold the refusal back. + if let Sub::Active(active) = &self.sub + && let Some(validity) = &active.unanswered + && let Err(err) = validity.check_unanswered(waiter) + { + tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "unanswered subscribe refused: instance replaced"); + return Poll::Ready(ServeEnd::GiveBack(err)); + } + // Deliver any buffered SUBSCRIBE_UPDATE before selecting, so the // demand that produced it is on the wire. if let Sub::Active(active) = &mut self.sub { @@ -7675,6 +7817,7 @@ impl ServeLoop { // subscription now. A query still held once that subscription ends // stays local: nothing reopens the TRACK for it. self.held = None; + active.unanswered = None; match res { Ok(Some(msg)) => { match &msg { diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 12474e48c46..76922008349 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -571,8 +571,12 @@ pub(crate) enum RequestValidity { Unrestricted, Route(kio::Consumer<()>), Selected(std::sync::Arc), - /// One period of a selection, captured by [`Self::start`]. - Period(kio::Consumer<()>, std::sync::Arc), + /// One period of a selection and its answering lease, captured by [`Self::start`]. + Period( + kio::Consumer<()>, + kio::Consumer<()>, + std::sync::Arc, + ), } impl RequestValidity { @@ -592,7 +596,7 @@ impl RequestValidity { /// Capture one selection period so a pending request never revives after a replacement. pub(crate) fn start(&self) -> Self { match self { - Self::Selected(selected) => Self::Period(selected.current(), selected.clone()), + Self::Selected(selected) => Self::Period(selected.current(), selected.answering(), selected.clone()), other => other.clone(), } } @@ -601,7 +605,7 @@ impl RequestValidity { match self { Self::Route(route) if route.poll_closed(waiter).is_ready() => Err(crate::Error::Unroutable), // A replacement retires the period only once its held `Restart` is released. - Self::Period(route, selected) + Self::Period(route, _, selected) if route.poll_closed(waiter).is_ready() || selected.poll_retired(waiter).is_ready() => { Err(crate::Error::Unroutable) @@ -610,6 +614,23 @@ impl RequestValidity { _ => Ok(()), } } + + /// [`Self::check`] for a request already sent but not yet answered: only another + /// instance taking the path refuses it, since without an epoch that instance may be + /// the one answering. A withdrawal leaves it to its answer, as moq-lite has the + /// publisher conclude in-flight requests. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn check_unanswered(&self, waiter: &kio::Waiter) -> Result<(), crate::Error> { + match self { + Self::Period(_, answering, selected) + if answering.poll_closed(waiter).is_ready() || selected.poll_retired(waiter).is_ready() => + { + Err(crate::Error::Unroutable) + } + Self::Selected(_) => self.start().check_unanswered(waiter), + _ => Ok(()), + } + } } #[cfg(test)] diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 5a842b590e7..ea6c3debb60 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -82,4 +82,4 @@ pub(crate) use origin_impl::{InterestChange, Interests}; pub(crate) use origin_impl::AnnounceProducer; #[cfg(all(test, feature = "async"))] -pub(crate) use origin_impl::ProduceTest; +pub(crate) use origin_impl::{ProduceTest, TestSelection}; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 67ea9f94d5f..6b7f5c892cc 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4153,6 +4153,10 @@ struct Selected { prefix: PathOwned, /// The selection period, closed when the peer stops selecting the instance. current: Option>, + /// Open until another instance of the peer takes the path, which may answer a + /// request still unanswered. A withdrawal alone leaves it open, and a period the + /// instance wins again keeps it. + answering: Option>, /// The [`Release`] that ends `current`: the peer replaced the instance with a /// route without an epoch, whose `Restart` is held until then. retiring: Option, @@ -4168,34 +4172,49 @@ enum Change { Retire(Release), /// The peer no longer selects the instance, effective at once. End, + /// Another instance of the peer took the path, effective at once. + Replace, } impl Selected { /// How the period changes now that the peer's best route is `best`, if it does. fn change(&self, best: Option<&RouteEntry>, hold: &Hold) -> Option { - if best.is_some_and(|entry| entry.instance() == self.instance) { + let replaced = self.retiring.as_ref().is_some_and(Release::released); + let Some(best) = best else { + // A released replacement already took the path before this withdrawal. + self.current.as_ref()?; + return Some(if replaced { Change::Replace } else { Change::End }); + }; + if best.instance() == self.instance { return match (&self.current, &self.retiring) { (Some(_), None) => None, - (Some(_), Some(release)) if !release.released() => Some(Change::Continue), + (Some(_), Some(_)) if !replaced => Some(Change::Continue), _ => Some(Change::Start), }; } - self.current.as_ref()?; + if self.current.is_none() { + // Withdrawn earlier: the instance taking the path now may answer what is left. + return self.answering.is_some().then_some(Change::Replace); + } // Announce cursors hold a restart at the same prefix onto a route without an // epoch, so the period lasts as long. Anything else they deliver at once. - let held = best.is_some_and(|entry| entry.epoch.is_none() && entry.prefix == self.prefix); + let held = best.epoch.is_none() && best.prefix == self.prefix; match (held, &self.retiring) { (true, Some(_)) => None, - (true, None) => Some(hold.release().map_or(Change::End, Change::Retire)), - (false, _) => Some(Change::End), + (true, None) => Some(hold.release().map_or(Change::Replace, Change::Retire)), + (false, _) => Some(Change::Replace), } } fn apply(&mut self, change: Change) { match change { Change::Start => { + // Past a released replacement, the instance that answered in between + // was the replacement. + if self.retiring.take().is_some() || self.answering.is_none() { + self.answering = Some(kio::Producer::default()); + } self.current = Some(kio::Producer::default()); - self.retiring = None; } Change::Continue => self.retiring = None, Change::Retire(release) => self.retiring = Some(release), @@ -4203,6 +4222,11 @@ impl Selected { self.current = None; self.retiring = None; } + Change::Replace => { + self.current = None; + self.answering = None; + self.retiring = None; + } } } } @@ -4227,6 +4251,17 @@ impl Selection { .unwrap_or_else(|| kio::Producer::new(()).consume()) } + /// Closed once another instance takes the path, for a request already sent. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(super) fn answering(&self) -> kio::Consumer<()> { + self.signal + .read() + .answering + .as_ref() + .map(kio::Producer::consume) + .unwrap_or_else(|| kio::Producer::new(()).consume()) + } + /// `Ready` once the current period's replacement is released, registering /// `waiter` until then. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -5426,6 +5461,7 @@ impl Dynamic { instance, prefix, current: active.then(kio::Producer::default), + answering: active.then(kio::Producer::default), retiring: None, }); let signal = producer.consume(); @@ -7174,6 +7210,49 @@ impl ProduceTest for Hop { } } +/// Test-only: one peer's route selected for `room/a`, which a test withdraws or +/// replaces to drive the [`broadcast::RequestValidity`](crate::model::broadcast::RequestValidity) +/// it hands out. +#[cfg(all(test, feature = "async"))] +pub(crate) struct TestSelection { + origin: Producer, + route: Option, + replacement: Option, +} + +#[cfg(all(test, feature = "async"))] +impl TestSelection { + const PEER: u64 = 2; + + pub(crate) fn new() -> Self { + let origin = Hop::new(1).unwrap().produce(); + let route = origin + .dynamic_route("room", Route::default().with_via(Hop::new(Self::PEER).unwrap())) + .unwrap(); + Self { + origin, + route: Some(route), + replacement: None, + } + } + + /// What a request for `room/a` through the selected route must still hold. + pub(crate) fn validity(&self) -> crate::model::broadcast::RequestValidity { + crate::model::broadcast::RequestValidity::for_source(self.route.as_ref().unwrap(), &Path::new("room/a")) + } + + /// The peer withdraws the route. + pub(crate) fn withdraw(&mut self) { + self.route = None; + } + + /// Another instance of the peer takes the path. + pub(crate) fn replace(&mut self) { + let route = Route::default().with_via(Hop::new(Self::PEER).unwrap()); + self.replacement = Some(self.origin.dynamic_route("room/a", route).unwrap()); + } +} + #[cfg(test)] mod tests { use super::*; @@ -13046,6 +13125,43 @@ mod tests { assert!(fresh.is_closed()); } + /// The answering lease outlives a withdrawal, since the peer still answers for the + /// instance it retracted, and ends once another instance takes the path. + #[moq_clock::test] + async fn source_selection_answering_ends_only_on_a_replacement() { + let producer = origin(1).produce(); + let path = Path::new("room/a"); + let first = producer + .dynamic_route("room", Route::default().with_via(origin(2))) + .unwrap(); + let selected = first.request_validity(&path).unwrap(); + let answering = selected.answering(); + // A route that loses to the selected one replaces nothing. + let worse = producer + .dynamic_route("room", Route::default().with_via(origin(2)).with_cost(Cost::from(7))) + .unwrap(); + assert!(!answering.is_closed()); + drop(worse); + + let replacement = producer + .dynamic_route("room/a", Route::default().with_via(origin(2))) + .unwrap(); + assert!(answering.is_closed()); + // Winning again opens a new lease, which a withdrawal leaves open. + drop(replacement); + let answering = selected.answering(); + assert!(!answering.is_closed()); + drop(first); + assert!(selected.current().is_closed()); + assert!(!answering.is_closed()); + + // Announced again, the path has a new instance: a replacement. + let _again = producer + .dynamic_route("room", Route::default().with_via(origin(2))) + .unwrap(); + assert!(answering.is_closed()); + } + #[moq_clock::test] async fn source_selection_unregisters_without_retaining_the_source() { let producer = origin(1).produce();