diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index c2d7299554b..a70ee885972 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -2462,6 +2462,58 @@ test.each( } }); +// A session that dies once the publisher ended the track, but before the tail settled, cuts it +// off: the track ends with the session's error unless the declared end settled first. That end +// is SUBSCRIBE_END on lite-05+, and the FIN before it. +test.each([ + ["lite-03 ends clean with its last group finished", Version.DRAFT_03, true], + ["lite-03 aborts with its last group still open", Version.DRAFT_03, false], + ["lite-05 ends clean with its last group finished", Version.DRAFT_05, true], + ["lite-05 aborts with its last group still owed", Version.DRAFT_05, false], +] as const)("a session death before the tail settles: %s", async (_, version, settled) => { + const installed = new Manual().install(); + const { quic, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const broadcast = subscriber.consume(Path.from("room")); + const track = broadcast.track("video").subscribe(); + const died = new Error("session died"); + // Lite-05 asks for TRACK_INFO first; untimed, a frame is its size and payload. + const legacy = version === Version.DRAFT_03; + const frame = (controller: ReadableStreamDefaultController) => + controller.enqueue(new Uint8Array([1, 0])); + try { + if (!legacy) await answerTrackInfo(await opened(0), version); + const subscribe = await opened(legacy ? 0 : 1); + await subscribe.reading; + const responses: SubscribeResponse[] = legacy + ? [{ ok: new SubscribeOk({}) }] + : [{ start: new SubscribeStart([0]) }, { end: new SubscribeEnd(1) }]; + await respond(subscribe, version, responses, true); + if (settled) { + const group = new ReadableStream({ + start: (controller) => { + frame(controller); + controller.close(); + }, + }); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0 }), new Reader(group, undefined, version)); + } else if (legacy) { + // Lite-03 says nothing about which groups are owed, so only a stream still being read is. + const group = new ReadableStream({ start: frame }); + void subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0 }), new Reader(group, undefined, version)); + } + await drainReactions(); + + subscriber.close(died); + expect(await track.closed).toBe(settled ? null : died); + } finally { + track.close(); + broadcast.close(); + subscriber.close(); + installed.close(); + } +}); + // The FIN may answer the bounded range before the publisher read an update that widened it, // so the track fails rather than ending clean short of the demand. test.each([ diff --git a/quest/m1/README.md b/quest/m1/README.md index 26bf657f386..08c012b00a0 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -171,7 +171,6 @@ removed). - [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - nightly ffprobe checks that advertised timing, sync flags, and bytes match the media served - [Unbounded groups](/quest/m1/unbounded-groups/README.md) - groups and frames lose their size caps; a group keeps a 32 MB tail, and readers behind it replay through paced flights or refresh - [Next Object on a cold Rust copy](/quest/m1/next-object-cold-copy.md) - a join after an upstream group report starts at the first frame received, never skipping -- [Lite track end at session death](/quest/m1/lite-track-end.md) - Rust and JS abort an ended lite track whose tail never settled, and agree on what declares the end - [Legacy lite close drain](/quest/m1/lite-legacy-close-drain.md) - a lite-01..06 graceful close never cuts a final group the subscriber has not read - [moqsrc max-delay](/quest/m1/gst-max-delay.md) - GStreamer's `moqsrc` takes a `max-delay` property instead of a hard-coded 1 s budget - [Poll-first async](/quest/m1/poll-first/README.md) - library crates compile no coroutines by default: async wrappers are generated from `poll_*` and gated behind an `async` feature diff --git a/quest/m1/lite-track-end.md b/quest/m1/lite-track-end.md deleted file mode 100644 index 0eb797d8513..00000000000 --- a/quest/m1/lite-track-end.md +++ /dev/null @@ -1,37 +0,0 @@ -# [S] An ended lite track with an unsettled tail aborts at session death - -## Goal - -When a session dies after a lite publisher ended a track but before its tail -settled, Rust and JS both abort the track, and both end it cleanly once the -tail settled. Both agree on what declares the end. - -## Plan - -Found while landing [moq-dev/moq#6969](https://github.com/moq-dev/moq/pull/6969). -Rust lite marks a track ended on the subscribe stream's FIN, on every -version, and its cleanup then calls `finish()` without checking the tail: -`SubscriptionCleanup::abort` and `close`, and `ServeMode::Tail`'s -`ServeEnd::Finished`, in `rs/moq-net/src/lite/subscriber.rs`. JS declares -the end on SUBSCRIBE_END (and, after #6969, the FIN on lite-01..04), and -`track.Producer#close` downgrades an abort to a clean end only once the -declared end settled. Rust's IETF subscriber already aborts. - -Decided 2026-10-28 (maintainer): - -- Rust aborts like JS, since `finish()` on an unsettled tail reports - incomplete content as complete. Its cleanup always calls `abort` and relies - on `track::Producer::abort`'s existing downgrade when the declared end - settled. So does `ServeMode::Tail` when the session closes before the tail - settles: that branch returns `ServeEnd::Finished` today, which - `TrackServeRun` turns into `serving.finish()`, so it must abort instead. -- Align the end signal in both languages: SUBSCRIBE_END on lite-05+, the - stream FIN on lite-01..04. On lite-07 a bounded range also FINs, which must - not end the track. - -Cover it with a test in each language for the settled and unsettled cases, -including Rust's `ServeMode::Tail` session close, and an interop case that -kills the session with groups unsettled. - -Public API: none. Wire: none; behavior only. A Rust consumer of an -unsettled track sees the session error instead of a clean finish. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index c2965b71114..764ce5639cf 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -640,15 +640,16 @@ impl SubscriptionCleanup { /// End every active subscription with the session's error. Group machines own /// their cleanup independently; this records the track's terminal state. /// - /// A subscription the publisher already ended finishes instead: its track is over, - /// and what arrived stays servable to readers still behind rather than handed to - /// another route. + /// Even a subscription the publisher already ended aborts: finishing it before its tail + /// settled would report what arrived as the whole track. Once the declared end settled, + /// [`track::Producer::abort`] ends its readers cleanly instead. Before lite-05 the FIN + /// is that end, declared here at the live edge since nothing more can arrive. fn abort(&self, err: &Error) { for (_, entry) in self.0.lock().drain() { - let _ = match entry.tail.read().is_ended() { - true => entry.producer.finish(), - false => entry.producer.abort(err.clone()), - }; + if entry.tail.read().is_ended() { + let _ = entry.producer.finish(); + } + let _ = entry.producer.abort(err.clone()); } } } @@ -3971,6 +3972,91 @@ mod tests { } } + /// SUBSCRIBE_START at group 0 and SUBSCRIBE_END at 1: the publisher ended the track + /// owing group 0. + fn owed_responses(version: Version) -> Vec { + let mut responses = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut responses, version.into()); + lite::SubscribeResponse::Start(lite::SubscribeStart { + largest: None, + starts: vec![Some(0)], + }) + .encode(&mut encoder, version) + .unwrap(); + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 1, error: None }) + .encode(&mut encoder, version) + .unwrap(); + responses + } + + /// Drive the subscriber with a peer's response bytes and FIN, owing a group that never + /// arrives, then close the session: the track ends with the session's error. + async fn check_owed_tail_death(version: Version, responses: Vec) { + let session = crate::lite::test_transport::ScriptedSession::eof(responses); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let serve = TrackServe { + validity: Default::default(), + route: None, + subscriber, + path: Path::from_static("room"), + epoch: None, + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let serving = ServeLoop::new(&serve, broadcast.reserve_track("video").unwrap(), Default::default()); + let mut reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let gate = serve.gate(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Serve(serving), + gate, + }; + assert!( + kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending(), + "{version:?}: the tail still owes group 0" + ); + crate::transport::poll::Session::close(&session, crate::SessionError::App(7).to_code(), "died"); + assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_ready()); + let res = reader.recv_group().await.err(); + assert!(matches!(res, Some(Error::Session(_))), "{version:?}: {res:?}"); + } + + #[kio::test] + async fn an_owed_tail_aborts_at_session_death() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + check_owed_tail_death(version, owed_responses(version)).await; + } + } + + /// Both runtimes abort a track whose tail the other's publisher still owed when the + /// session died. + #[kio::test] + #[ignore = "requires Bun; run by just test bare-fin in interop CI"] + async fn owed_tail_death_interop() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let responses = + crate::test_interop::owed_tail(crate::Version::from(version).alpn(), owed_responses(version)); + check_owed_tail_death(version, responses).await; + } + } + /// Removing a subscription both stops delivery and releases the session's handle /// on the producer, so the track (its cached groups, its stats subscription) ends /// rather than outliving the subscription it belonged to. @@ -4733,19 +4819,27 @@ mod tests { } } - /// A session that dies hands a live track back with the session's error, but finishes one - /// whose subscription the publisher already ended, as the driver's cleanup does. + /// A session that dies before a tail settles hands the track back with the session's error, + /// even one the publisher ended, and the abort `TrackServeRun` commits then ends readers + /// cleanly only if the declared end settled: SUBSCRIBE_END on lite-05+, the FIN before it. #[kio::test] - async fn internal_session_end_gives_back_live_and_finishes_ended() { + async fn internal_session_end_aborts_an_unsettled_tail() { use crate::transport::poll::Session as _; - for ended in [false, true] { + // The publisher ended the track; whether its last group finished. + for (version, ended) in [ + (Version::Lite04, None), + (Version::Lite04, Some(false)), + (Version::Lite04, Some(true)), + (Version::Lite05, Some(false)), + (Version::Lite05, Some(true)), + ] { let (session, peer) = crate::transport::memory::pair(Default::default()); let subscriber = Subscriber::new(SubscriberConfig { runtime: crate::time::sim(), session: crate::driver::Transport::new(session.clone()), origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), recv_bandwidth: None, - version: VERSION, + version, peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), peer_hop: None, cost: None, @@ -4766,7 +4860,15 @@ mod tests { assert!(serving.poll(&serve, &kio::Waiter::noop()).is_pending()); // Held for the whole round: a dropped tail settles at once. let tail = kio::Producer::new(Tail::default()); - if ended { + let mut group = serving.serving.append_group().unwrap(); + if let Some(finished) = ended { + if finished { + group.finish().unwrap(); + } + if version.has_track_stream() { + serving.serving.finish_at(Position::group(1), None).unwrap(); + } + // The tail machine has not seen group 0 accounted for, so it never settles. serving.mode = ServeMode::Tail { settle: Settle::new(&serve.subscriber.runtime, tail.consume()), owed: Some(std::iter::once(0..1).collect()), @@ -4776,13 +4878,23 @@ mod tests { } peer.close(crate::SessionError::Unauthorized.to_code(), "revoked"); let end = serving.poll(&serve, &kio::Waiter::noop()); + let Poll::Ready(ServeEnd::GiveBack(err @ Error::Session(crate::SessionError::Unauthorized))) = end else { + panic!("{version:?} {ended:?}: the tail must hand the track back"); + }; + let track = serving.serving.clone(); + serving.serving.abort(err).unwrap(); + let outcome = track.poll_closed(&kio::Waiter::noop()); match ended { - true => assert!(matches!(end, Poll::Ready(ServeEnd::Finished))), - false => assert!(matches!( - end, - Poll::Ready(ServeEnd::GiveBack(Error::Session(crate::SessionError::Unauthorized))) - )), + Some(true) => assert!(matches!(outcome, Poll::Ready(Ok(()))), "{version:?}: {outcome:?}"), + _ => assert!( + matches!( + outcome, + Poll::Ready(Err(Error::Session(crate::SessionError::Unauthorized))) + ), + "{version:?} {ended:?}: {outcome:?}" + ), } + drop(group); } } @@ -8188,14 +8300,15 @@ impl ServeLoop { if self.fetches.is_empty() && crate::util::poll_unwanted(&mut self.demand, waiter).is_ready() { return Poll::Ready(ServeEnd::Idle); } - // The publisher already ended the track, so a session that dies now - // finishes it, as `SubscriptionCleanup` does, rather than hand it to - // another route. A copy whose subscription alone ended is handed back. + // The tail is still unsettled, so a session that dies now cuts the track + // off, even one the publisher ended: the abort ends readers cleanly only + // once the declared end settled. Before lite-05 the FIN is that end, as in + // `SubscriptionCleanup`. if let Poll::Ready(err) = self.closed.poll_closed(waiter) { - return Poll::Ready(match track { - true => ServeEnd::Finished, - false => ServeEnd::GiveBack(crate::lite::error::from_transport(err)), - }); + if *track && !serve.subscriber.version.has_track_stream() { + let _ = self.serving.finish(); + } + return Poll::Ready(ServeEnd::GiveBack(crate::lite::error::from_transport(err))); } return Poll::Pending; } @@ -8450,6 +8563,10 @@ impl ServeLoop { tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "subscribe covered"); let budget = self.serving.subscription().map(|sub| sub.budget()); if let Ok(mut tail) = active.tail.write() { + // Lite-05+ declared the end with SUBSCRIBE_END instead. + if !serve.subscriber.version.has_track_stream() { + tail.end(); + } tail.set_grace(tail::grace(budget.unwrap_or_default())); tail.expire(serve.subscriber.runtime.now()); } @@ -8477,7 +8594,10 @@ impl ServeLoop { // (it bounds presentation-time drift), but it is how long the // subscriber was willing to wait for a late group anyway. if let Ok(mut tail) = active.tail.write() { - tail.end(); + // Lite-05+ declared the end with SUBSCRIBE_END instead. + if !serve.subscriber.version.has_track_stream() { + tail.end(); + } tail.set_grace(tail::grace( subscription.map(|sub| sub.budget()).unwrap_or_default(), )); diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 476b89875a2..07b74137da9 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -25,8 +25,8 @@ pub(crate) struct Tail { streams: u64, // Streams whose header arrived and that are still being read. active: u64, - // The publisher ended the subscription, so the track is over even if the session - // dies before the tail settles. + // The publisher's FIN ended the track, its only end before lite-05, so a session that + // dies before the tail settles declares that end before aborting. ended: bool, } @@ -47,12 +47,12 @@ impl Tail { } } - /// Record that the publisher ended the subscription. + /// Record that the publisher's FIN ended the track. pub fn end(&mut self) { self.ended = true; } - /// Whether the publisher ended the subscription. + /// Whether the publisher's FIN ended the track. pub fn is_ended(&self) -> bool { self.ended } diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 8bf01e0a92f..cb1689a4f9a 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -24,6 +24,13 @@ pub(crate) fn fin(version: &str, started: bool, clean: bool, responses: Vec) serde_json::from_slice(&bun("bare-fin.ts", input)).expect("JS publisher returned response bytes") } +/// Feed a JS subscriber responses that end the track owing group 0, kill its session, and +/// return the JS publisher's encoding of the same responses. +pub(crate) fn owed_tail(version: &str, responses: Vec) -> Vec { + let input = serde_json::json!({ "version": version, "owed": true, "responses": responses }); + serde_json::from_slice(&bun("bare-fin.ts", input)).expect("JS publisher returned response bytes") +} + /// Rust updates keep the JS subscription alive, cover both unseen groups after narrowing, /// and fill the largest group's head under the update's ID on draft-22. #[test] diff --git a/rs/moq-net/tests/integration/track_tail.rs b/rs/moq-net/tests/integration/track_tail.rs index 238a1c46d76..35a6ad81d30 100644 --- a/rs/moq-net/tests/integration/track_tail.rs +++ b/rs/moq-net/tests/integration/track_tail.rs @@ -46,8 +46,13 @@ enum Late { Delivered, /// They never arrive, like a stream reset before its header. Lost, + /// The publisher's session dies before they arrive. + Killed, } +/// The error the publisher's session dies with. +const DEATH: moq_net::SessionError = moq_net::SessionError::App(7); + fn produce_origin(hop: u64) -> moq_net::origin::Producer { let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); support::harness::spawn(driver); @@ -172,6 +177,7 @@ async fn round(version: &str, late: Late, final_sequence: u64, max_delay: Option match late { Late::Delivered => pair.server_transport.release_unis(), Late::Lost => pair.server_transport.drop_unis(), + Late::Killed => pair.server.abort(moq_net::Error::Session(DEATH)), } let (frames, err, ended) = kio::clock::timeout(TIMEOUT, reader) @@ -238,6 +244,26 @@ async fn a_lost_group_ends_the_track_after_the_grace() { } } +/// A session that dies with the tail still owed cuts the track off, even though the publisher +/// ended it: a clean end would pass the missing group off as never sent. Lite-03 declares no +/// end, so nothing there says a group is owed. +#[kio::test] +async fn a_session_death_before_the_tail_aborts_the_track() { + for version in ["moq-lite-05", "moq-lite-07-wip"] { + let outcome = round(version, Late::Killed, 1, None).await; + assert!( + outcome.frames.is_empty(), + "{version}: got {} frame(s)", + outcome.frames.len() + ); + assert!( + matches!(&outcome.err, Some(moq_net::Error::Session(code)) if *code == DEATH), + "{version}: ended with {:?}", + outcome.err + ); + } +} + /// Skipped sequences have no stream, so lite-05+ publishers name them with SUBSCRIBE_DROP. #[kio::test] async fn skipped_groups_end_without_the_grace() { diff --git a/test/interop/README.md b/test/interop/README.md index 196437c0bdf..95e9144fc32 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -597,7 +597,8 @@ just test interop-report /tmp/results.jsonl /tmp/interop-report JS, then feeds them into each implementation's subscriber over an in-memory transport. It checks bare FIN before and after SUBSCRIBE\_START on lite-05/06/07, and FIN without PUBLISH\_DONE on IETF draft-18. Clean-end controls use the same -path. This tests response interoperability, not network delivery or relay behavior. +path. On lite-05/06/07 it also ends a track owing group 0 and kills the session +before that group arrives, which each subscriber must end with the session's error. This tests response interoperability, not network delivery or relay behavior. The interop workflow runs it alongside the real-transport matrix. ## Varints diff --git a/test/interop/bare-fin.ts b/test/interop/bare-fin.ts index 1b7ce09e5f8..430c575f575 100644 --- a/test/interop/bare-fin.ts +++ b/test/interop/bare-fin.ts @@ -19,7 +19,10 @@ import type { Subscriber as TrackSubscriber } from "../../js/net/src/track.ts"; // Stdout is the response-byte channel back to Rust. console.debug = console.error; -const input: { version: string; started: boolean; clean: boolean; responses: number[] } = JSON.parse(process.argv[2]); +// `owed` ends the track owing group 0 and kills the session before it arrives. +const input: { version: string; started?: boolean; clean?: boolean; owed?: boolean; responses: number[] } = JSON.parse( + process.argv[2], +); const pair = createMockTransportPair(input.version); const path = Path.from("room"); const lite: Record = { @@ -39,8 +42,10 @@ const output = new Writer( ); let reader: TrackSubscriber; let peer: Stream | undefined; +let liteSubscriber: LiteSubscriber | undefined; if (version !== undefined) { const subscriber = new LiteSubscriber(pair.client, version, randomHop()); + liteSubscriber = subscriber; reader = subscriber.consume(path).track("video").subscribe(); const info = await Stream.accept(pair.server, version); assert(info); @@ -52,8 +57,10 @@ if (version !== undefined) { assert(peer); assert.equal(await peer.reader.u53(), StreamId.Subscribe); await Subscribe.decode(peer.reader, version); - if (input.started) await encodeSubscribeResponse(output, { start: new SubscribeStart([0]) }, version); - if (input.clean) await encodeSubscribeResponse(output, { end: new SubscribeEnd(0) }, version); + if (input.started || input.owed) await encodeSubscribeResponse(output, { start: new SubscribeStart([0]) }, version); + if (input.clean || input.owed) { + await encodeSubscribeResponse(output, { end: new SubscribeEnd(input.owed ? 1 : 0) }, version); + } } else { assert.equal(input.version, "moqt-18"); const version = IetfVersion.DRAFT_18; @@ -73,13 +80,23 @@ if (version !== undefined) { // These bytes came from Rust's encoder; only the omitted end message is malformed. if (input.responses.length) await peer.writer.write(Uint8Array.from(input.responses)); await peer.writer.close(); -const closed = await reader.closed; -if (input.clean) { - assert.equal(closed, null); - assert.equal(await reader.recvGroup(), undefined); +if (input.owed) { + assert(liteSubscriber); + // Once the responses are read to the FIN, the session dies with group 0 still owed. + await liteSubscriber.drain(); + const died = new Error("session died"); + liteSubscriber.close(died); + assert.equal(await reader.closed, died); + await assert.rejects(reader.recvGroup(), died); } else { - assert(closed instanceof ProtocolViolation); - await assert.rejects(reader.recvGroup(), ProtocolViolation); + const closed = await reader.closed; + if (input.clean) { + assert.equal(closed, null); + assert.equal(await reader.recvGroup(), undefined); + } else { + assert(closed instanceof ProtocolViolation); + await assert.rejects(reader.recvGroup(), ProtocolViolation); + } } // Rust feeds the JS-encoded bytes into its own subscriber, followed by FIN. console.log(JSON.stringify(bytes)); diff --git a/test/justfile b/test/justfile index 0a3a41782d6..21a0d3f6fc5 100644 --- a/test/justfile +++ b/test/justfile @@ -255,7 +255,7 @@ receive-credit: # Cross-language subscription termination over in-memory transports. bare-fin: - just rs test -p moq-net --lib bare_fin_interop --run-ignored all + just rs test -p moq-net --lib bare_fin_interop owed_tail_death_interop --run-ignored all # Explore JSON snapshot codec sequences with a fresh replayable seed. json-properties: