diff --git a/js/net/src/group.ts b/js/net/src/group.ts index cae93c2f3f1..f3ea3604676 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -568,6 +568,13 @@ export class Consumer { }; hooks.groupArrived = (group) => group.#state.arrived; hooks.groupStarted = (group) => group.#state.total.peek() > 0 || group.#state.closed.peek() !== undefined; + hooks.groupFate = (group) => { + const state = group.#state; + return () => { + const closed = state.closed.peek(); + return closed instanceof Error ? closed : undefined; + }; + }; hooks.arriveGroup = (group, at) => { group.#state.arrived = at; }; diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index b5f8df25066..9263e057a0c 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -785,8 +785,7 @@ test("a group stream whose FIN never completes is reset when the group expires", write(0); await closeStarted; - // The newest group is never stale, however long its FIN takes. Kept inside the cache's - // idle window: a group evicted once superseded is no longer judged. + // The newest group is never stale, however long its FIN takes. await clock.advance(Nano.fromMilli(Milli(budget * 2))); expect(reset).toBe(false); @@ -806,6 +805,93 @@ test("a group stream whose FIN never completes is reset when the group expires", } }); +// Under a budget past the cache window, the cache evicts a stalled group and its successor +// before the budget convicts the group. The verdict still waits only for the successor's +// arrival plus the budget, not for whichever newer group the cache holds, and PUBLISH_DONE +// follows the reset. +test("a group stream whose FIN never completes is reset after the cache evicts it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const pair = createMockTransportPair(ALPN.DRAFT_19); + + let closing!: () => void; + const closeStarted = new Promise((resolve) => { + closing = resolve; + }); + let reset = false; + let resetting!: () => void; + const streamReset = new Promise((resolve) => { + resetting = resolve; + }); + const stalled = new WritableStream({ + start(controller) { + controller.signal.addEventListener("abort", () => { + reset = true; + resetting(); + }); + }, + close() { + closing(); + return new Promise(() => {}); + }, + }); + spyOn(pair.server, "createUnidirectionalStream").mockImplementationOnce(async () => stalled); + + // Twice the cache's idle window. + const budget = Milli(60_000); + const { pub, origin } = publisher(pair.server); + const broadcast = publish(origin, Path.from("test")); + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: budget }); + const client = await Stream.open(pair.client, { version: VERSION }); + const server = await Stream.accept(pair.server, VERSION); + if (!server) throw new Error("publisher never accepted the subscribe stream"); + + try { + const running = pub.runSubscribe( + new Subscribe({ + requestId: 0n, + trackNamespace: Path.from("test"), + trackName: "video", + subscriberPriority: 0, + }), + server, + ); + expect(await client.reader.u53()).toBe(SubscribeOk.id); + await SubscribeOk.decode(client.reader, VERSION); + + // Frozen timestamps, so only the wall clock since group 1 arrived convicts group 0. + const write = (sequence: number) => { + const group = new GroupProducer(sequence); + group.writeFrame({ payload: new TextEncoder().encode("frame"), timestamp: Timestamp.fromMillis(0) }); + group.close(); + track.writeGroup(group); + }; + + write(0); + await closeStarted; + write(1); + // A later group, so a verdict that forgot group 1 would wait for this one's deadline. + const later = Milli(10_000); + await clock.advance(Nano.fromMilli(later)); + write(2); + track.close(); + + await clock.advance(Nano.fromMilli(Milli(budget - later - 1))); + expect(reset).toBe(false); + await clock.advance(Nano.fromMilli(Milli(1))); + await streamReset; + + await running; + expect(await client.reader.u53()).toBe(PublishDone.id); + await PublishDone.decode(client.reader, VERSION); + } finally { + client.close(); + broadcast.close(); + origin.close(); + installed.close(); + } +}); + // A group can go stale while its stream is still opening. Serving it must abandon the group // without starting a write: an abandoned write rejects once the stream resets, and nothing // would handle it (Node exits on the first unhandled rejection). diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 111131b47df..e83649c0c39 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -535,6 +535,8 @@ export const hooks: { groupArrived: (group: GroupConsumer) => Instant; /** Whether a group counts as content: its first frame landed, or it ended. */ groupStarted: (group: GroupConsumer) => boolean; + /** Watch the error a group's copy closes with, ignoring readers' verdicts, without holding the consumer. */ + groupFate: (group: GroupConsumer) => () => Error | undefined; /** Record when a track's mirror of a group joined that track at this hop. */ arriveGroup: (group: GroupConsumer, at: Instant) => void; /** Keep applying a subscription's drift policy after it hands a group out. */ @@ -715,6 +717,9 @@ export const hooks: { groupStarted: () => { throw new Error("group.ts not loaded"); }, + groupFate: () => { + throw new Error("group.ts not loaded"); + }, expireGroup: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index c8f7d42d80e..8d148cc0e2b 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -1266,6 +1266,95 @@ test("a parked read wakes at its wall deadline", async () => { } }); +// Evicting a recorded successor nobody read leaves its mirror a reader-side gap, which is not +// a failure: the held group is still judged by that successor's arrival, not the next one's. +test("an evicted successor with unread frames still starts the wall clock", async () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + try { + const budget = 2 * CACHE_WINDOW_MS; + const track = producer.subscribe({ maxDelay: Milli(budget) }); + const write = () => { + const group = producer.appendGroup(); + group.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(0) }); + group.close(); + }; + write(); + const group = track.tryRecvGroup(); + if (!group) throw new Error("missing group"); + expect(await group.readString()).toBe("x"); + + let stale = false; + const guarded = hooks + .guardGroup(group, () => new Promise(() => {})) + .catch((err: unknown) => { + stale = true; + return err; + }); + write(); // the successor, left unread + await settle(); + // A later live edge, so the successor ages out of the cache. + await clock.advance(20_000); + write(); + + await clock.advance(10_000 + budget - 1); + expect(stale).toBe(false); + await clock.advance(10_000 + budget); + expect(stale).toBe(true); + expect(await guarded).toMatchObject({ code: StreamCode.Old }); + } finally { + producer.close(); + clock.installed.close(); + } +}); + +// A held group's recorded successor that failed hands the wall clock on to the next live +// group, even when the write that brings that group in also evicts the failed one before the +// guard looks again, rather than convicting by the failed one's arrival. +test("a failed successor hands the wall clock on after the cache evicts it", async () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + try { + const budget = 2 * CACHE_WINDOW_MS; + const track = producer.subscribe({ maxDelay: Milli(budget) }); + const held = producer.appendGroup(); + held.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(0) }); + held.close(); + const group = track.tryRecvGroup(); + if (!group) throw new Error("missing group"); + expect(await group.readString()).toBe("x"); + + let stale = false; + const guarded = hooks + .guardGroup(group, () => new Promise(() => {})) + .catch((err: unknown) => { + stale = true; + return err; + }); + const failed = producer.appendGroup(); + await settle(); + + // Past the cache window, so the next write evicts the failed successor at once. + const replacement = 10_000 + CACHE_WINDOW_MS + 10_000; + await clock.advance(replacement); + failed.close(new Error("upstream failed")); + const live = producer.appendGroup(); + live.writeFrame({ payload: enc.encode("y"), timestamp: Timestamp.fromMillis(0) }); + live.close(); + + await clock.advance(10_000 + budget + 1); + expect(stale).toBe(false); + await clock.advance(replacement + budget - 1); + expect(stale).toBe(false); + await clock.advance(replacement + budget); + expect(stale).toBe(true); + expect(await guarded).toMatchObject({ code: StreamCode.Old }); + } finally { + producer.close(); + clock.installed.close(); + } +}); + // Empty groups are handed out like any other and retired by the wall clock like an untimed // one, whether they finished without a frame or are still open with none. A publisher // flooding a run of them cannot pin the group before the run either. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index f4d7bd8a2db..c68d40bfb40 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -75,6 +75,18 @@ const HOLES: ReadonlySet = new Set([ StreamCode.Evicted, ]); +// The errors this cache closes a group with when it evicts it unfinished: an aged-out group, +// not a failed one. Mirrors Rust's cache-policy aborts in `group::Handle::commit_abort`. +const EVICTED = new WeakSet(); + +// Whether `successor`'s copy failed: closed with an error other than its eviction from this +// cache. Read from the copy, since a reader's own verdict (an eviction's gap, a stale group) +// says nothing about whether the group arrived. +function failed(successor: Successor): boolean { + const abort = successor.fate(); + return abort !== undefined && !EVICTED.has(abort); +} + /** * A position within a track: a group's sequence and a frame's index inside it. * @@ -1131,7 +1143,11 @@ export class Producer { // the mirrors below would otherwise report a clean finish to a reader that had // drained it. The usual case, an already-closed group aging out, keeps its own // terminal state. - if (!entry.group.isClosed) entry.group.close(new TooFarBehind()); + if (!entry.group.isClosed) { + const evicted = new TooFarBehind(); + EVICTED.add(evicted); + entry.group.close(evicted); + } const mirrors = [...entry.mirrors.values()]; for (const mirror of mirrors) hooks.evictGroup(mirror); this.#unlink(entry); @@ -1773,6 +1789,15 @@ type Drift = { end?: number; }; +// The successor a handed-out group was last judged by, so the group stays judgeable once the +// cache evicts both. Each reader keeps its own, so the memory is bounded by the groups held. +type Superseded = { successor?: Successor }; + +// A successor, when it arrived, and its copy's fate (see `hooks.groupFate`). Not the consumer: +// a handed-out successor's own record would retain the next one, chaining every later group to +// the oldest held. +type Successor = { sequence: number; arrived: Instant; fate: () => Error | undefined }; + /** * The read side of a live track subscription, mirroring the Rust `Subscriber`. * @@ -1904,19 +1929,18 @@ export class Subscriber { // The wall clock judges it too: a group is also stale once its successor arrived a full // budget ago, so a peer whose timestamps stop advancing, an untimed track, or an empty // successor that never presents cannot pin a superseded group. The successor is re-read, - // not latched, like the reach. The newest group has no successor, so neither clock ever - // convicts it. `wake` arms a timer for the wall deadline, for a held read that waits on it. - #isStale(group: GroupConsumer, drift: Drift, wake = false): boolean { - if (this.#state.timeline[timelineIndex(this.#state.timeline, group.sequence)] !== group) return false; - return this.#staleAt(group.sequence, drift, wake); - } - - // The drift verdict for `sequence` by its successor alone, whether or not this track holds it. - #staleAt(sequence: number, drift: Drift, wake = false): boolean { - const arrival = this.#arrival(sequence, drift.end); - if (!arrival) return false; + // not latched, like the reach, except by a reader holding the group: it passes `held`, so + // the wall clock outlives the successor's eviction (see {@link #supersededAt}). The newest + // group has no successor, so neither clock ever convicts it. `wake` arms a timer for the + // wall deadline, for a held read that waits on it. + // + // Judged by its successor alone, so the group need not be in this track: evicted, or held + // from another copy. + #staleAt(sequence: number, drift: Drift, wake = false, held?: Superseded): boolean { + const arrived = this.#supersededAt(this.#arrival(sequence, drift.end), drift, held); + if (arrived === undefined) return false; // Plain arithmetic: an unenforced budget is infinite, which no duration brand holds. - const deadline = hooks.groupArrived(arrival) + drift.budget * 1_000_000; + const deadline = arrived + drift.budget * 1_000_000; if (now() >= deadline) return true; if (wake) this.#wakeAt(deadline); @@ -1931,6 +1955,33 @@ export class Subscriber { ); } + // When the wall clock starts: the cached `arrival`'s arrival, or that of the one `held` + // recorded, unless a group landed between them since. The record outlives the successor's + // eviction: otherwise an evicted successor restarts the clock at whichever newer group the + // cache still holds, which a budget past the cache window never reaches. A recorded + // successor that failed hands the clock on, as a re-read does, whether or not the cache + // has evicted it yet. + #supersededAt(arrival: GroupConsumer | undefined, drift: Drift, held?: Superseded): Instant | undefined { + const arrived = arrival && hooks.groupArrived(arrival); + if (!held) return arrived; + const recorded = held.successor; + if ( + recorded && + !failed(recorded) && + (drift.end === undefined || recorded.sequence < drift.end) && + (!arrival || arrival.sequence > recorded.sequence) + ) { + return recorded.arrived; + } + if (!arrival || arrived === undefined) { + held.successor = undefined; + } else if (recorded?.sequence !== arrival.sequence || recorded.arrived !== arrived || failed(recorded)) { + // Re-recorded only on a change, so judging the same successor again allocates nothing. + held.successor = { sequence: arrival.sequence, arrived, fate: hooks.groupFate(arrival) }; + } + return arrived; + } + // Bump {@link #wall} at `at`, unless a wakeup is already armed sooner. Each held read // re-arms on the bump, so one timer covers every deadline this subscriber waits on. #wakeAt(at: number): void { @@ -1953,8 +2004,9 @@ export class Subscriber { } #guard(group: GroupConsumer): GroupConsumer { + const held: Superseded = {}; hooks.expireGroup(group, { - expired: () => this.#isStale(group, this.#drift(), true), + expired: () => this.#staleAt(group.sequence, this.#drift(), true, held), changed: this.#expiryChanged(), }); return group; @@ -2061,10 +2113,13 @@ export class Subscriber { !hooks.hasUnreadGroup(subscriber, start, end, delivered) ); }; - hooks.staleAt = (subscriber, sequence) => ({ - expired: () => subscriber.#staleAt(sequence, subscriber.#drift(), true), - changed: subscriber.#expiryChanged(), - }); + hooks.staleAt = (subscriber, sequence) => { + const held: Superseded = {}; + return { + expired: () => subscriber.#staleAt(sequence, subscriber.#drift(), true, held), + changed: subscriber.#expiryChanged(), + }; + }; hooks.cachedGroup = (subscriber, sequence) => { const producer = subscriber.#state.producer; return producer && cachedMirror(producer, sequence); @@ -2435,7 +2490,7 @@ export class Subscriber { group.close(); continue; } - if (this.#isStale(group, drift)) { + if (this.#staleAt(group.sequence, drift)) { group.close(); const dropped = report(group.sequence, group.sequence + 1, old()); if (dropped) return dropped; @@ -2611,7 +2666,7 @@ export class Subscriber { for (const group of this.#state.groups.peek()) { if (group.sequence < start) continue; if (cursor.end !== undefined && group.sequence >= cursor.end) return undefined; - if (!this.#delivered.has(group.sequence) && !this.#isStale(group, drift)) return group.sequence; + if (!this.#delivered.has(group.sequence) && !this.#staleAt(group.sequence, drift)) return group.sequence; } return undefined; } @@ -2633,7 +2688,7 @@ export class Subscriber { this.#nextSequence = group.sequence + 1; // Every frame this group could still hold is past the budget, so keep // scanning: one pass walks a whole backlog off rather than replaying it. - if (this.#isStale(group, drift)) { + if (this.#staleAt(group.sequence, drift)) { group.close(); continue; } diff --git a/quest/m1/README.md b/quest/m1/README.md index 63d7a3a8c71..9141ba13a1e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -125,7 +125,6 @@ audit's receive-memory, priority, and lease bugs rank ahead of that reshape. - [Rust identity enforcement](/quest/m1/catalog-identity-rs.md) - `moq-mux` refuses a redefined rendition, and every Rust publisher mints a new rendition name for a decoder identity change and updates everything else in place - [IETF takeover](/quest/m1/ietf-takeover.md) - carry the shared takeover contract through the negotiated IETF epoch extension - [Identity and recovery guide](/quest/m1/identity-guide.md) - explain pinned catalogs, lazy jobs, takeover, restart and version compatibility -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - a handed-out group the cache evicted goes stale once its successor's arrival plus the budget passes, so its stalled stream resets - [Native probing](/quest/m1/probe-native.md) - moq-quic retransmits in-flight data during BBR3's ProbeBW_UP so an app-limited sender validates headroom up to the target - [Native rung disable](/quest/m1/simulcast-rung-disable.md) - Rust/FFI and OBS publishers stop encoding the top renditions their grant cannot fund and resume them once probing raises the estimate - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 05b5c6daade..6608e9c6285 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -226,6 +226,5 @@ unchanged. ## Related -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - the sibling for groups a reader already holds after the cache evicts them - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - fills the cache miss an evicted run leaves - [JS unbounded groups](/quest/m1/unbounded-groups/js.md) - the JS byte bound for groups and the runs between them; not required, since JS runs never outgrow the JS cache diff --git a/quest/m1/evicted-group-stale.md b/quest/m1/evicted-group-stale.md deleted file mode 100644 index 0fa8f8dee59..00000000000 --- a/quest/m1/evicted-group-stale.md +++ /dev/null @@ -1,50 +0,0 @@ -# [S] An evicted group is judged stale by its successor - -## Goal - -A group already handed to a reader stays judgeable after the cache evicts -it: once its successor's arrival plus the reader's budget has passed, the -group is stale, the same verdict `hooks.staleAt` already gives a resumed -copy's group by sequence alone. A serve loop that waits on such a group (a -group stream whose FIN never completes) is then reset as `Old`, whatever the -budget. - -Today the verdict needs the group to still be cached. Rust `GroupExpiry` -(`rs/moq-net/src/model/track.rs`) requires the sequence in the cache lookup, -and JS `#isStale` (`js/net/src/track.ts`) requires the group in the -timeline. Both caches evict an idle group after their window (Rust -`cache::DEFAULT_EXPIRY`, JS `CACHE_WINDOW`, both 30 s), so under a budget of -about 30 s or more a stalled group is never convicted. Since -[#6059](https://github.com/moq-dev/moq/pull/6059) the same stalled stream -also holds the IETF PUBLISH_DONE, which waits on a finished group's FIN -under the budget. - -## Plan - -Found in #6059's follow-ups and decided with the maintainer on 2026-10-20 as -a sibling of [Drop set](/quest/m1/drop-set.md): that quest judges missing -groups, this one judges groups a reader already holds. - -- Judge by the successor, as `#staleAt` does for a sequence the track does - not hold. The verdict needs the successor's arrival (and its reach, for the - media-time arm), not the evicted group itself. -- Find out what the cache-membership checks guard before dropping them, such - as a different copy now occupying the sequence, and keep that guard. -- Settle what the verdict reads once the successor is evicted too. It must - never convict early: any later group arrived no earlier than the successor, - and the newest group is never stale. -- Check `FetchExpiry` beside `GroupExpiry`, which has the same cache-lookup - condition. - -Regression tests on mock time, Rust and JS, each failing without the fix, -with a budget of at least the cache window: a handed-out group whose stream -never completes is evicted, then reset as `Old` once its successor's arrival -plus the budget passes, and on IETF the PUBLISH_DONE then goes out. The -newest group is still never convicted. - -Public API: none. Wire: none. - -## Related - -- [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget -- [#6059](https://github.com/moq-dev/moq/pull/6059) - bounds a stalled group stream's FIN wait by the age budget, which this extends past eviction diff --git a/quest/m1/probe-native.md b/quest/m1/probe-native.md index e9e3c607ac8..e09c59bea70 100644 --- a/quest/m1/probe-native.md +++ b/quest/m1/probe-native.md @@ -39,8 +39,8 @@ mechanism wins first" step: stream as `Old`, even after its FIN while bytes await acknowledgment (`poll_expired_while_pending`), and moq-quic never resends a reset stream's bytes, so probing never sends past a group's expiry. A group the cache - evicted is only judged once [evicted-group staleness](/quest/m1/evicted-group-stale.md) - lands, so it is required, with a test that evicts a group mid-probe. Rejected: requiring + evicted is still judged by its successor's arrival; a test evicts a group + mid-probe to cover it. Rejected: requiring [per-stream deadlines](/quest/m2/quic-deadline.md) first. They predict whether bytes can still arrive before the deadline, which tightens loss recovery too, but expiry itself already reaches moq-quic as a reset. @@ -78,10 +78,6 @@ Padding is browser-only, so the native [viewer up-switch](/quest/m2/probe/viewer-upswitch.md) then go back to the maintainer instead of falling back to it. -## Required - -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - an evicted group's stream resets once stale, so probing never retransmits it - ## Related - [Per-stream deadlines](/quest/m2/quic-deadline.md) - decides when in-flight data has expired diff --git a/quest/m2/group-failed.md b/quest/m2/group-failed.md index 7e0d1fa52a7..7108bfc559f 100644 --- a/quest/m2/group-failed.md +++ b/quest/m2/group-failed.md @@ -36,7 +36,3 @@ distinct failed groups with no consumer reading keeps the track's retained state constant. Public API: new, mirrored across Rust and JS. Wire: none. - -## Related - -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - more stalled groups reset as `Old`, which this would surface diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 4996a1defb2..8868ef47f12 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -6912,11 +6912,20 @@ mod serve_tests { /// The same, with the broadcast announced on `route`. fn serve_route(version: Version, info: crate::track::Info, route: crate::origin::Route) -> Serve { + serve_session(version, info, route, ScriptedSession::per_stream(vec![Vec::new()])) + } + + /// The same, over `session`. + fn serve_session( + version: Version, + info: crate::track::Info, + route: crate::origin::Route, + session: ScriptedSession, + ) -> Serve { let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let broadcast = origin.publish("room", route).unwrap(); let track = broadcast.create_track("video", info).unwrap(); - let session = ScriptedSession::per_stream(vec![Vec::new()]); let log = session.log.clone(); let peer_setup = peer::PeerSetup::default(); @@ -7398,6 +7407,84 @@ mod serve_tests { wire.assert_streams_closed_before_publish_done(); } + /// A finished group's stream whose FIN is never acknowledged holds PUBLISH_DONE only until + /// the group goes stale: its successor's arrival plus the budget, even under a budget past + /// the cache window, which evicts both groups first. + #[moq_clock::test] + async fn publish_done_waits_on_an_unacknowledged_fin_until_the_group_is_stale() { + let version = Version::Draft21; + let session = ScriptedSession::per_stream(vec![Vec::new()]).with_unacked_uni_fins(1); + let h = serve_session( + version, + crate::track::Info::timed(), + crate::origin::Route::default(), + session, + ); + // Well past the cache window, which only dates idleness once a sweep passes. + let budget = crate::cache::DEFAULT_EXPIRY * 4; + let mut msg = subscribe(Filter::NextObject, None); + msg.max_age = Some(budget.as_millis() as u64); + + // `create_broadcast` registers the broadcast from a spawned task. + moq_clock::sleep(Duration::from_millis(1)).await; + let stream = Stream::open(&mut crate::driver::Transport::new(h.session.clone()), version) + .await + .unwrap(); + let mut serve = std::pin::pin!(h.publisher.clone().run_subscribe_stream(stream, msg)); + // The subscription then parks at the live edge. + for _ in 0..200 { + assert!(futures::poll!(serve.as_mut()).is_pending()); + } + + // Frozen timestamps, so only the wall clock since group 1 arrived convicts group 0. + let append = || { + let mut group = h.track.append_group().unwrap(); + group.write_frame(timestamp(), b"x".as_slice()).unwrap(); + group.finish().unwrap(); + }; + let finished = || { + data_streams(&h.log) + .values() + .filter(|sent| sent.contains(&Sent::Finish)) + .count() + }; + let step = Duration::from_secs(1); + async fn wait(mut serve: std::pin::Pin<&mut F>, seconds: u64) { + for _ in 0..seconds { + moq_clock::sim::advance(Duration::from_secs(1)).await; + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "PUBLISH_DONE went out early" + ); + } + } + append(); + serve_until(serve.as_mut(), "group 0's FIN went out", || finished() == 1).await; + append(); + serve_until(serve.as_mut(), "group 1's stream finished", || finished() == 2).await; + + // Group 2 arrives later, so a verdict that forgot group 1 would wait for 2's deadline. + wait(serve.as_mut(), 10).await; + append(); + h.track.finish().unwrap(); + serve_until(serve.as_mut(), "group 2's stream finished", || finished() == 3).await; + + // Groups 0 and 1 go idle past the cache window long before the deadline. + wait(serve.as_mut(), budget.as_secs() - 12).await; + assert!(h.log.resets().is_empty()); + + moq_clock::sim::advance(step).await; + moq_clock::timeout(step, serve) + .await + .expect("the stale group releases PUBLISH_DONE") + .unwrap(); + assert_eq!(h.log.resets(), vec![crate::StreamError::Cancel.to_code()]); + let wire = Wire::read(&h.log, version); + let (status, _) = wire.publish_done.expect("PUBLISH_DONE was sent"); + assert_eq!(status, ietf::PublishDoneStatus::TrackEnded.code(version)); + wire.assert_streams_closed_before_publish_done(); + } + /// A track that ends with an Unsupported error (a relay refusing a copy whose metadata /// changed does) ends its subscription with INTERNAL_ERROR: UPDATE_FAILED is only for an /// update the publisher could not apply. diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 0a7644ad1c3..100e0d58040 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -911,6 +911,8 @@ pub struct ScriptedSession { incoming_bidis: Arc>>>, /// Set by [`Self::with_unacked_fin`]. unacked_fin: bool, + /// Set by [`Self::with_unacked_uni_fins`]; shared, so every clone counts down one budget. + unacked_unis: Arc, datagram_size: usize, /// Set by [`Self::with_groups`]. groups: bool, @@ -939,6 +941,7 @@ impl ScriptedSession { incoming_unis: Arc::new(Mutex::new(std::collections::VecDeque::new())), incoming_bidis: Arc::new(Mutex::new(std::collections::VecDeque::new())), unacked_fin: false, + unacked_unis: Default::default(), datagram_size: 0, groups: false, } @@ -956,6 +959,13 @@ impl ScriptedSession { self } + /// Never acknowledge the FIN of the next `count` unidirectional streams opened, like a + /// peer stalled on those alone. + pub fn with_unacked_uni_fins(self, count: usize) -> Self { + self.unacked_unis.store(count, Ordering::Relaxed); + self + } + /// Have the peer open one unidirectional stream per script, in order, each replaying /// its bytes and then going quiet (or reporting EOF, on an `eof` session). /// @@ -1104,7 +1114,12 @@ impl poll::Session for ScriptedSession { _cx: &mut Context<'_>, options: poll::SendOptions<'_, SinkGroup>, ) -> Poll> { - Poll::Ready(Ok(SinkSend::new(self.log.clone()).opened(options))) + let mut send = SinkSend::new(self.log.clone()); + send.unacked_fin = self + .unacked_unis + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| count.checked_sub(1)) + .is_ok(); + Poll::Ready(Ok(send.opened(options))) } fn send_group(&mut self, priority: i64) -> Option { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 86606f313fa..dcbb7898125 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -966,6 +966,17 @@ pub(crate) struct Handle { alive: Arc, } +/// Whether a group failed, without holding it open or pinning its frames. +pub(super) struct Fate(Arc); + +impl Fate { + /// Whether the group was aborted other than by cache policy (`Old`, `Timeout`, + /// `Evicted`): it failed or was given up, rather than aged out of a cache. + pub(super) fn failed(&self) -> bool { + self.0.failed.load(Ordering::Acquire) + } +} + struct Alive { // Monotone mirror of `GroupState::abort` for track scans that already hold the // track lock. A stale false only hands out a group that is concurrently aborting; @@ -982,6 +993,9 @@ struct Alive { // policy (`Old`, `Timeout`, `Evicted`). Set before `aborted`, under the same rules, so // a track scan learns why without taking the group lock. cut: OnceLock, + // Set before `aborted` when the abort was not cache policy: the group failed or was given + // up rather than aged out of a cache. Read through [`Fate`] once the group is gone. + failed: AtomicBool, // Monotone mirror of `GroupState::started` for the first frame or the finish, set after // the group lock is released and before the track is told: a track scan reading a stale // false judges the group frameless until that wake re-judges it. @@ -1111,6 +1125,7 @@ impl Producer { aborted: AtomicBool::new(false), checked: AtomicBool::new(false), cut: OnceLock::new(), + failed: AtomicBool::new(false), started: AtomicBool::new(false), access, }); @@ -1872,6 +1887,9 @@ impl Handle { if cut { let _ = self.alive.cut.set(err.clone()); } + if !policy { + self.alive.failed.store(true, Ordering::Release); + } guard.abort = Some(err); self.alive.aborted.store(true, Ordering::Release); guard.release(salvage); @@ -1898,6 +1916,11 @@ impl Handle { self.alive.cut.get() } + /// This group's [`Fate`], readable after the group and its frames are gone. + pub(super) fn fate(&self) -> Fate { + Fate(self.alive.clone()) + } + /// Whether the group was finished: it holds every frame it will ever have. pub(crate) fn is_finished(&self) -> bool { self.state.read().fin.is_some() diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8c3cc505efa..922acaa758a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1354,13 +1354,44 @@ impl TrackState { /// so judging the newest group, the common case, costs no clock read. fn superseded(&self, sequence: u64, servable: &Servable, budget: Duration) -> bool { self.arrival(sequence, servable) - .is_some_and(|successor| self.aged(successor, budget)) + .is_some_and(|successor| self.aged(successor.arrived, budget)) } - /// Whether a full `budget` passed since `successor` arrived. - fn aged(&self, successor: &Slot, budget: Duration) -> bool { + /// Whether a full `budget` passed since a successor `arrived`. + fn aged(&self, arrived: crate::time::Instant, budget: Duration) -> bool { let now = self.cache.pool().now_wall(); - now.saturating_duration_since(successor.arrived) >= budget + now.saturating_duration_since(arrived) >= budget + } + + /// When a group was superseded, given `arrival`, its cached successor in `servable`: when + /// `arrival` arrived, or when the successor `superseded` recorded did, unless a group landed + /// between them since. The record outlives the successor's eviction: otherwise an evicted + /// successor restarts the clock at whichever newer group the cache still holds, which a + /// budget past the cache window never reaches. A recorded successor that failed hands the + /// clock on, as a re-read does, whether or not the cache has reclaimed it yet. + fn superseded_at( + &self, + arrival: Option<&Slot>, + servable: &Servable, + superseded: Option<&Superseded>, + ) -> Option { + let Some(superseded) = superseded else { + return arrival.map(|slot| slot.arrived); + }; + let mut last = superseded.0.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(recorded) = &*last + && !recorded.fate.failed() + && servable.contains(recorded.sequence) + && arrival.is_none_or(|slot| slot.group.sequence > recorded.sequence) + { + return Some(recorded.arrived); + } + *last = arrival.map(|slot| Successor { + sequence: slot.group.sequence, + arrived: slot.arrived, + fate: slot.group.fate(), + }); + arrival.map(|slot| slot.arrived) } /// Whether the group at `sequence` has drifted further behind `edge` than `budget` @@ -1419,18 +1450,27 @@ impl TrackState { /// `budget`. Parking on every track change instead wakes each of N parked reads per append. /// /// The caller parks on a group landing above `sequence` first, in the way that fits - /// whether this track holds it. - fn poll_drifted(&self, sequence: u64, servable: &Servable, budget: Duration, waiter: &kio::Waiter) -> bool { + /// whether this track holds it. A reader holding the group passes its `superseded`, so + /// the wall clock outlives the successor's eviction; see [`Self::superseded_at`]. + fn poll_drifted( + &self, + sequence: u64, + servable: &Servable, + budget: Duration, + superseded: Option<&Superseded>, + waiter: &kio::Waiter, + ) -> bool { // Unbounded until a group lands above. Registered before judging, so a racing abort // is observed or wakes us, and armed for whichever successor this pass selected, so // re-selecting re-arms it. - let Some(arrival) = self.watch_abort(waiter, |state| state.arrival(sequence, servable)) else { + let arrival = self.watch_abort(waiter, |state| state.arrival(sequence, servable)); + let Some(arrived) = self.superseded_at(arrival, servable, superseded) else { return false; }; - if let Some(deadline) = arrival.arrived.checked_add(budget) { + if let Some(deadline) = arrived.checked_add(budget) { self.cache.watch_wall(deadline, waiter); } - if self.aged(arrival, budget) { + if self.aged(arrived, budget) { return true; } let reach = loop { @@ -4463,7 +4503,7 @@ impl Consumer { let state = state.read(); let budget = clamp_max_delay(budget, state.max_age_bound()); state.cache.wakes().watch_held(sequence, waiter); - match state.poll_drifted(sequence, &Servable::ALL, budget, waiter) { + match state.poll_drifted(sequence, &Servable::ALL, budget, None, waiter) { true => Poll::Ready(()), false => Poll::Pending, } @@ -5150,7 +5190,7 @@ impl kio::Task for Fetching { let state = state.read(); let budget = clamp_max_delay(self.max_delay.unwrap_or(Duration::MAX), state.max_age_bound()); state.cache.wakes().watch_landing(sequence, waiter); - if state.poll_drifted(sequence, &Servable::ALL, budget, waiter) { + if state.poll_drifted(sequence, &Servable::ALL, budget, None, waiter) { return Poll::Ready(Err(Error::Old)); } } @@ -5161,6 +5201,7 @@ impl kio::Task for Fetching { state: state.weak(), sequence, max_delay: self.max_delay, + superseded: Superseded::default(), }); let resolve = |mut group: group::Consumer| { group.start_at(frame_start); @@ -5333,6 +5374,7 @@ struct GroupExpiry { /// What the cursor can be served; see [`Cursor::published`]. servable: kio::Consumer, sequence: u64, + superseded: Superseded, } /// Keeps one fetched group tied to this caller's age budget without adding live demand. @@ -5340,6 +5382,20 @@ struct FetchExpiry { state: kio::ConsumerWeak, sequence: u64, max_delay: Option, + superseded: Superseded, +} + +/// The successor a handed-out group was last judged by, so the group stays judgeable once +/// the cache evicts both; see [`TrackState::superseded_at`]. Each reader keeps its own, so +/// the memory is bounded by the groups readers hold. +#[derive(Default)] +struct Superseded(Mutex>); + +/// A successor, when it arrived, and whether it has since failed. +struct Successor { + sequence: u64, + arrived: crate::time::Instant, + fate: group::Fate, } impl group::Expiry for FetchExpiry { @@ -5350,6 +5406,7 @@ impl group::Expiry for FetchExpiry { state: state.unwrap_or_else(|| self.state.clone()), sequence: self.sequence, max_delay: self.max_delay, + superseded: Superseded::default(), }) } @@ -5363,7 +5420,7 @@ impl group::Expiry for FetchExpiry { }; let budget = clamp_max_delay(budget, state.max_age_bound()); state.cache.wakes().watch_landing(self.sequence, waiter); - state.poll_drifted(self.sequence, &Servable::ALL, budget, waiter) && state.lookup.contains_key(&self.sequence) + state.poll_drifted(self.sequence, &Servable::ALL, budget, Some(&self.superseded), waiter) } } @@ -5376,6 +5433,7 @@ impl group::Expiry for GroupExpiry { subscription: self.subscription.clone(), servable: self.servable.clone(), sequence: self.sequence, + superseded: Superseded::default(), }) } @@ -5398,7 +5456,7 @@ impl group::Expiry for GroupExpiry { let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); state.cache.wakes().watch_landing(self.sequence, waiter); - state.poll_drifted(self.sequence, &servable, budget, waiter) && state.lookup.contains_key(&self.sequence) + state.poll_drifted(self.sequence, &servable, budget, Some(&self.superseded), waiter) } } @@ -5483,6 +5541,14 @@ impl Servable { /// No group. const NONE: Self = Self::Run { start: 0, end: Some(0) }; + /// Whether group `sequence` is among these. + fn contains(&self, sequence: u64) -> bool { + match self { + Self::Run { start, end } => sequence >= *start && end.is_none_or(|end| sequence < end), + Self::Runs(runs) => runs.contains(&sequence), + } + } + /// The groups in `runs`, disjoint and lowest first. Only several runs allocate: every /// subscriber resolves one, so the common single run stays off the heap. fn new(runs: impl Iterator>) -> Self { @@ -6105,6 +6171,7 @@ impl Cursor { subscription: self.subscription.consume(), servable: self.published.consume(), sequence, + superseded: Superseded::default(), })) } @@ -9722,6 +9789,165 @@ mod test { } } + /// A handed-out group stays judgeable after the cache evicts it, and its successor too: + /// under a budget past the cache window, a stream still waiting on it (a FIN never + /// acknowledged) is convicted once its successor's arrival plus the budget passes, not + /// by whichever newer group the cache still holds. The newest group never is. A + /// subscribed group and a fetched one are judged alike. + #[test] + fn an_evicted_group_is_judged_by_its_evicted_successor() { + let mut producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); + let budget = cache::DEFAULT_EXPIRY * 2; + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget)); + let waiter = kio::Waiter::noop(); + let expired = |held: &mut [group::Consumer; 2]| { + held.each_mut() + .map(|group| group.poll_expired_while_pending(&waiter, true)) + }; + + // Frozen timestamps, so the wall clock alone convicts. + append_at(&mut producer, 0); + let fetch = group::Fetch::default().with_max_delay(budget); + let mut held = [ + subscriber.assert_group(), + producer + .consume() + .fetch_group(0, fetch) + .now_or_never() + .unwrap() + .unwrap(), + ]; + for group in &mut held { + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_none()); + } + assert_eq!(expired(&mut held), [false; 2], "the newest group"); + + elapse(&producer, Duration::from_secs(1)); + let mut last = append_at(&mut producer, 0); // the successor + assert_eq!(expired(&mut held), [false; 2]); + + // The track runs on, a group a second, until a second short of the deadline. + for _ in 0..budget.as_secs() - 1 { + elapse(&producer, Duration::from_secs(1)); + last = append_at(&mut producer, 0); + } + producer.broadcast.pool.sweep(); + let state = producer.state.read(); + assert!( + !state.lookup.contains_key(&0) && !state.lookup.contains_key(&1), + "both evicted" + ); + drop(state); + assert_eq!(expired(&mut held), [false; 2], "early"); + + elapse(&producer, Duration::from_secs(1)); + assert_eq!(expired(&mut held), [true; 2], "the successor arrived a budget ago"); + + // The newest group outlives any budget. + let mut newest = subscriber.assert_group(); + while newest.sequence < last { + newest = subscriber.assert_group(); + } + elapse(&producer, budget * 2); + assert!(!newest.poll_expired_while_pending(&waiter, true), "the newest group"); + } + + /// A recorded successor that failed hands the clock on to the next live group even once + /// the cache reclaimed it before the reader looked again, rather than convicting by the + /// failed one's arrival. A subscribed group and a fetched one are judged alike. + #[test] + fn a_reclaimed_failed_successor_hands_the_clock_on() { + let mut producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); + let budget = cache::DEFAULT_EXPIRY * 2; + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget)); + let waiter = kio::Waiter::noop(); + let expired = |held: &mut [group::Consumer; 2]| { + held.each_mut() + .map(|group| group.poll_expired_while_pending(&waiter, true)) + }; + + append_at(&mut producer, 0); + let fetch = group::Fetch::default().with_max_delay(budget); + let mut held = [ + subscriber.assert_group(), + producer + .consume() + .fetch_group(0, fetch) + .now_or_never() + .unwrap() + .unwrap(), + ]; + for group in &mut held { + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_none()); + } + + let failed = producer.append_group().unwrap(); + assert_eq!(expired(&mut held), [false; 2], "the successor is recorded"); + failed.abort(Error::Cancel).unwrap(); + let replacement = Duration::from_secs(10); + elapse(&producer, replacement); + append_at(&mut producer, 0); + producer.broadcast.pool.sweep(); + assert!(!producer.state.read().lookup.contains_key(&1), "reclaimed"); + + elapse(&producer, budget - replacement); + assert_eq!( + expired(&mut held), + [false; 2], + "the failed successor arrived a budget ago" + ); + + elapse(&producer, replacement); + assert_eq!(expired(&mut held), [true; 2], "the next live one arrived a budget ago"); + } + + /// On an ended track the cache expires idle groups in place rather than reclaiming them, + /// so the reads skip them like any aborted group. A recorded successor that aged out that + /// way still convicts the group before it once it arrived a budget ago. + #[test] + fn an_ended_track_judges_by_its_expired_successor() { + let pool = cache::Pool::new(cache::Config::default().with_expiry(cache::DEFAULT_EXPIRY)); + let mut producer = track_producer_pooled("test", pool.clone()); + let budget = cache::DEFAULT_EXPIRY * 2; + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget)); + let waiter = kio::Waiter::noop(); + + append_at(&mut producer, 0); + let mut held = subscriber.assert_group(); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_none()); + + pool.step(Duration::from_secs(1)); + append_at(&mut producer, 0); // the successor + assert!( + !held.poll_expired_while_pending(&waiter, true), + "the successor is recorded" + ); + let state = producer.state.downgrade(); + producer.finish().unwrap(); + drop(producer); + + // The first sweep dates the groups idle, the second expires them. + let idle = cache::DEFAULT_EXPIRY + Duration::from_secs(1); + pool.sweep(); + pool.step(idle); + pool.sweep(); + assert_eq!( + state.read(|state| state.lookup.get(&1).is_some_and(Slot::is_aborted)), + Some(true), + "expired in place" + ); + assert!(!held.poll_expired_while_pending(&waiter, true), "early"); + + pool.step(budget - idle); + assert!( + held.poll_expired_while_pending(&waiter, true), + "the successor arrived a budget ago" + ); + } + /// Empty groups are handed out like any other and retired by the wall clock like an /// untimed one, whether they finished without a frame or are still open with none. A /// publisher flooding a run of them cannot pin the group before the run either.