From 32d08444ef45d73f104087993894712ea97e7ab9 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:31:23 -0700 Subject: [PATCH 1/5] chore(quest): claim quest/m1/evicted-group-stale.md Quest-Claim: ce09c8130ee203f8 From a82c95ff185f9f66b4864f8ea5d625407bb9a5b4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:07:40 -0700 Subject: [PATCH 2/5] fix(net): judge an evicted group by the successor its reader saw A handed-out group's drift verdict required the group to still be cached (Rust `GroupExpiry`/`FetchExpiry` checked the cache lookup, JS `#isStale` the timeline), so under a budget past the cache window a stream waiting on an unacknowledged FIN was never reset, and on IETF it held PUBLISH_DONE. The check was left from when the verdict read the group's own timestamp; it now reads only the successor, so drop it. Dropping it alone is not enough: the cache evicts the successor too, and judging by whichever newer group it still holds restarts the clock past any budget over the window. Each reader now remembers the successor it was judged by and keeps its arrival once that successor leaves the cache, unless a group lands between them. An aborted successor still hands the clock on. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EeyiEy5FWz3z44DYZVo7vU --- js/net/src/ietf/publisher.test.ts | 90 ++++++++++++++- js/net/src/track.ts | 63 +++++++--- quest/m1/README.md | 1 - quest/m1/drop-set.md | 1 - quest/m1/evicted-group-stale.md | 50 -------- quest/m1/probe-native.md | 8 +- quest/m2/group-failed.md | 4 - rs/moq-net/src/ietf/publisher.rs | 87 +++++++++++++- rs/moq-net/src/lite/test_transport.rs | 17 ++- rs/moq-net/src/model/track.rs | 160 +++++++++++++++++++++++--- 10 files changed, 383 insertions(+), 98 deletions(-) delete mode 100644 quest/m1/evicted-group-stale.md diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index f4c31751de8..2c4eb5b4179 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -791,8 +791,7 @@ test("a group stream whose FIN never completes is reset when the group expires", write(0); await closeStarted; - // The newest group is never stale, however long its FIN takes. Kept inside the cache's - // idle window: a group evicted once superseded is no longer judged. + // The newest group is never stale, however long its FIN takes. await clock.advance(Nano.fromMilli(Milli(budget * 2))); expect(reset).toBe(false); @@ -812,6 +811,93 @@ test("a group stream whose FIN never completes is reset when the group expires", } }); +// Under a budget past the cache window, the cache evicts a stalled group and its successor +// before the budget convicts the group. The verdict still waits only for the successor's +// arrival plus the budget, not for whichever newer group the cache holds, and PUBLISH_DONE +// follows the reset. +test("a group stream whose FIN never completes is reset after the cache evicts it", async () => { + const clock = new Manual(); + const restore = clock.install(); + const pair = createMockTransportPair(ALPN.DRAFT_19); + + let closing!: () => void; + const closeStarted = new Promise((resolve) => { + closing = resolve; + }); + let reset = false; + let resetting!: () => void; + const streamReset = new Promise((resolve) => { + resetting = resolve; + }); + const stalled = new WritableStream({ + start(controller) { + controller.signal.addEventListener("abort", () => { + reset = true; + resetting(); + }); + }, + close() { + closing(); + return new Promise(() => {}); + }, + }); + spyOn(pair.server, "createUnidirectionalStream").mockImplementationOnce(async () => stalled); + + // Twice the cache's idle window. + const budget = Milli(60_000); + const { pub, origin } = publisher(pair.server); + const broadcast = publish(origin, Path.from("test")); + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: budget }); + const client = await Stream.open(pair.client, { version: VERSION }); + const server = await Stream.accept(pair.server, VERSION); + if (!server) throw new Error("publisher never accepted the subscribe stream"); + + try { + const running = pub.runSubscribe( + new Subscribe({ + requestId: 0n, + trackNamespace: Path.from("test"), + trackName: "video", + subscriberPriority: 0, + }), + server, + ); + expect(await client.reader.u53()).toBe(SubscribeOk.id); + await SubscribeOk.decode(client.reader, VERSION); + + // Frozen timestamps, so only the wall clock since group 1 arrived convicts group 0. + const write = (sequence: number) => { + const group = new GroupProducer(sequence); + group.writeFrame({ payload: new TextEncoder().encode("frame"), timestamp: Timestamp.fromMillis(0) }); + group.close(); + track.writeGroup(group); + }; + + write(0); + await closeStarted; + write(1); + // A later group, so a verdict that forgot group 1 would wait for this one's deadline. + const later = Milli(10_000); + await clock.advance(Nano.fromMilli(later)); + write(2); + track.close(); + + await clock.advance(Nano.fromMilli(Milli(budget - later - 1))); + expect(reset).toBe(false); + await clock.advance(Nano.fromMilli(Milli(1))); + await streamReset; + + await running; + expect(await client.reader.u53()).toBe(PublishDone.id); + await PublishDone.decode(client.reader, VERSION); + } finally { + client.close(); + broadcast.close(); + origin.close(); + restore(); + } +}); + // A group can go stale while its stream is still opening. Serving it must abandon the group // without starting a write: an abandoned write rejects once the stream resets, and nothing // would handle it (Node exits on the first unhandled rejection). diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 0fccc0d96ce..d578d7f45f7 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1786,6 +1786,10 @@ type Drift = { end?: number; }; +// The successor a handed-out group was last judged by, so the group stays judgeable once the +// cache evicts both. Each reader keeps its own, so the memory is bounded by the groups held. +type Superseded = { successor?: GroupConsumer }; + /** * The read side of a live track subscription, mirroring the Rust `Subscriber`. * @@ -1917,16 +1921,15 @@ export class Subscriber { // The wall clock judges it too: a group is also stale once its successor arrived a full // budget ago, so a peer whose timestamps stop advancing, an untimed track, or an empty // successor that never presents cannot pin a superseded group. The successor is re-read, - // not latched, like the reach. The newest group has no successor, so neither clock ever - // convicts it. `wake` arms a timer for the wall deadline, for a held read that waits on it. - #isStale(group: GroupConsumer, drift: Drift, wake = false): boolean { - if (this.#state.timeline[timelineIndex(this.#state.timeline, group.sequence)] !== group) return false; - return this.#staleAt(group.sequence, drift, wake); - } - - // The drift verdict for `sequence` by its successor alone, whether or not this track holds it. - #staleAt(sequence: number, drift: Drift, wake = false): boolean { - const arrival = this.#arrival(sequence, drift.end); + // not latched, like the reach, except by a reader holding the group: it passes `held`, so + // the wall clock outlives the successor's eviction (see {@link #supersededBy}). The newest + // group has no successor, so neither clock ever convicts it. `wake` arms a timer for the + // wall deadline, for a held read that waits on it. + // + // Judged by its successor alone, so the group need not be in this track: evicted, or held + // from another copy. + #staleAt(sequence: number, drift: Drift, wake = false, held?: Superseded): boolean { + const arrival = this.#supersededBy(this.#arrival(sequence, drift.end), drift, held); if (!arrival) return false; // Plain arithmetic: an unenforced budget is infinite, which no duration brand holds. const deadline = hooks.groupArrived(arrival) + drift.budget * 1_000_000; @@ -1944,6 +1947,26 @@ export class Subscriber { ); } + // The successor whose arrival starts the wall clock: the cached `arrival`, or once the one + // `held` recorded leaves the timeline, that one, unless a group landed between them since. + // Otherwise an evicted successor restarts the clock at whichever newer group the cache + // still holds, which a budget past the cache window never reaches. One still in the + // timeline but aborted hands the clock on, as a re-read does. + #supersededBy(arrival: GroupConsumer | undefined, drift: Drift, held?: Superseded): GroupConsumer | undefined { + if (!held) return arrival; + const evicted = held.successor; + if ( + evicted && + this.#state.timeline[timelineIndex(this.#state.timeline, evicted.sequence)] !== evicted && + (drift.end === undefined || evicted.sequence < drift.end) && + (!arrival || arrival.sequence > evicted.sequence) + ) { + return evicted; + } + held.successor = arrival; + return arrival; + } + // Bump {@link #wall} at `at`, unless a wakeup is already armed sooner. Each held read // re-arms on the bump, so one timer covers every deadline this subscriber waits on. #wakeAt(at: number): void { @@ -1966,8 +1989,9 @@ export class Subscriber { } #guard(group: GroupConsumer): GroupConsumer { + const held: Superseded = {}; hooks.expireGroup(group, { - expired: () => this.#isStale(group, this.#drift(), true), + expired: () => this.#staleAt(group.sequence, this.#drift(), true, held), changed: this.#expiryChanged(), }); return group; @@ -2077,10 +2101,13 @@ export class Subscriber { !hooks.hasUnreadGroup(subscriber, start, end, delivered) ); }; - hooks.staleAt = (subscriber, sequence) => ({ - expired: () => subscriber.#staleAt(sequence, subscriber.#drift(), true), - changed: subscriber.#expiryChanged(), - }); + hooks.staleAt = (subscriber, sequence) => { + const held: Superseded = {}; + return { + expired: () => subscriber.#staleAt(sequence, subscriber.#drift(), true, held), + changed: subscriber.#expiryChanged(), + }; + }; hooks.cachedGroup = (subscriber, sequence) => { const producer = subscriber.#state.producer; return producer && cachedMirror(producer, sequence); @@ -2456,7 +2483,7 @@ export class Subscriber { group.close(); continue; } - if (this.#isStale(group, drift)) { + if (this.#staleAt(group.sequence, drift)) { group.close(); const dropped = report(group.sequence, group.sequence + 1, old()); if (dropped) return dropped; @@ -2632,7 +2659,7 @@ export class Subscriber { for (const group of this.#state.groups.peek()) { if (group.sequence < start) continue; if (cursor.end !== undefined && group.sequence >= cursor.end) return undefined; - if (!this.#delivered.has(group.sequence) && !this.#isStale(group, drift)) return group.sequence; + if (!this.#delivered.has(group.sequence) && !this.#staleAt(group.sequence, drift)) return group.sequence; } return undefined; } @@ -2654,7 +2681,7 @@ export class Subscriber { this.#nextSequence = group.sequence + 1; // Every frame this group could still hold is past the budget, so keep // scanning: one pass walks a whole backlog off rather than replaying it. - if (this.#isStale(group, drift)) { + if (this.#staleAt(group.sequence, drift)) { group.close(); continue; } diff --git a/quest/m1/README.md b/quest/m1/README.md index c0bb4fbd6cf..5fdc318ef13 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -110,7 +110,6 @@ tracks, so per-group stats stop costing cluster announces. - [Rust identity enforcement](/quest/m1/catalog-identity-rs.md) - `moq-mux` refuses a redefined rendition, and every Rust publisher mints a new rendition name for a decoder identity change and updates everything else in place - [IETF takeover](/quest/m1/ietf-takeover.md) - carry the shared takeover contract through the negotiated IETF epoch extension - [Identity and recovery guide](/quest/m1/identity-guide.md) - explain pinned catalogs, lazy jobs, takeover, restart and version compatibility -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - a handed-out group the cache evicted goes stale once its successor's arrival plus the budget passes, so its stalled stream resets - [Native probing](/quest/m1/probe-native.md) - moq-quic retransmits in-flight data during BBR3's ProbeBW_UP so an app-limited sender validates headroom up to the target - [Native rung disable](/quest/m1/simulcast-rung-disable.md) - Rust/FFI and OBS publishers stop encoding the top renditions their grant cannot fund and resume them once probing raises the estimate - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 957622b3b20..c2922d8a5bf 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -219,6 +219,5 @@ unchanged. ## Related -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - the sibling for groups a reader already holds after the cache evicts them - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - fills the cache miss an evicted run leaves - [JS unbounded groups](/quest/m2/unbounded-groups/js.md) - the JS byte bound for groups and the runs between them; not required, since JS runs never outgrow the JS cache diff --git a/quest/m1/evicted-group-stale.md b/quest/m1/evicted-group-stale.md deleted file mode 100644 index 0fa8f8dee59..00000000000 --- a/quest/m1/evicted-group-stale.md +++ /dev/null @@ -1,50 +0,0 @@ -# [S] An evicted group is judged stale by its successor - -## Goal - -A group already handed to a reader stays judgeable after the cache evicts -it: once its successor's arrival plus the reader's budget has passed, the -group is stale, the same verdict `hooks.staleAt` already gives a resumed -copy's group by sequence alone. A serve loop that waits on such a group (a -group stream whose FIN never completes) is then reset as `Old`, whatever the -budget. - -Today the verdict needs the group to still be cached. Rust `GroupExpiry` -(`rs/moq-net/src/model/track.rs`) requires the sequence in the cache lookup, -and JS `#isStale` (`js/net/src/track.ts`) requires the group in the -timeline. Both caches evict an idle group after their window (Rust -`cache::DEFAULT_EXPIRY`, JS `CACHE_WINDOW`, both 30 s), so under a budget of -about 30 s or more a stalled group is never convicted. Since -[#6059](https://github.com/moq-dev/moq/pull/6059) the same stalled stream -also holds the IETF PUBLISH_DONE, which waits on a finished group's FIN -under the budget. - -## Plan - -Found in #6059's follow-ups and decided with the maintainer on 2026-10-20 as -a sibling of [Drop set](/quest/m1/drop-set.md): that quest judges missing -groups, this one judges groups a reader already holds. - -- Judge by the successor, as `#staleAt` does for a sequence the track does - not hold. The verdict needs the successor's arrival (and its reach, for the - media-time arm), not the evicted group itself. -- Find out what the cache-membership checks guard before dropping them, such - as a different copy now occupying the sequence, and keep that guard. -- Settle what the verdict reads once the successor is evicted too. It must - never convict early: any later group arrived no earlier than the successor, - and the newest group is never stale. -- Check `FetchExpiry` beside `GroupExpiry`, which has the same cache-lookup - condition. - -Regression tests on mock time, Rust and JS, each failing without the fix, -with a budget of at least the cache window: a handed-out group whose stream -never completes is evicted, then reset as `Old` once its successor's arrival -plus the budget passes, and on IETF the PUBLISH_DONE then goes out. The -newest group is still never convicted. - -Public API: none. Wire: none. - -## Related - -- [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget -- [#6059](https://github.com/moq-dev/moq/pull/6059) - bounds a stalled group stream's FIN wait by the age budget, which this extends past eviction diff --git a/quest/m1/probe-native.md b/quest/m1/probe-native.md index e9e3c607ac8..e09c59bea70 100644 --- a/quest/m1/probe-native.md +++ b/quest/m1/probe-native.md @@ -39,8 +39,8 @@ mechanism wins first" step: stream as `Old`, even after its FIN while bytes await acknowledgment (`poll_expired_while_pending`), and moq-quic never resends a reset stream's bytes, so probing never sends past a group's expiry. A group the cache - evicted is only judged once [evicted-group staleness](/quest/m1/evicted-group-stale.md) - lands, so it is required, with a test that evicts a group mid-probe. Rejected: requiring + evicted is still judged by its successor's arrival; a test evicts a group + mid-probe to cover it. Rejected: requiring [per-stream deadlines](/quest/m2/quic-deadline.md) first. They predict whether bytes can still arrive before the deadline, which tightens loss recovery too, but expiry itself already reaches moq-quic as a reset. @@ -78,10 +78,6 @@ Padding is browser-only, so the native [viewer up-switch](/quest/m2/probe/viewer-upswitch.md) then go back to the maintainer instead of falling back to it. -## Required - -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - an evicted group's stream resets once stale, so probing never retransmits it - ## Related - [Per-stream deadlines](/quest/m2/quic-deadline.md) - decides when in-flight data has expired diff --git a/quest/m2/group-failed.md b/quest/m2/group-failed.md index 7e0d1fa52a7..7108bfc559f 100644 --- a/quest/m2/group-failed.md +++ b/quest/m2/group-failed.md @@ -36,7 +36,3 @@ distinct failed groups with no consumer reading keeps the track's retained state constant. Public API: new, mirrored across Rust and JS. Wire: none. - -## Related - -- [An evicted group is judged stale](/quest/m1/evicted-group-stale.md) - more stalled groups reset as `Old`, which this would surface diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 41307317744..69fcad52e1e 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -6702,11 +6702,20 @@ mod serve_tests { /// The same, with the broadcast announced on `route`. fn serve_route(version: Version, info: crate::track::Info, route: crate::origin::Route) -> Serve { + serve_session(version, info, route, ScriptedSession::per_stream(vec![Vec::new()])) + } + + /// The same, over `session`. + fn serve_session( + version: Version, + info: crate::track::Info, + route: crate::origin::Route, + session: ScriptedSession, + ) -> Serve { let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let broadcast = origin.publish("room", route).unwrap(); let track = broadcast.create_track("video", info).unwrap(); - let session = ScriptedSession::per_stream(vec![Vec::new()]); let log = session.log.clone(); let peer_setup = peer::PeerSetup::default(); @@ -7187,6 +7196,82 @@ mod serve_tests { wire.assert_streams_closed_before_publish_done(); } + /// A finished group's stream whose FIN is never acknowledged holds PUBLISH_DONE only until + /// the group goes stale: its successor's arrival plus the budget, even under a budget past + /// the cache window, which evicts both groups first. + #[moq_clock::test] + async fn publish_done_waits_on_an_unacknowledged_fin_until_the_group_is_stale() { + let version = Version::Draft21; + let session = ScriptedSession::per_stream(vec![Vec::new()]).with_unacked_uni_fins(1); + let h = serve_session( + version, + crate::track::Info::timed(), + crate::origin::Route::default(), + session, + ); + // Well past the cache window, which only dates idleness once a sweep passes. + let budget = crate::cache::DEFAULT_EXPIRY * 4; + let mut msg = subscribe(Filter::NextObject, None); + msg.max_age = Some(budget.as_millis() as u64); + + // `create_broadcast` registers the broadcast from a spawned task. + moq_clock::sleep(Duration::from_millis(1)).await; + let stream = Stream::open(&mut h.session.clone(), version).await.unwrap(); + let mut serve = std::pin::pin!(h.publisher.clone().run_subscribe_stream(stream, msg)); + // The subscription then parks at the live edge. + for _ in 0..200 { + assert!(futures::poll!(serve.as_mut()).is_pending()); + } + + // Frozen timestamps, so only the wall clock since group 1 arrived convicts group 0. + let append = || { + let mut group = h.track.append_group().unwrap(); + group.write_frame(timestamp(), b"x".as_slice()).unwrap(); + group.finish().unwrap(); + }; + let finished = || { + data_streams(&h.log) + .values() + .filter(|sent| sent.contains(&Sent::Finish)) + .count() + }; + let step = Duration::from_secs(1); + async fn wait(mut serve: std::pin::Pin<&mut F>, seconds: u64) { + for _ in 0..seconds { + moq_clock::sim::advance(Duration::from_secs(1)).await; + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "PUBLISH_DONE went out early" + ); + } + } + append(); + serve_until(serve.as_mut(), "group 0's FIN went out", || finished() == 1).await; + append(); + serve_until(serve.as_mut(), "group 1's stream finished", || finished() == 2).await; + + // Group 2 arrives later, so a verdict that forgot group 1 would wait for 2's deadline. + wait(serve.as_mut(), 10).await; + append(); + h.track.finish().unwrap(); + serve_until(serve.as_mut(), "group 2's stream finished", || finished() == 3).await; + + // Groups 0 and 1 go idle past the cache window long before the deadline. + wait(serve.as_mut(), budget.as_secs() - 12).await; + assert!(h.log.resets().is_empty()); + + moq_clock::sim::advance(step).await; + moq_clock::timeout(step, serve) + .await + .expect("the stale group releases PUBLISH_DONE") + .unwrap(); + assert_eq!(h.log.resets(), vec![crate::StreamError::Cancel.to_code()]); + let wire = Wire::read(&h.log, version); + let (status, _) = wire.publish_done.expect("PUBLISH_DONE was sent"); + assert_eq!(status, ietf::PublishDoneStatus::TrackEnded.code(version)); + wire.assert_streams_closed_before_publish_done(); + } + /// A track that ends with an Unsupported error (a relay refusing a copy whose metadata /// changed does) ends its subscription with INTERNAL_ERROR: UPDATE_FAILED is only for an /// update the publisher could not apply. diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 0a7644ad1c3..100e0d58040 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -911,6 +911,8 @@ pub struct ScriptedSession { incoming_bidis: Arc>>>, /// Set by [`Self::with_unacked_fin`]. unacked_fin: bool, + /// Set by [`Self::with_unacked_uni_fins`]; shared, so every clone counts down one budget. + unacked_unis: Arc, datagram_size: usize, /// Set by [`Self::with_groups`]. groups: bool, @@ -939,6 +941,7 @@ impl ScriptedSession { incoming_unis: Arc::new(Mutex::new(std::collections::VecDeque::new())), incoming_bidis: Arc::new(Mutex::new(std::collections::VecDeque::new())), unacked_fin: false, + unacked_unis: Default::default(), datagram_size: 0, groups: false, } @@ -956,6 +959,13 @@ impl ScriptedSession { self } + /// Never acknowledge the FIN of the next `count` unidirectional streams opened, like a + /// peer stalled on those alone. + pub fn with_unacked_uni_fins(self, count: usize) -> Self { + self.unacked_unis.store(count, Ordering::Relaxed); + self + } + /// Have the peer open one unidirectional stream per script, in order, each replaying /// its bytes and then going quiet (or reporting EOF, on an `eof` session). /// @@ -1104,7 +1114,12 @@ impl poll::Session for ScriptedSession { _cx: &mut Context<'_>, options: poll::SendOptions<'_, SinkGroup>, ) -> Poll> { - Poll::Ready(Ok(SinkSend::new(self.log.clone()).opened(options))) + let mut send = SinkSend::new(self.log.clone()); + send.unacked_fin = self + .unacked_unis + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| count.checked_sub(1)) + .is_ok(); + Poll::Ready(Ok(send.opened(options))) } fn send_group(&mut self, priority: i64) -> Option { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 62707ee8125..a40de14c922 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1284,13 +1284,44 @@ impl TrackState { /// so judging the newest group, the common case, costs no clock read. fn superseded(&self, sequence: u64, servable: &Servable, budget: Duration) -> bool { self.arrival(sequence, servable) - .is_some_and(|successor| self.aged(successor, budget)) + .is_some_and(|successor| self.aged(successor.arrived, budget)) } - /// Whether a full `budget` passed since `successor` arrived. - fn aged(&self, successor: &Slot, budget: Duration) -> bool { + /// Whether a full `budget` passed since a successor `arrived`. + fn aged(&self, arrived: crate::time::Instant, budget: Duration) -> bool { let now = self.cache.pool().now_wall(); - now.saturating_duration_since(successor.arrived) >= budget + now.saturating_duration_since(arrived) >= budget + } + + /// When a group was superseded, given `arrival`, its cached successor in `servable`: when + /// `arrival` arrived, or once the successor `superseded` recorded has left the cache, when + /// that one did, unless a group landed between them since. Without the record, an evicted + /// successor restarts the clock at whichever newer group the cache still holds, which a + /// budget past the cache window never reaches. + fn superseded_at( + &self, + arrival: Option<&Slot>, + servable: &Servable, + superseded: Option<&Superseded>, + ) -> Option { + let Some(superseded) = superseded else { + return arrival.map(|slot| slot.arrived); + }; + let mut last = superseded.0.lock().unwrap_or_else(PoisonError::into_inner); + // One still cached but aborted hands the clock on, as a re-read does. + if let Some(evicted) = *last + && !self.is_current(evicted.sequence, evicted.stamp) + && servable.contains(evicted.sequence) + && arrival.is_none_or(|slot| slot.group.sequence > evicted.sequence) + { + return Some(evicted.arrived); + } + *last = arrival.map(|slot| Successor { + sequence: slot.group.sequence, + stamp: slot.stamp, + arrived: slot.arrived, + }); + arrival.map(|slot| slot.arrived) } /// Whether the group at `sequence` has drifted further behind `edge` than `budget` @@ -1349,18 +1380,27 @@ impl TrackState { /// `budget`. Parking on every track change instead wakes each of N parked reads per append. /// /// The caller parks on a group landing above `sequence` first, in the way that fits - /// whether this track holds it. - fn poll_drifted(&self, sequence: u64, servable: &Servable, budget: Duration, waiter: &kio::Waiter) -> bool { + /// whether this track holds it. A reader holding the group passes its `superseded`, so + /// the wall clock outlives the successor's eviction; see [`Self::superseded_at`]. + fn poll_drifted( + &self, + sequence: u64, + servable: &Servable, + budget: Duration, + superseded: Option<&Superseded>, + waiter: &kio::Waiter, + ) -> bool { // Unbounded until a group lands above. Registered before judging, so a racing abort // is observed or wakes us, and armed for whichever successor this pass selected, so // re-selecting re-arms it. - let Some(arrival) = self.watch_abort(waiter, |state| state.arrival(sequence, servable)) else { + let arrival = self.watch_abort(waiter, |state| state.arrival(sequence, servable)); + let Some(arrived) = self.superseded_at(arrival, servable, superseded) else { return false; }; - if let Some(deadline) = arrival.arrived.checked_add(budget) { + if let Some(deadline) = arrived.checked_add(budget) { self.cache.watch_wall(deadline, waiter); } - if self.aged(arrival, budget) { + if self.aged(arrived, budget) { return true; } let reach = loop { @@ -4397,7 +4437,7 @@ impl Consumer { let state = state.read(); let budget = clamp_max_delay(budget, state.max_age_bound()); state.cache.wakes().watch_held(sequence, waiter); - match state.poll_drifted(sequence, &Servable::ALL, budget, waiter) { + match state.poll_drifted(sequence, &Servable::ALL, budget, None, waiter) { true => Poll::Ready(()), false => Poll::Pending, } @@ -5089,7 +5129,7 @@ impl kio::Task for Fetching { let state = state.read(); let budget = clamp_max_delay(self.max_delay.unwrap_or(Duration::MAX), state.max_age_bound()); state.cache.wakes().watch_landing(sequence, waiter); - if state.poll_drifted(sequence, &Servable::ALL, budget, waiter) { + if state.poll_drifted(sequence, &Servable::ALL, budget, None, waiter) { return Poll::Ready(Err(Error::Old)); } } @@ -5100,6 +5140,7 @@ impl kio::Task for Fetching { state: state.weak(), sequence, max_delay: self.max_delay, + superseded: Superseded::default(), }); let resolve = |mut group: group::Consumer| { group.start_at(frame_start); @@ -5272,6 +5313,7 @@ struct GroupExpiry { /// What the cursor can be served; see [`Cursor::published`]. servable: kio::Consumer, sequence: u64, + superseded: Superseded, } /// Keeps one fetched group tied to this caller's age budget without adding live demand. @@ -5279,6 +5321,21 @@ struct FetchExpiry { state: kio::ConsumerWeak, sequence: u64, max_delay: Option, + superseded: Superseded, +} + +/// The successor a handed-out group was last judged by, so the group stays judgeable once +/// the cache evicts both; see [`TrackState::superseded_at`]. Each reader keeps its own, so +/// the memory is bounded by the groups readers hold. +#[derive(Default)] +struct Superseded(Mutex>); + +/// A successor's slot incarnation, and when it arrived. +#[derive(Clone, Copy)] +struct Successor { + sequence: u64, + stamp: u32, + arrived: crate::time::Instant, } impl group::Expiry for FetchExpiry { @@ -5289,6 +5346,7 @@ impl group::Expiry for FetchExpiry { state: state.unwrap_or_else(|| self.state.clone()), sequence: self.sequence, max_delay: self.max_delay, + superseded: Superseded::default(), }) } @@ -5302,7 +5360,7 @@ impl group::Expiry for FetchExpiry { }; let budget = clamp_max_delay(budget, state.max_age_bound()); state.cache.wakes().watch_landing(self.sequence, waiter); - state.poll_drifted(self.sequence, &Servable::ALL, budget, waiter) && state.lookup.contains_key(&self.sequence) + state.poll_drifted(self.sequence, &Servable::ALL, budget, Some(&self.superseded), waiter) } } @@ -5315,6 +5373,7 @@ impl group::Expiry for GroupExpiry { subscription: self.subscription.clone(), servable: self.servable.clone(), sequence: self.sequence, + superseded: Superseded::default(), }) } @@ -5337,7 +5396,7 @@ impl group::Expiry for GroupExpiry { let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); state.cache.wakes().watch_landing(self.sequence, waiter); - state.poll_drifted(self.sequence, &servable, budget, waiter) && state.lookup.contains_key(&self.sequence) + state.poll_drifted(self.sequence, &servable, budget, Some(&self.superseded), waiter) } } @@ -5422,6 +5481,14 @@ impl Servable { /// No group. const NONE: Self = Self::Run { start: 0, end: Some(0) }; + /// Whether group `sequence` is among these. + fn contains(&self, sequence: u64) -> bool { + match self { + Self::Run { start, end } => sequence >= *start && end.is_none_or(|end| sequence < end), + Self::Runs(runs) => runs.contains(&sequence), + } + } + /// The groups in `runs`, disjoint and lowest first. Only several runs allocate: every /// subscriber resolves one, so the common single run stays off the heap. fn new(runs: impl Iterator>) -> Self { @@ -6011,7 +6078,7 @@ impl Cursor { if successor.is_none_or(|slot| slot.group.sequence <= oldest) { successor = state.successor(oldest, &edge.servable); } - let past = state.aged(first, budget) + let past = state.aged(first.arrived, budget) || successor.is_some_and(|slot| { state .live_above(oldest, &edge) @@ -6045,6 +6112,7 @@ impl Cursor { subscription: self.subscription.consume(), servable: self.published.consume(), sequence, + superseded: Superseded::default(), })) } @@ -9529,6 +9597,70 @@ mod test { } } + /// A handed-out group stays judgeable after the cache evicts it, and its successor too: + /// under a budget past the cache window, a stream still waiting on it (a FIN never + /// acknowledged) is convicted once its successor's arrival plus the budget passes, not + /// by whichever newer group the cache still holds. The newest group never is. A + /// subscribed group and a fetched one are judged alike. + #[test] + fn an_evicted_group_is_judged_by_its_evicted_successor() { + let mut producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); + let budget = cache::DEFAULT_EXPIRY * 2; + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget)); + let waiter = kio::Waiter::noop(); + let expired = |held: &mut [group::Consumer; 2]| { + held.each_mut() + .map(|group| group.poll_expired_while_pending(&waiter, true)) + }; + + // Frozen timestamps, so the wall clock alone convicts. + append_at(&mut producer, 0); + let fetch = group::Fetch::default().with_max_delay(budget); + let mut held = [ + subscriber.assert_group(), + producer + .consume() + .fetch_group(0, fetch) + .now_or_never() + .unwrap() + .unwrap(), + ]; + for group in &mut held { + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_none()); + } + assert_eq!(expired(&mut held), [false; 2], "the newest group"); + + elapse(&producer, Duration::from_secs(1)); + let mut last = append_at(&mut producer, 0); // the successor + assert_eq!(expired(&mut held), [false; 2]); + + // The track runs on, a group a second, until a second short of the deadline. + for _ in 0..budget.as_secs() - 1 { + elapse(&producer, Duration::from_secs(1)); + last = append_at(&mut producer, 0); + } + producer.broadcast.pool.sweep(); + let state = producer.state.read(); + assert!( + !state.lookup.contains_key(&0) && !state.lookup.contains_key(&1), + "both evicted" + ); + drop(state); + assert_eq!(expired(&mut held), [false; 2], "early"); + + elapse(&producer, Duration::from_secs(1)); + assert_eq!(expired(&mut held), [true; 2], "the successor arrived a budget ago"); + + // The newest group outlives any budget. + let mut newest = subscriber.assert_group(); + while newest.sequence < last { + newest = subscriber.assert_group(); + } + elapse(&producer, budget * 2); + assert!(!newest.poll_expired_while_pending(&waiter, true), "the newest group"); + } + /// Empty groups are handed out like any other and retired by the wall clock like an /// untimed one, whether they finished without a frame or are still open with none. A /// publisher flooding a run of them cannot pin the group before the run either. From 51544b4def0d21874738eec52f4cd5385d777fdc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 18:46:40 -0700 Subject: [PATCH 3/5] fix(net): hand the clock on from a recorded successor that failed A held group's recorded successor kept its arrival once it left the cache, even when it had failed first and the sweep reclaimed it before the reader looked again, so the verdict depended on sweep timing. Judge the record by whether the successor failed (aborted other than by cache policy) instead of by cache membership: a failed one hands the clock on, an evicted one keeps it. This also covers an ended track, whose cache expires groups in place. Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.test.ts | 46 +++++++++++++ js/net/src/track.ts | 39 +++++++---- rs/moq-net/src/model/group.rs | 23 +++++++ rs/moq-net/src/model/track.rs | 120 ++++++++++++++++++++++++++++++---- 4 files changed, 203 insertions(+), 25 deletions(-) diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 6e869f1ece2..2a83968678f 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -1230,6 +1230,52 @@ test("a parked read wakes at its wall deadline", async () => { } }); +// A held group's recorded successor that failed hands the wall clock on to the next live +// group, even when the write that brings that group in also evicts the failed one before the +// guard looks again, rather than convicting by the failed one's arrival. +test("a failed successor hands the wall clock on after the cache evicts it", async () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + try { + const budget = 2 * CACHE_WINDOW_MS; + const track = producer.subscribe({ maxDelay: Milli(budget) }); + const held = producer.appendGroup(); + held.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(0) }); + held.close(); + const group = track.tryRecvGroup(); + if (!group) throw new Error("missing group"); + expect(await group.readString()).toBe("x"); + + let stale = false; + const guarded = hooks + .guardGroup(group, () => new Promise(() => {})) + .catch((err: unknown) => { + stale = true; + return err; + }); + const failed = producer.appendGroup(); + await settle(); + + // Past the cache window, so the next write evicts the failed successor at once. + const replacement = 10_000 + CACHE_WINDOW_MS + 10_000; + await clock.advance(replacement); + failed.close(new Error("upstream failed")); + const live = producer.appendGroup(); + live.writeFrame({ payload: enc.encode("y"), timestamp: Timestamp.fromMillis(0) }); + live.close(); + + await clock.advance(10_000 + budget + 1); + expect(stale).toBe(false); + await clock.advance(replacement + budget - 1); + expect(stale).toBe(false); + await clock.advance(replacement + budget); + expect(await guarded).toMatchObject({ code: StreamCode.Old }); + } finally { + producer.close(); + clock.restore(); + } +}); + // Empty groups are handed out like any other and retired by the wall clock like an untimed // one, whether they finished without a frame or are still open with none. A publisher // flooding a run of them cannot pin the group before the run either. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index c61b9662a1e..abba8ad22be 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -75,6 +75,16 @@ const HOLES: ReadonlySet = new Set([ StreamCode.Evicted, ]); +// The errors this cache closes a group with when it evicts it unfinished: an aged-out group, +// not a failed one. Mirrors Rust's cache-policy aborts in `group::Handle::commit_abort`. +const EVICTED = new WeakSet(); + +// Whether `group` failed: closed with an error other than its eviction from this cache. +function failed(group: GroupConsumer): boolean { + const closed = group.closed.peek(); + return closed instanceof Error && !EVICTED.has(closed); +} + /** * A position within a track: a group's sequence and a frame's index inside it. * @@ -1130,7 +1140,11 @@ export class Producer { // the mirrors below would otherwise report a clean finish to a reader that had // drained it. The usual case, an already-closed group aging out, keeps its own // terminal state. - if (!entry.group.isClosed) entry.group.close(new TooFarBehind()); + if (!entry.group.isClosed) { + const evicted = new TooFarBehind(); + EVICTED.add(evicted); + entry.group.close(evicted); + } const mirrors = [...entry.mirrors.values()]; for (const mirror of mirrors) hooks.evictGroup(mirror); this.#unlink(entry); @@ -1936,21 +1950,22 @@ export class Subscriber { ); } - // The successor whose arrival starts the wall clock: the cached `arrival`, or once the one - // `held` recorded leaves the timeline, that one, unless a group landed between them since. - // Otherwise an evicted successor restarts the clock at whichever newer group the cache - // still holds, which a budget past the cache window never reaches. One still in the - // timeline but aborted hands the clock on, as a re-read does. + // The successor whose arrival starts the wall clock: the cached `arrival`, or the one `held` + // recorded, unless a group landed between them since. The record outlives the successor's + // eviction: otherwise an evicted successor restarts the clock at whichever newer group the + // cache still holds, which a budget past the cache window never reaches. A recorded + // successor that failed hands the clock on, as a re-read does, whether or not the cache + // has evicted it yet. #supersededBy(arrival: GroupConsumer | undefined, drift: Drift, held?: Superseded): GroupConsumer | undefined { if (!held) return arrival; - const evicted = held.successor; + const recorded = held.successor; if ( - evicted && - this.#state.timeline[timelineIndex(this.#state.timeline, evicted.sequence)] !== evicted && - (drift.end === undefined || evicted.sequence < drift.end) && - (!arrival || arrival.sequence > evicted.sequence) + recorded && + !failed(recorded) && + (drift.end === undefined || recorded.sequence < drift.end) && + (!arrival || arrival.sequence > recorded.sequence) ) { - return evicted; + return recorded; } held.successor = arrival; return arrival; diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 86606f313fa..dcbb7898125 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -966,6 +966,17 @@ pub(crate) struct Handle { alive: Arc, } +/// Whether a group failed, without holding it open or pinning its frames. +pub(super) struct Fate(Arc); + +impl Fate { + /// Whether the group was aborted other than by cache policy (`Old`, `Timeout`, + /// `Evicted`): it failed or was given up, rather than aged out of a cache. + pub(super) fn failed(&self) -> bool { + self.0.failed.load(Ordering::Acquire) + } +} + struct Alive { // Monotone mirror of `GroupState::abort` for track scans that already hold the // track lock. A stale false only hands out a group that is concurrently aborting; @@ -982,6 +993,9 @@ struct Alive { // policy (`Old`, `Timeout`, `Evicted`). Set before `aborted`, under the same rules, so // a track scan learns why without taking the group lock. cut: OnceLock, + // Set before `aborted` when the abort was not cache policy: the group failed or was given + // up rather than aged out of a cache. Read through [`Fate`] once the group is gone. + failed: AtomicBool, // Monotone mirror of `GroupState::started` for the first frame or the finish, set after // the group lock is released and before the track is told: a track scan reading a stale // false judges the group frameless until that wake re-judges it. @@ -1111,6 +1125,7 @@ impl Producer { aborted: AtomicBool::new(false), checked: AtomicBool::new(false), cut: OnceLock::new(), + failed: AtomicBool::new(false), started: AtomicBool::new(false), access, }); @@ -1872,6 +1887,9 @@ impl Handle { if cut { let _ = self.alive.cut.set(err.clone()); } + if !policy { + self.alive.failed.store(true, Ordering::Release); + } guard.abort = Some(err); self.alive.aborted.store(true, Ordering::Release); guard.release(salvage); @@ -1898,6 +1916,11 @@ impl Handle { self.alive.cut.get() } + /// This group's [`Fate`], readable after the group and its frames are gone. + pub(super) fn fate(&self) -> Fate { + Fate(self.alive.clone()) + } + /// Whether the group was finished: it holds every frame it will ever have. pub(crate) fn is_finished(&self) -> bool { self.state.read().fin.is_some() diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8c431be5084..d393f516f2c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1345,10 +1345,11 @@ impl TrackState { } /// When a group was superseded, given `arrival`, its cached successor in `servable`: when - /// `arrival` arrived, or once the successor `superseded` recorded has left the cache, when - /// that one did, unless a group landed between them since. Without the record, an evicted + /// `arrival` arrived, or when the successor `superseded` recorded did, unless a group landed + /// between them since. The record outlives the successor's eviction: otherwise an evicted /// successor restarts the clock at whichever newer group the cache still holds, which a - /// budget past the cache window never reaches. + /// budget past the cache window never reaches. A recorded successor that failed hands the + /// clock on, as a re-read does, whether or not the cache has reclaimed it yet. fn superseded_at( &self, arrival: Option<&Slot>, @@ -1359,18 +1360,17 @@ impl TrackState { return arrival.map(|slot| slot.arrived); }; let mut last = superseded.0.lock().unwrap_or_else(PoisonError::into_inner); - // One still cached but aborted hands the clock on, as a re-read does. - if let Some(evicted) = *last - && !self.is_current(evicted.sequence, evicted.stamp) - && servable.contains(evicted.sequence) - && arrival.is_none_or(|slot| slot.group.sequence > evicted.sequence) + if let Some(recorded) = &*last + && !recorded.fate.failed() + && servable.contains(recorded.sequence) + && arrival.is_none_or(|slot| slot.group.sequence > recorded.sequence) { - return Some(evicted.arrived); + return Some(recorded.arrived); } *last = arrival.map(|slot| Successor { sequence: slot.group.sequence, - stamp: slot.stamp, arrived: slot.arrived, + fate: slot.group.fate(), }); arrival.map(|slot| slot.arrived) } @@ -5341,12 +5341,11 @@ struct FetchExpiry { #[derive(Default)] struct Superseded(Mutex>); -/// A successor's slot incarnation, and when it arrived. -#[derive(Clone, Copy)] +/// A successor, when it arrived, and whether it has since failed. struct Successor { sequence: u64, - stamp: u32, arrived: crate::time::Instant, + fate: group::Fate, } impl group::Expiry for FetchExpiry { @@ -9803,6 +9802,101 @@ mod test { assert!(!newest.poll_expired_while_pending(&waiter, true), "the newest group"); } + /// A recorded successor that failed hands the clock on to the next live group even once + /// the cache reclaimed it before the reader looked again, rather than convicting by the + /// failed one's arrival. A subscribed group and a fetched one are judged alike. + #[test] + fn a_reclaimed_failed_successor_hands_the_clock_on() { + let mut producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); + let budget = cache::DEFAULT_EXPIRY * 2; + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget)); + let waiter = kio::Waiter::noop(); + let expired = |held: &mut [group::Consumer; 2]| { + held.each_mut() + .map(|group| group.poll_expired_while_pending(&waiter, true)) + }; + + append_at(&mut producer, 0); + let fetch = group::Fetch::default().with_max_delay(budget); + let mut held = [ + subscriber.assert_group(), + producer + .consume() + .fetch_group(0, fetch) + .now_or_never() + .unwrap() + .unwrap(), + ]; + for group in &mut held { + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); + assert!(group.read_frame().now_or_never().unwrap().unwrap().is_none()); + } + + let failed = producer.append_group().unwrap(); + assert_eq!(expired(&mut held), [false; 2], "the successor is recorded"); + failed.abort(Error::Cancel).unwrap(); + let replacement = Duration::from_secs(10); + elapse(&producer, replacement); + append_at(&mut producer, 0); + producer.broadcast.pool.sweep(); + assert!(!producer.state.read().lookup.contains_key(&1), "reclaimed"); + + elapse(&producer, budget - replacement); + assert_eq!( + expired(&mut held), + [false; 2], + "the failed successor arrived a budget ago" + ); + + elapse(&producer, replacement); + assert_eq!(expired(&mut held), [true; 2], "the next live one arrived a budget ago"); + } + + /// On an ended track the cache expires idle groups in place rather than reclaiming them, + /// so the reads skip them like any aborted group. A recorded successor that aged out that + /// way still convicts the group before it once it arrived a budget ago. + #[test] + fn an_ended_track_judges_by_its_expired_successor() { + let pool = cache::Pool::new(cache::Config::default().with_expiry(cache::DEFAULT_EXPIRY)); + let mut producer = track_producer_pooled("test", pool.clone()); + let budget = cache::DEFAULT_EXPIRY * 2; + let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(budget)); + let waiter = kio::Waiter::noop(); + + append_at(&mut producer, 0); + let mut held = subscriber.assert_group(); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_none()); + + pool.step(Duration::from_secs(1)); + append_at(&mut producer, 0); // the successor + assert!( + !held.poll_expired_while_pending(&waiter, true), + "the successor is recorded" + ); + let state = producer.state.downgrade(); + producer.finish().unwrap(); + drop(producer); + + // The first sweep dates the groups idle, the second expires them. + let idle = cache::DEFAULT_EXPIRY + Duration::from_secs(1); + pool.sweep(); + pool.step(idle); + pool.sweep(); + assert_eq!( + state.read(|state| state.lookup.get(&1).is_some_and(Slot::is_aborted)), + Some(true), + "expired in place" + ); + assert!(!held.poll_expired_while_pending(&waiter, true), "early"); + + pool.step(budget - idle); + assert!( + held.poll_expired_while_pending(&waiter, true), + "the successor arrived a budget ago" + ); + } + /// Empty groups are handed out like any other and retired by the wall clock like an /// untimed one, whether they finished without a frame or are still open with none. A /// publisher flooding a run of them cannot pin the group before the run either. From 96eb6f640a4fed0b1971909445e013c3eb2e57eb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 19:06:20 -0700 Subject: [PATCH 4/5] fix(net): judge a JS successor's failure by its copy, not the reader A recorded successor evicted with unread frames gets a reader-side TooFarBehind verdict, which the consumer's `closed` reports, so the JS record took an ordinary eviction for a failure and handed the clock on. Read the copy's own close error through a new internal hook instead. Co-Authored-By: Claude Opus 5.5 --- js/net/src/group.ts | 4 ++++ js/net/src/internal.ts | 5 +++++ js/net/src/track.test.ts | 43 ++++++++++++++++++++++++++++++++++++++++ js/net/src/track.ts | 8 +++++--- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/js/net/src/group.ts b/js/net/src/group.ts index cae93c2f3f1..9237c102040 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -568,6 +568,10 @@ export class Consumer { }; hooks.groupArrived = (group) => group.#state.arrived; hooks.groupStarted = (group) => group.#state.total.peek() > 0 || group.#state.closed.peek() !== undefined; + hooks.groupAbort = (group) => { + const closed = group.#state.closed.peek(); + return closed instanceof Error ? closed : undefined; + }; hooks.arriveGroup = (group, at) => { group.#state.arrived = at; }; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 4b9a02bd2f2..7584b2a2052 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -535,6 +535,8 @@ export const hooks: { groupArrived: (group: GroupConsumer) => Instant; /** Whether a group counts as content: its first frame landed, or it ended. */ groupStarted: (group: GroupConsumer) => boolean; + /** Return the error a group's copy was closed with, ignoring this reader's own verdicts. */ + groupAbort: (group: GroupConsumer) => Error | undefined; /** Record when a track's mirror of a group joined that track at this hop. */ arriveGroup: (group: GroupConsumer, at: Instant) => void; /** Keep applying a subscription's drift policy after it hands a group out. */ @@ -713,6 +715,9 @@ export const hooks: { groupStarted: () => { throw new Error("group.ts not loaded"); }, + groupAbort: () => { + throw new Error("group.ts not loaded"); + }, expireGroup: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 2a83968678f..eb863a2a829 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -1230,6 +1230,48 @@ test("a parked read wakes at its wall deadline", async () => { } }); +// Evicting a recorded successor nobody read leaves its mirror a reader-side gap, which is not +// a failure: the held group is still judged by that successor's arrival, not the next one's. +test("an evicted successor with unread frames still starts the wall clock", async () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + try { + const budget = 2 * CACHE_WINDOW_MS; + const track = producer.subscribe({ maxDelay: Milli(budget) }); + const write = () => { + const group = producer.appendGroup(); + group.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(0) }); + group.close(); + }; + write(); + const group = track.tryRecvGroup(); + if (!group) throw new Error("missing group"); + expect(await group.readString()).toBe("x"); + + let stale = false; + const guarded = hooks + .guardGroup(group, () => new Promise(() => {})) + .catch((err: unknown) => { + stale = true; + return err; + }); + write(); // the successor, left unread + await settle(); + // A later live edge, so the successor ages out of the cache. + await clock.advance(20_000); + write(); + + await clock.advance(10_000 + budget - 1); + expect(stale).toBe(false); + await clock.advance(10_000 + budget); + expect(stale).toBe(true); + expect(await guarded).toMatchObject({ code: StreamCode.Old }); + } finally { + producer.close(); + clock.restore(); + } +}); + // A held group's recorded successor that failed hands the wall clock on to the next live // group, even when the write that brings that group in also evicts the failed one before the // guard looks again, rather than convicting by the failed one's arrival. @@ -1269,6 +1311,7 @@ test("a failed successor hands the wall clock on after the cache evicts it", asy await clock.advance(replacement + budget - 1); expect(stale).toBe(false); await clock.advance(replacement + budget); + expect(stale).toBe(true); expect(await guarded).toMatchObject({ code: StreamCode.Old }); } finally { producer.close(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index abba8ad22be..1fe932ccbd2 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -79,10 +79,12 @@ const HOLES: ReadonlySet = new Set([ // not a failed one. Mirrors Rust's cache-policy aborts in `group::Handle::commit_abort`. const EVICTED = new WeakSet(); -// Whether `group` failed: closed with an error other than its eviction from this cache. +// Whether `group`'s copy failed: closed with an error other than its eviction from this cache. +// Read from the copy, since a reader's own verdict (an eviction's gap, a stale group) says +// nothing about whether the group arrived. function failed(group: GroupConsumer): boolean { - const closed = group.closed.peek(); - return closed instanceof Error && !EVICTED.has(closed); + const abort = hooks.groupAbort(group); + return abort !== undefined && !EVICTED.has(abort); } /** From 85a69ac60133d5afdd916e2898d3c6bb4263eccc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 19:18:21 -0700 Subject: [PATCH 5/5] fix(net): record a JS successor's metadata, not its consumer A recorded successor consumer carries its own expiry closure, whose record holds the next successor, so one long-held group chained every later handed-out consumer behind it. Record the sequence, the arrival, and a fate getter over the copy's state instead, as Rust records `group::Fate`, and re-record only when the successor changes. Co-Authored-By: Claude Opus 5.5 --- js/net/src/group.ts | 9 ++++++--- js/net/src/internal.ts | 6 +++--- js/net/src/track.ts | 43 ++++++++++++++++++++++++++---------------- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 9237c102040..f3ea3604676 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -568,9 +568,12 @@ export class Consumer { }; hooks.groupArrived = (group) => group.#state.arrived; hooks.groupStarted = (group) => group.#state.total.peek() > 0 || group.#state.closed.peek() !== undefined; - hooks.groupAbort = (group) => { - const closed = group.#state.closed.peek(); - return closed instanceof Error ? closed : undefined; + hooks.groupFate = (group) => { + const state = group.#state; + return () => { + const closed = state.closed.peek(); + return closed instanceof Error ? closed : undefined; + }; }; hooks.arriveGroup = (group, at) => { group.#state.arrived = at; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 7584b2a2052..7bb99206e22 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -535,8 +535,8 @@ export const hooks: { groupArrived: (group: GroupConsumer) => Instant; /** Whether a group counts as content: its first frame landed, or it ended. */ groupStarted: (group: GroupConsumer) => boolean; - /** Return the error a group's copy was closed with, ignoring this reader's own verdicts. */ - groupAbort: (group: GroupConsumer) => Error | undefined; + /** Watch the error a group's copy closes with, ignoring readers' verdicts, without holding the consumer. */ + groupFate: (group: GroupConsumer) => () => Error | undefined; /** Record when a track's mirror of a group joined that track at this hop. */ arriveGroup: (group: GroupConsumer, at: Instant) => void; /** Keep applying a subscription's drift policy after it hands a group out. */ @@ -715,7 +715,7 @@ export const hooks: { groupStarted: () => { throw new Error("group.ts not loaded"); }, - groupAbort: () => { + groupFate: () => { throw new Error("group.ts not loaded"); }, expireGroup: () => { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 1fe932ccbd2..e354bf32087 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -79,11 +79,11 @@ const HOLES: ReadonlySet = new Set([ // not a failed one. Mirrors Rust's cache-policy aborts in `group::Handle::commit_abort`. const EVICTED = new WeakSet(); -// Whether `group`'s copy failed: closed with an error other than its eviction from this cache. -// Read from the copy, since a reader's own verdict (an eviction's gap, a stale group) says -// nothing about whether the group arrived. -function failed(group: GroupConsumer): boolean { - const abort = hooks.groupAbort(group); +// Whether `successor`'s copy failed: closed with an error other than its eviction from this +// cache. Read from the copy, since a reader's own verdict (an eviction's gap, a stale group) +// says nothing about whether the group arrived. +function failed(successor: Successor): boolean { + const abort = successor.fate(); return abort !== undefined && !EVICTED.has(abort); } @@ -1793,7 +1793,12 @@ type Drift = { // The successor a handed-out group was last judged by, so the group stays judgeable once the // cache evicts both. Each reader keeps its own, so the memory is bounded by the groups held. -type Superseded = { successor?: GroupConsumer }; +type Superseded = { successor?: Successor }; + +// A successor, when it arrived, and its copy's fate (see `hooks.groupFate`). Not the consumer: +// a handed-out successor's own record would retain the next one, chaining every later group to +// the oldest held. +type Successor = { sequence: number; arrived: Instant; fate: () => Error | undefined }; /** * The read side of a live track subscription, mirroring the Rust `Subscriber`. @@ -1927,17 +1932,17 @@ export class Subscriber { // budget ago, so a peer whose timestamps stop advancing, an untimed track, or an empty // successor that never presents cannot pin a superseded group. The successor is re-read, // not latched, like the reach, except by a reader holding the group: it passes `held`, so - // the wall clock outlives the successor's eviction (see {@link #supersededBy}). The newest + // the wall clock outlives the successor's eviction (see {@link #supersededAt}). The newest // group has no successor, so neither clock ever convicts it. `wake` arms a timer for the // wall deadline, for a held read that waits on it. // // Judged by its successor alone, so the group need not be in this track: evicted, or held // from another copy. #staleAt(sequence: number, drift: Drift, wake = false, held?: Superseded): boolean { - const arrival = this.#supersededBy(this.#arrival(sequence, drift.end), drift, held); - if (!arrival) return false; + const arrived = this.#supersededAt(this.#arrival(sequence, drift.end), drift, held); + if (arrived === undefined) return false; // Plain arithmetic: an unenforced budget is infinite, which no duration brand holds. - const deadline = hooks.groupArrived(arrival) + drift.budget * 1_000_000; + const deadline = arrived + drift.budget * 1_000_000; if (now() >= deadline) return true; if (wake) this.#wakeAt(deadline); @@ -1952,14 +1957,15 @@ export class Subscriber { ); } - // The successor whose arrival starts the wall clock: the cached `arrival`, or the one `held` + // When the wall clock starts: the cached `arrival`'s arrival, or that of the one `held` // recorded, unless a group landed between them since. The record outlives the successor's // eviction: otherwise an evicted successor restarts the clock at whichever newer group the // cache still holds, which a budget past the cache window never reaches. A recorded // successor that failed hands the clock on, as a re-read does, whether or not the cache // has evicted it yet. - #supersededBy(arrival: GroupConsumer | undefined, drift: Drift, held?: Superseded): GroupConsumer | undefined { - if (!held) return arrival; + #supersededAt(arrival: GroupConsumer | undefined, drift: Drift, held?: Superseded): Instant | undefined { + const arrived = arrival && hooks.groupArrived(arrival); + if (!held) return arrived; const recorded = held.successor; if ( recorded && @@ -1967,10 +1973,15 @@ export class Subscriber { (drift.end === undefined || recorded.sequence < drift.end) && (!arrival || arrival.sequence > recorded.sequence) ) { - return recorded; + return recorded.arrived; + } + if (!arrival || arrived === undefined) { + held.successor = undefined; + } else if (recorded?.sequence !== arrival.sequence || recorded.arrived !== arrived || failed(recorded)) { + // Re-recorded only on a change, so judging the same successor again allocates nothing. + held.successor = { sequence: arrival.sequence, arrived, fate: hooks.groupFate(arrival) }; } - held.successor = arrival; - return arrival; + return arrived; } // Bump {@link #wall} at `at`, unless a wakeup is already armed sooner. Each held read