diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 1f068aa70e5..ec795e4172c 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -604,6 +604,8 @@ export const hooks: { * Whether a subscriber's track ended and it holds nothing left to read: no group in `[start, end)` that `delivered` lacks. */ spent: (subscriber: Subscriber, start: bigint, end: bigint | undefined, delivered: Delivered) => boolean; + /** Whether a subscriber's last read waits on a fill that still owes it a group. Mirrors the Rust `awaits_fill`. */ + awaitsFill: (subscriber: Subscriber) => boolean; /** * Mark a wire publisher's reader to park past the end of a bounded window rather than end the * read there, since its peer reads the end of a stream as the track's. Mirrors the Rust @@ -893,6 +895,9 @@ export const hooks: { spent: () => { throw new Error("track.ts not loaded"); }, + awaitsFill: () => { + throw new Error("track.ts not loaded"); + }, parked: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index cf06c9f77fc..07e179baf35 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -1049,6 +1049,19 @@ test("regressions", async () => { ], finish: false, }, + // A switch's backfill of a group the finished route never had lets the track end. + { + ops: [ + { kind: "up", instance: 1, slot: 1 }, + { kind: "up", instance: 1, slot: 2 }, + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, false) }, + { kind: "write", instance: 1, slot: 1, group: 11, frames: 1 }, + { kind: "write", instance: 1, slot: 2, group: 1, frames: 1 }, + { kind: "down", instance: 1, slot: 2 }, + { kind: "end", instance: 1, slot: 1 }, + ], + finish: false, + }, ]; const failures: string[] = []; for (const [i, c] of regressions.entries()) { diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 1d2eeaf38a4..c046d32d615 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -267,6 +267,60 @@ test("a group below the replacement's resume point is backfilled", async () => { } }); +// The switch's backfill waits on a fill the finished new route's handler still owes, rather +// than counting as drained while the fill is out. +test("a backfill waits on a fill the finished route still owes", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + const requests = tb.producer.requests(); + const fills: track.Fill[] = []; + void (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) fills.push(fill); + })(); + tb.write(6n); + tb.producer.finish(); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(3_600_000) } }); + try { + await clock.advance(Nano.zero); + ta.write(5n); + expect(drain(sub)).toEqual([5n]); + front.serve(handles[1]); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([6n]); + await clock.advance(Nano.zero); + expect(fills.length).toBeGreaterThan(0); + expect(drain(sub)).toEqual([]); + expect(sub.closed.peek()).toBeUndefined(); + // Only group 2 is in the route's history. + for (const fill of fills) { + for (let sequence = fill.wanted(); sequence !== undefined; sequence = fill.wanted()) { + if (sequence !== 2n) { + fill.dropGroups(sequence, sequence + 1n); + continue; + } + const group = fill.accept(sequence); + group.writeFrame({ payload: enc.encode("2.0") }); + group.finish(); + } + fill.close(); + } + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([2n]); + } finally { + requests.close(); + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + test("a routed source start follows the serving copy and its subscriber floor", async () => { const a = new BroadcastProducer(); const b = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index a7d3cf908a8..969b137f282 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -794,6 +794,7 @@ class Reader implements ResumeSource { copy.done = copy.sub.closed.peek() ?? null; if (copy.done === null && hooks.subscriberEnd(copy.sub)) this.#settled = true; } else if (recv.kind === "error") copy.done = recv.error; + else if (this.#drained(copy, delivered)) copy.done = null; break; } } @@ -881,6 +882,18 @@ class Reader implements ResumeSource { return undefined; } + // Whether `copy`, bounded where it was replaced, ran out under that bound: its track ended, no + // fill still owes it a group, and it holds nothing there unread. Its subscriber would park on a + // group past the bound instead, as one past a local cap waits for it to rise, but the reader + // moves this bound itself by asking again. Mirrors the Rust arrival cursor ending on a complete + // track. Judged right after a read, which records whether it waits on a fill. + #drained(copy: Copy, delivered: Delivered): boolean { + const end = copyEnd(copy); + if (end === undefined || hooks.awaitsFill(copy.sub)) return false; + const start = max(this.#shell.cursor.peek().start, copy.floor ?? 0n); + return hooks.spent(copy.sub, start, end, delivered); + } + // Whether `copy` may still feed groups: its channel has not run out, and what it holds is // within the reader's budget, which ends a full budget after a newer group superseded it. #feeds(copy: Copy): boolean { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index cdd2f1945ee..ce398334f08 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2890,6 +2890,7 @@ export class Subscriber { hooks.spent = (subscriber, start, end, delivered) => (subscriber.#state.closed.peek() !== undefined || subscriber.#complete()) && !hooks.hasUnreadGroup(subscriber, start, end, delivered); + hooks.awaitsFill = (subscriber) => subscriber.#fillWait; hooks.staleAt = (subscriber, sequence) => { const held: Superseded = {}; return { diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 2fd31b9f0dd..902e5cd538a 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -45,4 +45,3 @@ Public API: none. Wire: none. - [Nightly's Miri job stays inside its time budget](/quest/m0/nightly/miri.md) - both Miri passes go green, with ARM timings inside the 60-minute job and 15-minute-per-test budgets - [Nightly's macOS platform job passes](/quest/m0/nightly/macos.md) - `moq-audio`'s tests compile on macOS again and the websocket version test's timeout is fixed if it recurs - [The OBS stop-drain test reads no gap](/quest/m0/nightly/obs-stop-drain.md) - OBS and OBS TSan read every delivered frame and the track's end after `Stop()`, with no gap -- [The JS resume model finds no missing groups](/quest/m0/nightly/resume-model.md) - the JS resume model property test hands every reader each group it is owed diff --git a/quest/m0/nightly/resume-model.md b/quest/m0/nightly/resume-model.md deleted file mode 100644 index ec1febe1986..00000000000 --- a/quest/m0/nightly/resume-model.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] The JS resume model finds no missing groups - -## Goal - -Nightly's `Tests (test resume-model)` passes: the JS resume model property -test (`js/net/src/resume.model.test.ts`) hands every reader each group its -spec owes on every generated case. Find the root cause and fix it in -`@moq/net`, or in the model if the model is wrong. - -## Plan - -Added 2026-11-05 (maintainer, /quest-audit), from scheduled Nightly -[run 42050683248](https://github.com/moq-dev/moq/actions/runs/42050683248/job/127325640208). -The Rust half (`model::resume_model::readers_match_the_logical_track`) -passed; the JS half failed: - -```text -error: Property failed after 2700 tests -error: reader 0: missing groups 2,3,4,5,6,7,8,9,10; handed out 1,11,0, dropped , floor undefined, spec {"ordered":false,"budget":"hour","drops":false} -``` - -Replay it with `MOQ_RESUME_SEED=764723972 just test resume-model`. The shrunk -counterexample: instance 1 brings up slots 1 and 2, reader 0 joins unordered -with an hour budget, slot 1 writes group 11 and slot 2 group 1, slot 2 goes -down, slot 1 ends, and reader 0 joins again. - -The two earlier scheduled runs failed this job with a different symptom, -`the track ended cleanly though no route declared an end` -([41852517544](https://github.com/moq-dev/moq/actions/runs/41852517544/job/126652296304), -[41656613559](https://github.com/moq-dev/moq/actions/runs/41656613559/job/125988510774)). -Check whether one cause explains both before closing this. - -Public API: none expected. Wire: none. diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 324bcc0702d..10287fb8891 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -333,6 +333,7 @@ impl Buffer { } /// How many frames a single fill can hold. + #[cfg(feature = "async")] pub const fn capacity(&self) -> usize { N } @@ -343,11 +344,13 @@ impl Buffer { } /// Whether the buffer holds no frames. + #[cfg(feature = "async")] pub fn is_empty(&self) -> bool { self.0.is_empty() } /// Whether the buffer is at capacity, so [`Self::push`] would refuse. + #[cfg(feature = "async")] pub(crate) fn is_full(&self) -> bool { self.0.is_full() } @@ -380,6 +383,7 @@ impl Buffer { } /// Drop the current batch, leaving the buffer empty. + #[cfg(feature = "async")] pub fn clear(&mut self) { self.0.clear(); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index e8f17f1db6b..68203c6de0a 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -4246,6 +4246,7 @@ impl Consumer { /// This is a short read: it returns as soon as anything is ready rather than /// waiting for `out` to fill. A zero count means the group ended when the buffer /// has non-zero capacity. + #[cfg(feature = "async")] #[cfg(any(test, feature = "fuzz"))] pub(crate) fn poll_read_frames( &mut self, diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index cac76dd195c..e4a280230bc 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -1224,6 +1224,32 @@ fn regressions() { ], finish: true, }, + // A switch's backfill of a group the finished route never had lets the track end. + Case { + ops: vec![ + Op::Up { instance: 1, slot: 1 }, + Op::Up { instance: 1, slot: 2 }, + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Write { + instance: 1, + slot: 1, + group: Some(11), + frames: 1, + }, + Op::Write { + instance: 1, + slot: 2, + group: Some(1), + frames: 1, + }, + Op::Down { instance: 1, slot: 2 }, + Op::End { instance: 1, slot: 1 }, + ], + finish: false, + }, ]; for case in cases { if let Err(err) = run(&case) { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 1e869a15af6..61afe7bd198 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3878,7 +3878,7 @@ impl Producer { } /// Evict group `sequence` from the cache, as expiry does. - #[cfg(test)] + #[cfg(all(test, feature = "async"))] pub(crate) fn evict(&self, sequence: u64) { self.modify().unwrap().evict_group(sequence, Error::Evicted); } @@ -23347,6 +23347,7 @@ mod test { } /// The frame a read takes right now, as its number in [`numbered`]. + #[cfg(feature = "async")] fn read_numbered(reader: &mut group::Consumer) -> Option { let frame = reader.read_frame().now_or_never()?; let frame = frame.expect("read failed").expect("the group ended"); @@ -23463,6 +23464,7 @@ mod test { } /// The size of `payload` as a frame, for [`group::Producer::poll_ready`]. + #[cfg(feature = "async")] fn frame_info(payload: &bytes::Bytes) -> frame::Info { frame::Info { size: payload.len() as u64, @@ -23472,6 +23474,7 @@ mod test { /// A live group of `frames` numbered frames past its tail, read along by a live reader, /// on a track with a handler. + #[cfg(feature = "async")] fn shed_live_group(frames: u32) -> (Producer, Requests, group::Producer, group::Consumer) { let producer = track_producer("test", Info::default()); let requests = producer.requests(); @@ -23485,6 +23488,7 @@ mod test { } /// The next fill asked of `requests`, if any. + #[cfg(feature = "async")] fn asked(requests: &Requests) -> Option { match requests.poll_requested_fill(&kio::Waiter::noop()) { Poll::Ready(fill) => Some(fill.unwrap()), diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 64ee72f116c..d0a0ef544b2 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -28,8 +28,10 @@ use std::{ use crate::Path; +#[cfg(feature = "async")] +pub(crate) use proof::EXPORTER_LABEL; pub use proof::RouteKey; -pub(crate) use proof::{EXPORTER_LABEL, Proof, Transcript}; +pub(crate) use proof::{Proof, Transcript}; pub(crate) use view::{View, overlaps, within}; /// An opaque routing identity, declared by an origin or derived by [`Scope`]. diff --git a/rs/moq-net/src/route/proof.rs b/rs/moq-net/src/route/proof.rs index 7bc6fec6772..92cbd101a8d 100644 --- a/rs/moq-net/src/route/proof.rs +++ b/rs/moq-net/src/route/proof.rs @@ -10,6 +10,7 @@ use rind::signature::{Ed25519KeyPair, PublicKey, verify::ED25519}; use zeroize::Zeroizing; /// The label every transport binding exports the session secret under. +#[cfg(feature = "async")] pub(crate) const EXPORTER_LABEL: &[u8] = b"EXPORTER-moq-lite-route-key"; /// Bytes of exported secret a proof signs. diff --git a/test/resume-model.sh b/test/resume-model.sh index 1cdb99d055b..3523f9b753a 100755 --- a/test/resume-model.sh +++ b/test/resume-model.sh @@ -7,6 +7,6 @@ MOQ_RESUME_SEED=${MOQ_RESUME_SEED:-$(python3 -c 'import secrets; print(secrets.r MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} export MOQ_RESUME_SEED MOQ_RESUME_CASES echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" -just rs test -p moq-net --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' +just rs test -p moq-net --features async --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' bun install --frozen-lockfile bun test ./js/net/src/resume.model.test.ts