From 8d83767b92c6f299225a4d5be5605953ed387e72 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 15:56:43 -0800 Subject: [PATCH 1/4] fix(hang): ordered reads skip a hole or a blocking group only once Rust's age bound proves it #8071 made the JS ordered read drop groups that arrive below its cursor, as Rust does. JS still moved the cursor by two weaker rules than Rust's, and the backfill it removed had been rescuing what they skipped: - A hole was proven lost once the head's newest frame passed where presentation left off by the max delay. That proves only the oldest missing content stale. Groups go out newest first, so a flush's newest group lands just ahead of the rest, the cursor jumped past them, and they were dropped on arrival. Now, as Rust's `aged`, the hole is lost once the first group past it starts a whole max delay behind the newest content. - Once warmed, the age skip dropped a blocking group by its own first frame and jumped to the next buffered group, past groups still in flight. Now, as Rust's blocking-group skip, it waits until the successor starts a whole budget behind the newest content. The pause guard #8123 added to the old hole rule is subsumed, so `#paused` goes. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 5 +- js/hang/src/container/consumer.holes.test.ts | 52 +++++++++++++- js/hang/src/container/consumer.skips.test.ts | 49 +++++++++++-- js/hang/src/container/consumer.test.ts | 18 +++-- js/hang/src/container/consumer.ts | 69 ++++++++----------- 5 files changed, 138 insertions(+), 55 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 5b783113db4..35711904f8e 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -461,11 +461,12 @@ test("Consumer raises a malformed group an age skip passes after the track finis expect(await within(consumer.next())).toMatchObject({ kind: "frame", group: 0n }); await flush(); - // Group 1 holds a valid frame then a malformed one, and group 2 starts past the max delay, so the - // next read's decode skips group 1 for its age. + // Group 1 holds a valid frame then a malformed one, and group 2 starts a whole max delay behind + // its own newest frame, so the next read's decode skips group 1 for its age. write(malformed, 10_000); malformed.writeFrame({ payload: new Uint8Array([0x80]), timestamp: Time.Timestamp.now() }); write(later, 2_000_000); + write(later, 2_500_000); await expect(within(consumer.next())).rejects.toThrow("buffer too short"); } finally { consumer.close(); diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index 1562e8080da..8dbbbeaeb0e 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -286,12 +286,13 @@ test.each([true, false])("the head after a declared end is waited for (arrives: write(2n, [60_000_000]); await flush(); expect(results.slice(3)).toEqual([frame(2n, 60_000_000), frame(3n, 60_060_000)]); - // Resumed, a later hole past the budget is proven again without a wait. - write(5n, [61_000_000]); + // Resumed, a later hole whose head starts a budget behind the newest is proven without a wait. + write(5n, [61_000_000], [61_300_000]); await flush(); expect(results.slice(5)).toEqual([ { kind: "gap", timestamp: Micro(60_060_000), duration: Micro(940_000) }, frame(5n, 61_000_000), + frame(5n, 61_300_000), ]); } else { await clock.advance(Nano.fromMilli(Milli(1))); @@ -308,3 +309,50 @@ test.each([true, false])("the head after a declared end is waited for (arrives: installed.close(); } }); + +// Groups go out newest first, so a flush's newest group lands just ahead of the rest. Where +// presentation left off is a budget behind it, but the missing groups end where it starts, so they +// may still play: skipping the hole would drop them on arrival, below the cursor. +test("a newest-first flush past a hole delivers the groups landing behind its newest", async () => { + const track = new Track.Producer("audio"); + const consumer = new Consumer(track.subscribe({ delay: { max: Milli(30_000) } }), { + format: new Format("audio"), + delay: { max: Milli(80) }, + }); + const results: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + results.push(read); + } + })(); + // One 20ms frame per group, as a voice publisher writes them, all sent at once. + const sent = Time.Timestamp.now(); + const write = (sequence: number) => { + const group = track.createGroup(BigInt(sequence)); + const timestamp = Micro(sequence * 20_000); + group.writeFrame({ payload: encodeFrame(new Uint8Array([1]), timestamp), timestamp: sent }); + group.finish(); + }; + try { + for (const sequence of [0, 1, 2]) write(sequence); + await flush(); + write(7); + await flush(); + for (const sequence of [3, 4, 5, 6]) write(sequence); + await flush(); + expect(results).toEqual( + [0, 1, 2, 3, 4, 5, 6, 7].map((sequence) => ({ + kind: "frame", + group: BigInt(sequence), + frame: expect.objectContaining({ timestamp: Micro(sequence * 20_000) }), + })), + ); + } finally { + consumer.close(); + track.finish(); + track.close(); + await reading; + } +}); diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 4a772d54e13..0788ed763e3 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -5,6 +5,8 @@ import { Signal } from "@moq/signals"; import { Consumer } from "./consumer.ts"; import { Format } from "./legacy.ts"; +const flush = () => new Promise((resolve) => setImmediate(resolve)); + for (const count of [1, 16, 256]) { test(`Consumer summarizes ${count} skipped groups once per catch-up`, async () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); @@ -165,8 +167,8 @@ test("Consumer measures each skip's span over every group left, not just the new ] as const) write(sequence, timestamp); await buffered(12); - // Reading the first frame warms the reader, so the age skip runs on the max delay alone. - expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0n }); + // Before the first read, the span alone ages a group out: once reading began, its successor's + // start bounds it instead, which the span always reaches past. delay.set({ max: Milli(5) }); write(4n, 13_000); await buffered(13); @@ -175,8 +177,7 @@ test("Consumer measures each skip's span over every group left, not just the new { start: Milli(12), end: Milli(12) }, { start: Milli(13), end: Milli(13) }, ]); - // The skipped groups held media past the first frame, so their span reads as a gap first. - expect(await consumer.next()).toMatchObject({ kind: "gap", timestamp: 0, duration: 12_000 }); + // Joining mid-track is not a gap: nothing was read before it. expect(await consumer.next()).toMatchObject({ kind: "frame", group: 3n }); } finally { consumer.close(); @@ -185,3 +186,43 @@ test("Consumer measures each skip's span over every group left, not just the new warn.mockRestore(); } }); + +// An open group blocks until its successor starts a whole budget behind the newest content, not until +// its own first frame does: the skip moves the cursor past the groups still in flight between, which +// a group arriving below the cursor never delivers. +test("a blocking group is kept until its successor starts a budget behind the newest", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("tone"); + const consumer = new Consumer(track.subscribe({ delay: { max: Milli(30_000) } }), { + format: new Format("data"), + delay: { max: Milli(150) }, + }); + const sent = Time.Timestamp.now(); + const write = (sequence: bigint, timestamp: number) => { + const group = track.createGroup(sequence); + group.writeFrame({ payload: Varint.encode(timestamp), timestamp: sent }); + return group; + }; + try { + const blocking = write(0n, 0); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0n }); + // Group 3's stream wins the race, 160ms past group 0's first frame. + write(3n, 160_000).finish(); + await flush(); + write(1n, 100_000).finish(); + write(2n, 130_000).finish(); + blocking.finish(); + const reads = [await consumer.next(), await consumer.next(), await consumer.next()]; + expect(reads).toMatchObject([ + { kind: "frame", group: 1n }, + { kind: "frame", group: 2n }, + { kind: "frame", group: 3n }, + ]); + expect(warn).not.toHaveBeenCalled(); + } finally { + consumer.close(); + track.finish(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 022cb80e9aa..30bf5525266 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1355,8 +1355,9 @@ test("Consumer reads a gap when retiring an open group", async () => { await flush(); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Micro); + // Group 1 starts a whole budget behind the newest content, so the open group is retired. writeGroupWithLegacyFrames(track, 1n, [100_000 as Micro]); - writeGroupWithLegacyFrames(track, 2n, [200_000 as Micro]); + writeGroupWithLegacyFrames(track, 2n, [250_000 as Micro]); await flush(); // Legacy carries no duration, so the gap starts at the last frame read. @@ -1733,9 +1734,9 @@ test("Consumer delivers a contiguous group after one that completed out of order consumer.close(); }); -// A resubscribe can replay a stale group, lose the group after it, then carry on live. Once the live -// head reaches past where presentation left off by more than the budget, whatever is missing would -// arrive too old to play, so the head is the group to play next and must not be dropped as "slow". +// A resubscribe can replay a stale group, lose the group after it, then carry on live. Once the head +// past the hole starts a whole budget behind the newest content, whatever is missing would arrive too +// old to play, so the head is the group to play next and must not be dropped as "slow". test("Consumer plays the head once a waited-out gap exceeds the budget (Legacy)", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { @@ -1752,10 +1753,15 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (Legacy)" // The reader is parked on the missing 1001, as a decoder is while it waits for the next group. const pending = consumer.next(); - // B (seq 2000) starts at 90_000 ticks (1_000_000µs), a second past A's end: the missing group - // could only ever arrive far beyond the 100ms budget, so B plays from its first frame. + // B (seq 2000) starts at 90_000 ticks (1_000_000µs), a second past A's end. The missing groups end + // where B starts, so B alone proves nothing: one may hold media just before it. const b = track.createGroup(2000n); b.writeFrame(timedFrame(0x02, 90_000, 1)); + expect(await poll(pending)).toBe("pending"); + + // C (seq 2001) carries on live 200ms past B's start, so anything missing below B is too old to play. + const c = track.createGroup(2001n); + c.writeFrame(timedFrame(0x03, 108_000, 2)); // The missing group's span was never delivered, so a gap from A's end to B comes first. const result = await poll(pending); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index ff97e6cb370..e499591f54e 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -199,9 +199,6 @@ export class Consumer { // Group of the last media frame next() returned, so an endpoint after it reads as that frame's // duration rather than a declared stop. #mediaGroup?: bigint; - // The group of the last declared end read, until a frame from a later group resumes the timeline. - // Not at the next frame: the stopped encoder's final packets may follow the end in its group. - #paused?: bigint; // Media went missing since the last item read, so the next one is preceded by a gap. #lost = false; // The furthest presentation point read so far: max(timestamp + duration), endpoints included. @@ -569,22 +566,25 @@ export class Consumer { const age = max - min; if (age <= threshold) break; - // Until the reader warms, a group may cover the start's reach though it began before it. - if (this.#warming()) { - const nextStart = this.#groups[1].start; - if (nextStart === undefined) break; - if (this.#edge === undefined) { - // Before the first item, a backlog that landed whole starts within the budget, so only - // the group covering the live edge less the reach is kept: one whose successor starts - // past that point. Rust's start skip, in `Consumer::poll_event_once`, measures the same. - // `max` is the live edge of the groups left, so this stays O(1) per skip. + const successor = this.#groups[1].start; + if (this.#edge === undefined) { + // Until the reader warms, a group may cover the start's reach though it began before it. + // Before the first item, a backlog that landed whole starts within the budget, so only + // the group covering the live edge less the reach is kept: one whose successor starts + // past that point. Rust's start skip, in `Consumer::poll_event_once`, measures the same. + // `max` is the live edge of the groups left, so this stays O(1) per skip. + if (this.#warming()) { + if (successor === undefined) break; const target = this.#startTarget(this.#reach(this.#delay.peek()), max); - if (target !== undefined && nextStart > target) break; - } else if (max - nextStart < threshold) { - // Once reading began, it is stale only once its successor starts a whole budget behind - // the newest content. - break; + if (target !== undefined && successor > target) break; } + } else if (successor === undefined || max - successor < threshold) { + // Once reading began, the head is stale only once its successor starts a whole budget + // behind the newest content: everything the head and any missing group after it could + // still present ends there. Its own first frame is no bound, since the skip moves the + // cursor past every missing group in between, which a group arriving below it then never + // delivers. Mirrors Rust's skip of a blocking group. + break; } this.#groups.shift(); @@ -826,11 +826,13 @@ export class Consumer { // only applies at startup. // Otherwise promote #active when the first buffered group continues the timeline we left off at, // when a completed empty group can be walked (empty groups mean nothing), or when the hole - // is proven: the head already reaches past where presentation left off by more than the - // max delay, so anything still missing in between would arrive too old to play. Proving it - // here matters: #checkMaxDelay would instead drop the head, the very group to play next. - // A declared end proves nothing: the jump past it is the pause, and the media that resumes - // it may still be in flight behind the group after it. + // is proven: everything a missing group could still present ends where the first group past + // it begins, so once that start falls a whole max delay behind the newest content, all of it + // would arrive too old to play. Mirrors Rust's `aged`. Where presentation left off proves only + // the oldest of it stale, and groups go out newest first, so a flush's newest group lands just + // ahead of the groups behind it, which would then arrive below the cursor and never play. + // Proving it here matters: #checkMaxDelay would instead drop the head, the very group to play + // next. // After track termination no missing group can arrive, so drain across any remaining gap. // A missing sequence also expires on elapsed monotonic time, so timestamp-free markers // behind a hole can end a paused track without more arrivals. It is late since a group @@ -845,29 +847,16 @@ export class Consumer { const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp); const empty = head.empty && head.consumer.done; const maxDelay = maxMicros(this.#delay.peek()); - const skipHole = - head.frames.length > 0 && - (maxDelay === 0 || - (this.#paused === undefined && - this.#presentedEnd !== undefined && - head.latest !== undefined && - head.latest - this.#presentedEnd > maxDelay)); + const bound = this.#groups.find((group) => group.start !== undefined)?.start; + const edge = this.#liveEdge(); + const aged = bound !== undefined && edge !== undefined && edge - bound >= maxDelay; const elapsed = Instant.durationSince(now(), this.#hole) >= maxDelay * 1000; // The ranges ask for nothing in between, so nothing there is owed. const next = this.#track.nextInRange(this.#active); const excluded = next === undefined || next >= head.consumer.sequence; // The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it. const full = this.#full(); - if ( - startup || - empty || - contiguous || - skipHole || - elapsed || - excluded || - full || - ended !== undefined - ) { + if (startup || empty || contiguous || aged || elapsed || excluded || full || ended !== undefined) { if (!contiguous && !empty) { this.#lost = true; if (!startup) { @@ -1082,8 +1071,6 @@ export class Consumer { const edge = this.#edge; const lost = this.#lost; this.#lost = false; - if (read.kind === "end") this.#paused = read.group; - else if (this.#paused !== undefined && read.group > this.#paused) this.#paused = undefined; this.#advance(read.kind === "frame" ? Micro(resume + (read.frame.duration ?? Micro.zero)) : resume); if (!lost || edge === undefined) return read; From ce7584ffaad31788b9bbe4c1a3ac905b345da6ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 16:07:16 -0800 Subject: [PATCH 2/4] fix(hang): a blocking group's age bound looks past unstamped successors As Rust's `poll_next_group`, the post-read age skip measures from the first stamped group past the head, so an empty or still-unstamped successor no longer blocks it. The newest-first flush test holds a manual clock so only timestamps can prove the hole. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.holes.test.ts | 4 +++ js/hang/src/container/consumer.skips.test.ts | 30 ++++++++++++++++++++ js/hang/src/container/consumer.ts | 18 ++++++------ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index 8dbbbeaeb0e..9fc0ef36e7b 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -314,6 +314,9 @@ test.each([true, false])("the head after a declared end is waited for (arrives: // presentation left off is a budget behind it, but the missing groups end where it starts, so they // may still play: skipping the hole would drop them on arrival, below the cursor. test("a newest-first flush past a hole delivers the groups landing behind its newest", async () => { + // Time stands still while the flush lands, so only the timestamps can prove the hole. + const clock = new Manual(); + const installed = clock.install(); const track = new Track.Producer("audio"); const consumer = new Consumer(track.subscribe({ delay: { max: Milli(30_000) } }), { format: new Format("audio"), @@ -354,5 +357,6 @@ test("a newest-first flush past a hole delivers the groups landing behind its ne track.finish(); track.close(); await reading; + installed.close(); } }); diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 0788ed763e3..b5f61390881 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -226,3 +226,33 @@ test("a blocking group is kept until its successor starts a budget behind the ne warn.mockRestore(); } }); + +// A group with no frame says nothing about the timeline, so the age bound is the first stamped group +// past the head, and the unstamped one between ages out with it. +test("a blocking group ages out past an unstamped successor", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("tone"); + const consumer = new Consumer(track.subscribe({ delay: { max: Milli(30_000) } }), { + format: new Format("data"), + delay: { max: Milli(150) }, + }); + const sent = Time.Timestamp.now(); + try { + const blocking = track.createGroup(0n); + blocking.writeFrame({ payload: Varint.encode(0), timestamp: sent }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0n }); + track.createGroup(1n).finish(); + const later = track.createGroup(2n); + for (const timestamp of [1_000_000, 2_000_000]) + later.writeFrame({ payload: Varint.encode(timestamp), timestamp: sent }); + await flush(); + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 2n }); + blocking.finish(); + } finally { + consumer.close(); + track.finish(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index e499591f54e..212104c8ae5 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -566,7 +566,6 @@ export class Consumer { const age = max - min; if (age <= threshold) break; - const successor = this.#groups[1].start; if (this.#edge === undefined) { // Until the reader warms, a group may cover the start's reach though it began before it. // Before the first item, a backlog that landed whole starts within the budget, so only @@ -574,17 +573,20 @@ export class Consumer { // past that point. Rust's start skip, in `Consumer::poll_event_once`, measures the same. // `max` is the live edge of the groups left, so this stays O(1) per skip. if (this.#warming()) { + const successor = this.#groups[1].start; if (successor === undefined) break; const target = this.#startTarget(this.#reach(this.#delay.peek()), max); if (target !== undefined && successor > target) break; } - } else if (successor === undefined || max - successor < threshold) { - // Once reading began, the head is stale only once its successor starts a whole budget - // behind the newest content: everything the head and any missing group after it could - // still present ends there. Its own first frame is no bound, since the skip moves the - // cursor past every missing group in between, which a group arriving below it then never - // delivers. Mirrors Rust's skip of a blocking group. - break; + } else { + // Once reading began, the head is stale only once the first stamped group past it starts a + // whole budget behind the newest content: everything the head and any group between could + // still present ends there. Its own first frame is no bound, since the skip moves the cursor + // past every missing group in between, which a group arriving below it then never delivers. + // An unstamped group between ages out on a later pass, as the head. Mirrors Rust's skip of + // a blocking group, which measures from `poll_next_group`. + const successor = this.#groups.find((group, i) => i > 0 && group.start !== undefined)?.start; + if (successor === undefined || max - successor < threshold) break; } this.#groups.shift(); From e0ce3c7a91b2c077d0b35d48a4b7907e528ecbc6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 17:01:37 -0800 Subject: [PATCH 3/4] test(audio-quality): re-measure relay-bbb and relay-mic-rtt40; plan the replay merge gate Maintainer decision 2026-11-07: re-measure the two replay rows the ordered-read fix leaves above their ceilings, by the file's exact-measurement rule. relay-bbb: player underruns 12.1 -> 16.1/min, stalled 0.017 -> 0.025, silence 0.016 -> 0.02. relay-mic-rtt40: player underruns 7.9 -> 8.2/min, and discarded 229.3 -> 70.9 ms/min and target p95 780 -> 320 ms tighten to what they measure. relay-mic measures its ceilings exactly and is unchanged. The note records why neither loses audio the previous player heard. Plans the [XS] m0 quest that puts the replays in `just ci`. Co-Authored-By: Claude Opus 5.5 --- quest/m0/ci/README.md | 1 + quest/m0/ci/audio-replay-gate.md | 24 ++++++++++++++++++++++++ test/audio-quality/budgets.json | 14 +++++++------- 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 quest/m0/ci/audio-replay-gate.md diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 673b75b4f36..1e347f9f4c3 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -180,4 +180,5 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Archive the @moq packages on JSR](/quest/m0/ci/jsr-retire.md) - once `release` stops publishing there, the maintainer archives every JSR package, ending its OIDC link - [Verify the R2 Rust cache](/quest/m0/ci/mbx-r2-verify.md) - confirm the hosted Nightly jobs reuse the previous Nightly, then measure restore time and expiry - [Release tags push to nix.moq.dev, and Cachix stops taking writes](/quest/m0/ci/cachix-retire.md) - backport `release-nix.yml` and its Just module to `release`, drop `cachix.yml`, prove a tag substitutes from nix.moq.dev, then revoke the Cachix token (the cache stays for old tags) +- [The audio replay rows gate every merge](/quest/m0/ci/audio-replay-gate.md) - `just ci` runs the deterministic replays under `--enforce`, so a PR that moves a row fails the gate - [Measure the Linux cache effect of native media exclusions](/quest/m0/ci/hakari-followups.md) - scheduled Linux jobs measure cache reuse after unrelated tests stop compiling PipeWire and libclang diff --git a/quest/m0/ci/audio-replay-gate.md b/quest/m0/ci/audio-replay-gate.md new file mode 100644 index 00000000000..de17e2e358b --- /dev/null +++ b/quest/m0/ci/audio-replay-gate.md @@ -0,0 +1,24 @@ +# [XS] The audio replay rows gate every merge + +## Goal + +`just test audio-quality --replays --enforce` runs in `just ci`, so a PR that +moves a replay row fails the merge gate unless it updates `budgets.json` in the +same PR. + +## Plan + +Decided 2026-11-07 (maintainer, /quest-complete on +[moq-dev/moq#8218](https://github.com/moq-dev/moq/pull/8218)): ✅ the replays +join `just ci`. [moq-dev/moq#8071](https://github.com/moq-dev/moq/pull/8071) +broke two replay rows on `main` with no PR check failing. The replays run only +from `.github/workflows/audio-quality.yml`, which nightly and release PRs call, +so the breakage surfaced only once a later PR ran them by hand. + +The replays are deterministic and take seconds: no browser, relay, or shaper, +only Bun and the player's `js/` packages. Register them as a module in +`sh/dispatch.sh`'s impact map. Select it on whatever the replay plays through +(`js/hang/`, `js/watch/`, `js/net/`, `js/signals/`, `js/clock/`) and on the +harness itself (`test/audio-quality/`), so the queue's `just ci` runs it too. +Then decide whether `audio-quality.yml`'s release-PR job still earns its +runner, and update the audio-quality README's account of where the replays run. diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index e419b11f5e2..4b9be542079 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The AAC rows predate the TS importer publishing each PES as one group. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 discarded ms/min, 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch must bring to zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only, exactly as measured on 2026-11-03; it counts the run-dry re-stalls the tap files as stalled time. The local-aac replay's stalled ceiling rises from 0.008 to 0.017 and silence from 0.015 to 0.020 because replay no longer stretches each frame to nearby timestamp gaps; the production player is unchanged. All other changed replay ceilings tighten to the exact measurement. That replay calibration leaves browser and native ceilings unchanged. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On that host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows.", + "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The AAC rows predate the TS importer publishing each PES as one group. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 discarded ms/min, 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch must bring to zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only, exactly as measured on 2026-11-03; it counts the run-dry re-stalls the tap files as stalled time. The local-aac replay's stalled ceiling rises from 0.008 to 0.017 and silence from 0.015 to 0.020 because replay no longer stretches each frame to nearby timestamp gaps; the production player is unchanged. All other changed replay ceilings tighten to the exact measurement. That replay calibration leaves browser and native ceilings unchanged. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb and relay-mic-rtt40 were re-measured to exactly what they play: relay-bbb's player underruns rise from 12.1 to 16.1/min, stalled from 0.017 to 0.025, and silence from 0.016 to 0.02; relay-mic-rtt40's player underruns rise from 7.9 to 8.2/min while its discarded audio tightens from 229.3 to 70.9 ms/min and its target p95 from 780 to 320 ms. Neither loses audio the previous player heard. With the frames it received but never played removed from the trace, stale backlog that had raised its startup target to 780 ms, the previous player measured relay-mic-rtt40's same 8.2. relay-bbb's extra episode is the ring trimming one AAC frame (23 ms) deeper after the recorder's 10 s backlog burst, which the previous player's out-of-order backfill happened to leave higher; the current read plays every frame there that the previous one did, and more. relay-mic measures its ceilings exactly, so it is unchanged, although the previous player had played it at 4.0 underruns/min for the same reason as relay-bbb. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On that host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows.", "rows": [ { "runtime": "chromium", @@ -346,13 +346,13 @@ "codec": "aac", "rate": 44100, "profile": "relay-bbb", - "player_underruns_per_min": 12.1, + "player_underruns_per_min": 16.1, "underruns_per_min": 0, "underrun_ms_per_min": 0, "skip_aheads_per_min": 2, "discarded_ms_per_min": 87.8, - "stalled_share": 0.017, - "silence_share": 0.016, + "stalled_share": 0.025, + "silence_share": 0.02, "target_ms_p95": 223.2 }, { @@ -374,14 +374,14 @@ "codec": "opus", "rate": 48000, "profile": "relay-mic-rtt40", - "player_underruns_per_min": 7.9, + "player_underruns_per_min": 8.2, "underruns_per_min": 0.3, "underrun_ms_per_min": 0.5, "skip_aheads_per_min": 0.3, - "discarded_ms_per_min": 229.3, + "discarded_ms_per_min": 70.9, "stalled_share": 0.003, "silence_share": 0.003, - "target_ms_p95": 780 + "target_ms_p95": 320 } ] } From 18512bcee63fb82b10b0cbd88acc9466b07a5bb7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 21:54:15 -0800 Subject: [PATCH 4/4] fix(hang): keep the skip's stamped successor across a run of unstamped groups Restore the clock in the holes tests even when the reader rejects, and plan an exact replay comparison for the replay gate. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.holes.test.ts | 49 ++++++++++++++------ js/hang/src/container/consumer.ts | 7 ++- quest/m0/ci/audio-replay-gate.md | 6 +++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index 9fc0ef36e7b..fd68adc5d6c 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -46,8 +46,11 @@ test.each([0, 100])("a sequence hole before a paused data marker expires at %sms consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); @@ -97,8 +100,11 @@ test("a filled hole cancels its timer and the next hole gets a fresh budget", as consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); @@ -138,8 +144,11 @@ test("the holes a stalled source left behind share one budget", async () => { consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); @@ -182,8 +191,11 @@ test("a hole is late from the earliest arrival past it", async () => { consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); @@ -234,8 +246,11 @@ test("a hole in a drained backlog is late from the earliest arrival past it, not consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); @@ -305,8 +320,11 @@ test.each([true, false])("the head after a declared end is waited for (arrives: consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); @@ -356,7 +374,10 @@ test("a newest-first flush past a hole delivers the groups landing behind its ne consumer.close(); track.finish(); track.close(); - await reading; - installed.close(); + try { + await reading; + } finally { + installed.close(); + } } }); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 212104c8ae5..971653e28cf 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -534,6 +534,9 @@ export class Consumer { const start = this.#groups[0]?.consumer.sequence; // The span left after each further skip, computed once the first skip shows more may follow. let spans: Spans | undefined; + // The index of the first stamped group past the head, kept across skips so a run of unstamped + // groups ahead of it stays linear: each skip shifts it down one, and only a skip onto it rescans. + let stamped = 0; // Keep skipping the oldest group while the buffered span exceeds the max delay. // This also handles gaps in group sequence numbers: if #active points to a missing @@ -585,11 +588,13 @@ export class Consumer { // past every missing group in between, which a group arriving below it then never delivers. // An unstamped group between ages out on a later pass, as the head. Mirrors Rust's skip of // a blocking group, which measures from `poll_next_group`. - const successor = this.#groups.find((group, i) => i > 0 && group.start !== undefined)?.start; + if (stamped < 1) stamped = this.#groups.findIndex((group, i) => i > 0 && group.start !== undefined); + const successor = stamped < 1 ? undefined : this.#groups[stamped].start; if (successor === undefined || max - successor < threshold) break; } this.#groups.shift(); + stamped--; this.#active = this.#groups[0]?.consumer.sequence; const next = this.#groups[0]; diff --git a/quest/m0/ci/audio-replay-gate.md b/quest/m0/ci/audio-replay-gate.md index de17e2e358b..b147d28a51c 100644 --- a/quest/m0/ci/audio-replay-gate.md +++ b/quest/m0/ci/audio-replay-gate.md @@ -22,3 +22,9 @@ only Bun and the player's `js/` packages. Register them as a module in harness itself (`test/audio-quality/`), so the queue's `just ci` runs it too. Then decide whether `audio-quality.yml`'s release-PR job still earns its runner, and update the audio-quality README's account of where the replays run. + +`grade.ts --enforce` fails only a value over its ceiling, so a replay row that +improves still passes and leaves headroom for a later regression, against +`budgets.json`'s rule that a replay ceiling is exactly what it measures. Make +`--enforce` fail a replay row whose value differs from its budget in either +direction, keeping the ceiling comparison for the browser and native rows.