diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 3df5573d331..b80f4e4a880 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -49,12 +49,18 @@ group is observed when the stall clears. A missing group, one that has delivered no frame, is waited for only the playout delay, the subscription's `delay.min`, since past that it would play late: a reordered backlog's holes waited out to the whole budget would delay the frames behind them, which the estimator would measure as -jitter. The browser player waits less still, only until the group past the hole -trails its clock by the stretch threshold, past which the ring would skip the -hole's audio anyway: a viewer resuming behind a replayed backlog, which arrives -newest first, would otherwise hold the live audio for the backlog's oldest group, -while a late burst that fills a hole a moment after the group past it lands -still plays it. A caller that estimates keeps the budget past the target, which is why +jitter. The browser player leaves the hole to the transport: it reads its audio +through an ordered track reader whose own `delay.max` is what the ring holds of +the delay plus the stretch threshold, past which the ring would skip the hole's +audio anyway, and plays each gap that reader reports as the hole. The reader +gives up on a missing group by the same two clocks as any stale group: once the +newest frame is that far past where the group after it starts, or that group +arrived that long ago. A late burst that fills a hole a moment after the group +past it lands still plays it, and a viewer resuming behind a replayed backlog +does not hold the live audio for the backlog's oldest group. That reader's +budget is its own: a second subscription to the same track, read in arrival +order, observes every frame its own wider budget receives, so the late groups +the reader gives up on are still measured. A caller that estimates keeps the budget past the target, which is why `moq play` waits on a stalled audio group for the delay plus the estimate's 2 s ceiling. The observation is also only as honest as the caller is prompt: `read()` is timed when it resolves, so a player drains it @@ -69,8 +75,7 @@ the target: the estimate creeps up one bucket at a time while the frames it should have measured go unplayed. Both implementations subscribe with the delay plus the estimate's own 2 s ceiling, whatever the player's `stretch` setting: `moq play` through its age budget, and the browser through the audio -subscription alone, since its container consumer observes each frame before -applying the local budget. So any lateness the estimator can measure is +subscription it observes arrivals on, beside the narrower ordered one it plays. So any lateness the estimator can measure is observed at once. A buffered browser player keeps its subscription at least the delay plus its lookahead. Browser video keeps the shared budget, because a longer one fetches whole stale diff --git a/js/hang/api/container.api.md b/js/hang/api/container.api.md index 6438d36ea00..88108b5dabc 100644 --- a/js/hang/api/container.api.md +++ b/js/hang/api/container.api.md @@ -26,6 +26,7 @@ export class Consumer { export interface ConsumerProps { delay?: Signals.GetterInit; format: Format; + ordered?: Net.Track.Ordered; playhead?: Playhead; predecessor?: Consumer; warmup?: Net.Clock.Duration; diff --git a/js/hang/src/container/consumer.ordered.test.ts b/js/hang/src/container/consumer.ordered.test.ts new file mode 100644 index 00000000000..e5dabb08fa6 --- /dev/null +++ b/js/hang/src/container/consumer.ordered.test.ts @@ -0,0 +1,142 @@ +// A consumer reading an ordered reader: the reader decides every hole, and the arrival subscription +// only feeds the estimate. +import { expect, test } from "bun:test"; +import { Duration } from "@moq/clock"; +import { Manual } from "@moq/clock/test"; +import { Time, Track } from "@moq/net"; +import { Consumer, type Read } from "./consumer.ts"; +import { encodeFrame, Format } from "./legacy.ts"; + +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +// An audio track whose missing groups an unserved fill owes, as a session's feed would, read by a +// consumer whose ordered reader waits `wait` for a hole and whose arrival subscription reaches 2 s. +function setup(wait: number) { + const clock = new Manual(); + const installed = clock.install(); + const track = new Track.Producer("audio"); + const requests = track.requests(); + const ordered = track.subscribe({ delay: { min: Duration.ZERO, max: Duration.millis(wait) } }).ordered(); + const consumer = new Consumer(track.subscribe({ delay: { max: Duration.secs(2) } }), { + format: new Format("audio"), + delay: { min: Duration.ZERO, max: Duration.secs(2) }, + ordered, + }); + const reads: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + reads.push(read); + } + })(); + // One finished group holding a 20 ms frame at `ms`. + const write = (sequence: bigint, ms: number) => { + const group = track.createGroup(sequence); + group.writeFrame({ + payload: encodeFrame(new Uint8Array([1]), Duration.millis(ms)), + timestamp: Time.Timestamp.fromMillis(ms), + }); + group.finish(); + }; + return { + clock, + track, + consumer, + reads, + write, + // What was read, as `frame 20` or `gap 20+20`, in ms. + read: () => + reads.map((read) => + read.kind === "gap" + ? `gap ${read.timestamp.asMillis()}+${read.duration.asMillis()}` + : read.kind === "frame" + ? `frame ${read.frame.timestamp.asMillis()}` + : `end ${read.end.asMillis()}`, + ), + async close() { + consumer.close(); + requests.close(); + track.finish(); + track.close(); + try { + await reading; + } finally { + installed.close(); + } + }, + }; +} + +test("a hole is waited for as long as the ordered reader waits, then read as a gap", async () => { + const t = setup(100); + try { + t.write(0n, 0); + t.write(2n, 40); + await flush(); + expect(t.read()).toEqual(["frame 0"]); + // The consumer keeps no wait of its own: the reader's budget alone holds the group past the hole. + await t.clock.advance(Duration.millis(99)); + expect(t.read()).toEqual(["frame 0"]); + await t.clock.advance(Duration.millis(1)); + await flush(); + expect(t.read()).toEqual(["frame 0", "gap 0+40", "frame 40"]); + } finally { + await t.close(); + } +}); + +test("a gap the reader reports before an empty group still reads as a gap before the next frame", async () => { + const t = setup(100); + try { + t.write(0n, 0); + // Group 2 holds nothing. The hole at 1 ages from group 2's arrival, while group 2 ages from + // group 3's later one, so the reader hands out the empty group 2 with the gap on it. + t.track.createGroup(2n).finish(); + await t.clock.advance(Duration.millis(90)); + t.write(3n, 60); + await t.clock.advance(Duration.millis(10)); + await flush(); + expect(t.read()).toEqual(["frame 0", "gap 0+60", "frame 60"]); + } finally { + await t.close(); + } +}); + +test("a group that lands within the reader's wait fills its hole", async () => { + const t = setup(100); + try { + t.write(0n, 0); + t.write(2n, 40); + await t.clock.advance(Duration.millis(50)); + t.write(1n, 20); + await flush(); + expect(t.read()).toEqual(["frame 0", "frame 20", "frame 40"]); + } finally { + await t.close(); + } +}); + +test("the estimate observes late groups the ordered reader gave up on", async () => { + // The spread after 5 s of groups 20 ms apart, every tenth landing `late` ms past its turn. + const spread = async (late: number) => { + const t = setup(50); + try { + const arrivals = Array.from({ length: 250 }, (_, i) => ({ i, at: i * 20 + (i % 10 === 5 ? late : 0) })); + arrivals.sort((a, b) => a.at - b.at); + let at = 0; + for (const arrival of arrivals) { + await t.clock.advance(Duration.millis(arrival.at - at)); + at = arrival.at; + t.write(BigInt(arrival.i), arrival.i * 20); + } + await flush(); + // The reader passed each late group long before it landed, so it plays as a gap. + if (late > 0) expect(t.read()).not.toContain("frame 100"); + return t.consumer.spread.peek(); + } finally { + await t.close(); + } + }; + expect((await spread(300)).gt(await spread(0))).toBe(true); +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index ea361469e86..3da434e5de1 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -74,6 +74,15 @@ export interface ConsumerProps { * switch neither lowers nor raises {@link Consumer.spread} on its own. */ predecessor?: Consumer; + /** + * Read the media in key order from this reader of the same track instead of `track`'s arrival + * order, so it decides every hole. It must be a second subscription, never `track`'s own, which + * keeps reading in arrival order to feed the estimate: a missing group is a hole once the reader reports it as a gap, + * and never waited for here. Its `delay.max` is then how long a hole is waited for, so size it to + * what the player can still play late. `track` only feeds the arrival estimate, read frame by frame + * as it lands, so give it a `delay.max` reaching as far as lateness can move the estimate. + */ + ordered?: Moq.Track.Ordered; /** * The playout clock this reader feeds, such as the player's `Sync` (default: none). A missing * group whose successor starts at or before the oldest media the clock still plays could only @@ -129,6 +138,7 @@ interface Group { // and the groups past it, kept on insert so a hole reads it without scanning the backlog. late: Instant; frames: ReceivedFrame[]; // decode order + gap?: boolean; // An ordered reader gave up on groups just below this one. costs: number[]; // What each of frames holds against the max buffer empty: boolean; // no wire frame was published; empty groups mean nothing media: boolean; // a decodable (non-marker) frame was buffered @@ -197,6 +207,10 @@ function maxMicros(delay: Moq.Track.Delay): number { */ export class Consumer { #track: Moq.Track.Subscriber; + // The reader that decides every hole, when one does; `#track` then only feeds the estimate. + #ordered?: Moq.Track.Ordered; + // The ordered reader gave up on groups since the last one it returned. + #gap = false; #format: Format; #delay: Getter; #playhead?: Playhead; @@ -287,6 +301,7 @@ export class Consumer { /** Start consuming the given track, decoding frames with `props.format`. */ constructor(track: Moq.Track.Subscriber, props: ConsumerProps) { this.#track = track; + this.#ordered = props.ordered; this.#format = props.format; this.#delay = getter(props.delay ?? {}); this.#playhead = props.playhead; @@ -301,14 +316,21 @@ export class Consumer { // later raise the largest past it. The start says only once the info or a first group shows the // subscription established. Off the delivery path: a range capped below the largest never // reports one, which reports no backlog rather than stalling delivery. - this.#signals.spawn(async () => { - await race([this.#track.info().catch(() => undefined), this.#tuned]); - await this.#track.start(); - this.#joined = (await this.#track.largest())?.group; - }); - this.#signals.spawn(this.#run.bind(this)); + const ordered = this.#ordered; + if (ordered) { + this.#signals.spawn(this.#runOrdered.bind(this, ordered)); + this.#signals.spawn(this.#observe.bind(this)); + } else { + this.#signals.spawn(async () => { + await race([this.#track.info().catch(() => undefined), this.#tuned]); + await this.#track.start(); + this.#joined = (await this.#track.largest())?.group; + }); + this.#signals.spawn(this.#run.bind(this)); + } this.#signals.cleanup(() => { this.#track.close(); + this.#ordered?.close(); for (const group of this.#groups) { group.consumer.close(); } @@ -336,6 +358,11 @@ export class Consumer { const consumer = await this.#track.recvGroup(); if (!consumer) break; + // Closed while the read waited: cleanup already ran, so nothing would release it. + if (this.#signals.abort.aborted) { + consumer.close(); + break; + } // Delivery starts at the floor the ranges name, raised to where the publisher says its feed // starts: it serves nothing below, and resolved a relative start against its own latest @@ -362,37 +389,46 @@ export class Consumer { consumer.close(); continue; } - const group: Group = { - consumer, - // When the group landed, not when it was pulled: a full max buffer delays pulls. - late: consumer.arrived, - frames: [], - costs: [], - empty: true, - media: false, - index: 0, - charged: 0, - }; - this.#charge(group, GROUP_COST); - - // Insert into #groups based on the group sequence number (ascending). - // This is used to cancel old groups. - let index = this.#groups.findIndex((other) => other.consumer.start > consumer.start); - if (index === -1) index = this.#groups.length; - // Each group is late from the earliest arrival among it and the groups past it. A drained - // backlog comes in sequence order, not arrival order, so an arrival can lower those below it. - const next = this.#groups.at(index); - if (next?.late.lt(group.late)) group.late = next.late; - for (let i = index - 1; i >= 0 && this.#groups[i].late.gt(group.late); i--) - this.#groups[i].late = group.late; - this.#groups.splice(index, 0, group); - // Group overhead alone can fill the max buffer, past which next() moves on from a stalled head. - if (this.#full()) { - this.#notify?.(); - this.#notify = undefined; - } + this.#add(consumer); + } + if (!this.#signals.abort.aborted) { + this.#delivered = true; + this.#settle(); + } + this.#finish(null); + } catch (err) { + this.#finish(err instanceof Error ? err : new Error(String(err))); + } + } - this.#signals.spawn(this.#watch.bind(this, group)); + // Read groups in key order from `ordered`, noting each gap it reports on the group after it. + async #runOrdered(ordered: Moq.Track.Ordered) { + try { + for (;;) { + // As #run: past the max buffer, later groups wait in the transport's cache. + while (this.#full() && this.#groups.length >= 2 && !this.#signals.abort.aborted) await this.#park(); + if (this.#signals.abort.aborted) break; + + const item = await ordered.nextItem(); + if (!item) break; + // Closed while the read waited: cleanup already ran, so nothing would release it. + if (this.#signals.abort.aborted) { + item.group?.close(); + break; + } + if (item.kind === "gap") { + this.#gap = true; + continue; + } + // Delivery starts at the first group: the reader already skipped what is too stale to + // play, and waited for the groups below it that its feed owed. + if (this.#active === undefined) { + this.#tuned.set(true); + this.#active = item.group.start; + this.#starting = false; + } + this.#add(item.group, this.#gap); + this.#gap = false; } if (!this.#signals.abort.aborted) { this.#delivered = true; @@ -404,6 +440,108 @@ export class Consumer { } } + // Buffer a group in sequence order and watch it for frames. + #add(consumer: Moq.Group.Consumer, gap = false): void { + const group: Group = { + consumer, + // When the group landed, not when it was pulled: a full max buffer delays pulls. + late: consumer.arrived, + frames: [], + costs: [], + gap, + empty: true, + media: false, + index: 0, + charged: 0, + }; + this.#charge(group, GROUP_COST); + + // Insert into #groups based on the group sequence number (ascending). + // This is used to cancel old groups. + let index = this.#groups.findIndex((other) => other.consumer.start > consumer.start); + if (index === -1) index = this.#groups.length; + // Each group is late from the earliest arrival among it and the groups past it. A drained + // backlog comes in sequence order, not arrival order, so an arrival can lower those below it. + const next = this.#groups.at(index); + if (next?.late.lt(group.late)) group.late = next.late; + for (let i = index - 1; i >= 0 && this.#groups[i].late.gt(group.late); i--) this.#groups[i].late = group.late; + this.#groups.splice(index, 0, group); + // Group overhead alone can fill the max buffer, past which next() moves on from a stalled head. + if (this.#full()) { + this.#notify?.(); + this.#notify = undefined; + } + + this.#signals.spawn(this.#watch.bind(this, group)); + } + + // Feed the arrival estimate from every frame of `#track` as it lands, while an ordered reader + // delivers: late media that reader gives up on must still move the estimate, or it could never rise. + async #observe() { + try { + for (;;) { + const group = await this.#track.recvGroup(); + if (!group) return; + // Closed while the read waited: cleanup already ran, so nothing would release it. + if (this.#signals.abort.aborted) { + group.close(); + return; + } + this.#signals.spawn(this.#observeGroup.bind(this, group)); + } + } catch { + // The ordered reader reports how the track ended. + } + } + + async #observeGroup(group: Moq.Group.Consumer) { + let media = false; + try { + for (;;) { + const wire = await group.readFrame(); + if (!wire || this.#signals.abort.aborted) return; + // A payload that fails to decode is the ordered read's to raise. + let decoded: Frame[]; + try { + decoded = this.#format.decode(wire.payload, wire.timestamp); + } catch { + return; + } + const arrived = (wire.arrived ?? now()).since(Instant.ZERO).asMillis(); + if (this.#format.kind === "data" && decoded.length === 0) this.#declare(group.start); + for (const sample of decoded) { + const marker = this.#format.end?.(sample) !== undefined; + this.#note(group.start, marker, media, sample.timestamp, arrived); + if (!marker) media = true; + } + } + } catch { + // A group cut short has no more frames to observe. + } finally { + group.close(); + } + } + + // Observe one decoded sample of group `sequence` for the arrival estimate: media is observed, + // and a marker ahead of any media (`media` false) declares an end. + #note(sequence: bigint, marker: boolean, media: boolean, timestamp: Duration, arrived: number): void { + // Observed before any gap handling or age budget: a target derived from what survives the + // budget would only ever confirm the budget it was cut to. + if (!marker) { + if (this.#mediaSequence === undefined || sequence > this.#mediaSequence) this.#mediaSequence = sequence; + if (this.#declared !== undefined && sequence > this.#declared) { + this.#jitter.end(); + this.#declared = undefined; + } + this.#jitter.observe(arrived, timestamp.asMillis()); + const spread = this.#jitter.measured; + if (spread !== this.#spread.peek().asMillis()) this.#spread.set(Duration.millis(spread)); + } else if (!media) { + // Only a marker ahead of any media declares an end; one after it is a frame's endpoint. + this.#declare(sequence); + } + } + // Watch `group` for wire frames and its end, waking the reader. Only #look decodes, and the // watcher runs it at once rather than leave it to the next read, so buffered ranges, arrival times // and the age budget keep up while the reader is busy. Completion, faults and the pressure skip are @@ -484,9 +622,8 @@ export class Consumer { return start; } - // Note an end the publisher declared in `group`, which a later group's first frame resumes from. - #declare(group: Group): void { - const sequence = group.consumer.start; + // Note an end the publisher declared in group `sequence`, which a later group's first frame resumes from. + #declare(sequence: bigint): void { if (this.#mediaSequence !== undefined && sequence < this.#mediaSequence) return; if (this.#declared === undefined || sequence > this.#declared) this.#declared = sequence; } @@ -515,7 +652,7 @@ export class Consumer { return; } group.bare = decoded.length === 0; - if (group.bare) this.#declare(group); + if (group.bare && !this.#ordered) this.#declare(group.consumer.start); } // Decoded payloads are views of the wire payload, which stays allocated until the last of them is @@ -523,23 +660,8 @@ export class Consumer { let views = 0; for (const sample of decoded) { const marker = this.#format.end?.(sample) !== undefined; - - // Observed here, before any gap handling or age budget: a target derived from - // what survives the budget would only ever confirm the budget it was cut to. - if (!marker) { - if (this.#mediaSequence === undefined || group.consumer.start > this.#mediaSequence) - this.#mediaSequence = group.consumer.start; - if (this.#declared !== undefined && group.consumer.start > this.#declared) { - this.#jitter.end(); - this.#declared = undefined; - } - this.#jitter.observe(arrived, sample.timestamp.asMillis()); - const spread = this.#jitter.measured; - if (spread !== this.#spread.peek().asMillis()) this.#spread.set(Duration.millis(spread)); - } else if (!group.media) { - // Only a marker ahead of any media declares an end; one after it is a frame's endpoint. - this.#declare(group); - } + // Observed here, as it comes off the transport, unless #observe sees every arrival instead. + if (!this.#ordered) this.#note(group.consumer.start, marker, group.media, sample.timestamp, arrived); const frame: ReceivedFrame = { arrival, payload: sample.payload, @@ -857,6 +979,7 @@ export class Consumer { this.#error = error; this.#finish(error); this.#track.close(error); + this.#ordered?.close(error); this.#notify?.(); this.#notify = undefined; // Nothing more is read, so stop the group watchers and release their groups rather than let a @@ -947,7 +1070,10 @@ export class Consumer { const excluded = next === undefined || next >= head.consumer.start; // The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it. const full = this.#full(); + // An ordered reader returns a group only once it resolved every key below it. + const decided = this.#ordered !== undefined; if ( + decided || startup || empty || contiguous || @@ -958,7 +1084,9 @@ export class Consumer { full || ended !== undefined ) { - if (!contiguous && !empty) { + // What an ordered reader gives up it reports as a gap; a key it skips without one was never a + // group. A gap an empty group carries still marks the next frame, which follows the loss. + if (!contiguous && (decided ? head.gap : !empty)) { this.#lost = true; if (!startup) { console.warn( diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index e50fdcd271e..1eef85f1f6f 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -748,6 +748,16 @@ export namespace Track { export interface InfoOptions { signal?: AbortSignal; } + export type Item = { + group: Group.Consumer; + kind: "group"; + } | { + end: bigint; + error: Error; + group?: never; + kind: "gap"; + start: bigint; + }; export type Order = "ascending" | "descending"; export class Ordered { close(abort?: Error): void; @@ -758,6 +768,7 @@ export namespace Track { largest(): Promise; readonly name: string; nextGroup(): Promise; + nextItem(): Promise; readFrame(): Promise<{ frame: number; group: bigint; @@ -766,6 +777,7 @@ export namespace Track { readString(): Promise; get subscription(): Signals.Getter; tryNextGroup(): Group.Consumer | undefined; + tryNextItem(): Track.Item | undefined; update(options: Track.Subscription): void; withRange(groups: Track.Groups): this; } diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 24b86efa33d..60fac91431a 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -474,8 +474,11 @@ export interface ResumeSource { * `dispositions`, a run no copy may still deliver is reported once as a `drop`. */ tryGroup(ordered: boolean, delivered: Delivered, dispositions: boolean): Recv; - /** The next run no copy may still deliver, reported once as a `drop`, without taking a group. */ - tryDrop(delivered: Delivered): (Recv & { kind: "drop" }) | undefined; + /** + * The next run no copy may still deliver, reported once as a `drop`, without taking a group. + * Below `passing`, which a key-order read is about to pass, no copy can deliver one later. + */ + tryDrop(delivered: Delivered, passing?: bigint): (Recv & { kind: "drop" }) | undefined; /** * Whether the reader's bounded `window` has nothing left to deliver, judged by the serving * copy against what it and the logical reader (`delivered`) handed out. A copy that may still @@ -825,10 +828,14 @@ export const hooks: { exhaustedFor: (subscriber: Subscriber, delivered: Delivered) => boolean; /** How many runs of handed-out groups a subscriber remembers: one per gap, plus the floor. Tests only. */ deliveredRuns: (subscriber: Subscriber) => number; + /** How many runs a subscriber's record of what it reported holds. Tests only. */ + reportedRuns: (subscriber: Subscriber) => number; /** Whether the subscriber already handed out group `key`, or no longer expects it. */ handedOut: (subscriber: Subscriber, key: bigint) => boolean; /** Forget the gaps in another reader's `delivered` below each group past this subscriber's budget. */ forgetPastBudget: (subscriber: Subscriber, delivered: Delivered) => void; + /** Fold a route copy's record below `at`, where the reader it feeds is, which reports nothing there. */ + fold: (subscriber: Subscriber, at: bigint) => void; /** * What a lite publisher told its wire about a subscriber's keys, kept in the * subscriber's own record. Mirrors the Rust crate-private `track::Subscriber::record_mut`. @@ -1169,6 +1176,9 @@ export const hooks: { forgetPastBudget: () => { throw new Error("track.ts not loaded"); }, + fold: () => { + throw new Error("track.ts not loaded"); + }, wire: () => { throw new Error("track.ts not loaded"); }, @@ -1178,6 +1188,9 @@ export const hooks: { deliveredRuns: () => { throw new Error("track.ts not loaded"); }, + reportedRuns: () => { + throw new Error("track.ts not loaded"); + }, handedOut: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/ordered.test.ts b/js/net/src/ordered.test.ts new file mode 100644 index 00000000000..b29bdcf5b2f --- /dev/null +++ b/js/net/src/ordered.test.ts @@ -0,0 +1,407 @@ +// What an ordered read returns from nextItem: groups in key order, and a gap for every group it gives up on. +// Ports the Rust `track::Ordered` item tests, so both read the same. +import { expect, test } from "bun:test"; +import { Duration, sleep } from "@moq/clock"; +import { Manual } from "@moq/clock/test"; +import { StreamCode, StreamError } from "./error.ts"; +import { hooks, staleHole } from "./internal.ts"; +import { textFrame } from "./mock.ts"; +import { Timestamp } from "./time.ts"; +import { type Ordered, Producer as TrackProducer } from "./track.ts"; + +// A session-fed copy owes holes; an in-process producer does not. +function receivedTrack() { + const producer = new TrackProducer("test").accept({ maxAge: null }); + hooks.receiveTrack(producer); + return producer; +} + +// A finished one-frame group, presenting at its key in milliseconds on a timed track. +function put(producer: TrackProducer, key: bigint, timed = true) { + const group = producer.createGroup(key); + group.writeFrame(textFrame("x", timed ? Timestamp.fromMillis(Number(key)) : Timestamp.unscaled(key))); + group.finish(); +} + +function drop(producer: TrackProducer, start: bigint, end: bigint) { + hooks.dropTrackGroups(producer, start, end, { error: new StreamError(StreamCode.DeliveryTimeout) }); +} + +const NAMES: Record = { + [StreamCode.DeliveryTimeout]: "delivery timeout", + [StreamCode.Old]: "old", + [StreamCode.NotFound]: "not found", +}; + +// What an ordered read returns until it would wait, as `group 2` or `gap 1..2 (old)`. +function items(ordered: Ordered): string[] { + const out: string[] = []; + for (;;) { + if (out.length > 20) throw new Error(`no end to the items: ${out.join(", ")}`); + const item = ordered.tryNextItem(); + if (!item) return out; + if (item.kind === "group") { + out.push(`group ${item.group.start}`); + item.group.close(); + } else { + const code = item.error instanceof StreamError ? item.error.code : -1; + out.push(`gap ${item.start}..${item.end} (${NAMES[code] ?? item.error.message})`); + } + } +} + +function withClock(fn: (clock: Manual) => void) { + const clock = new Manual(); + const installed = clock.install(); + try { + fn(clock); + } finally { + installed.close(); + } +} + +test("an ordered read reports what it gives up, in key order, but no skip", () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + put(producer, 3n); + drop(producer, 1n, 3n); + producer.skipGroups(4n, 6n); + put(producer, 6n); + expect(items(ordered)).toEqual(["group 0", "gap 1..3 (delivery timeout)", "group 3", "group 6"]); + + // A group handed out and then dropped is never reported again. + const cut = producer.createGroup(7n); + const read = ordered.tryNextItem(); + expect(read?.group?.start).toBe(7n); + drop(producer, 7n, 8n); + cut.abort(new StreamError(StreamCode.Cancel)); + expect(items(ordered)).toEqual([]); +}); + +test("nextGroup and tryNextGroup pass the gaps nextItem reports", async () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + drop(producer, 1n, 2n); + put(producer, 2n); + drop(producer, 3n, 4n); + put(producer, 4n); + expect((await ordered.nextGroup())?.start).toBe(0n); + expect(ordered.tryNextGroup()?.start).toBe(2n); + expect(ordered.tryNextItem()).toMatchObject({ kind: "gap", start: 3n, end: 4n }); + expect((await ordered.nextGroup())?.start).toBe(4n); +}); + +test("keys below where a session's feed starts are no gap unless a fill owed them", () => { + const producer = receivedTrack(); + hooks.startTrack(producer, { start: 10n }); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + // Group 0 comes from history, as a fill would bring it; the feed owes nothing below 10. + put(producer, 0n, false); + put(producer, 10n, false); + expect(items(ordered)).toEqual(["group 0", "group 10"]); +}); + +test("a group held behind a gap is not returned once the reader closes", () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + drop(producer, 1n, 2n); + put(producer, 2n); + expect(ordered.tryNextItem()?.group?.start).toBe(0n); + expect(ordered.tryNextItem()).toMatchObject({ kind: "gap", start: 1n, end: 2n }); + ordered.close(); + expect(ordered.tryNextItem()).toBeUndefined(); +}); + +test("a key passed under a stale verdict that does not settle this reader is still a gap", () => + withClock((clock) => { + const producer = receivedTrack(); + const ordered = producer.subscribe({ delay: { max: Duration.millis(50) } }).ordered(); + put(producer, 0n, false); + expect(items(ordered)).toEqual(["group 0"]); + // The feed gave key 1 up under a 10 ms budget, which binds no reader waiting 50 ms. + hooks.dropTrackGroups(producer, 1n, 2n, staleHole(10)); + put(producer, 2n, false); + clock.jump(Duration.millis(60)); + put(producer, 3n, false); + expect(items(ordered)).toEqual(["gap 1..2 (old)", "group 2", "group 3"]); + })); + +test("a key a local track never produced is no gap", () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: Duration.secs(1) } }).ordered(); + put(producer, 0n); + put(producer, 2n); + expect(items(ordered)).toEqual(["group 0", "group 2"]); +}); + +test("a gap waits behind what is owed below it, and goes out before the end", () => + withClock(() => { + const producer = receivedTrack(); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n, false); + put(producer, 2n, false); + drop(producer, 4n, 5n); + expect(items(ordered)).toEqual(["group 0"]); + put(producer, 1n, false); + expect(items(ordered)).toEqual(["group 1", "group 2"]); + put(producer, 3n, false); + expect(items(ordered)).toEqual(["group 3", "gap 4..5 (delivery timeout)"]); + })); + +test("the last gap goes out before the end, and the read then ends", async () => { + const local = new TrackProducer("test"); + const read = local.subscribe({ delay: { max: null } }).ordered(); + put(local, 0n); + drop(local, 1n, 2n); + local.finish(); + expect(items(read)).toEqual(["group 0", "gap 1..2 (delivery timeout)"]); + const end = await Promise.race([read.nextItem(), sleep(Duration.millis(100)).then(() => "hung")]); + expect(end).toBeUndefined(); +}); + +test("a gap is returned once, even when a later drop names it again", () => + withClock((clock) => { + const producer = receivedTrack(); + const ordered = producer.subscribe({ delay: { max: Duration.millis(50) } }).ordered(); + put(producer, 0n, false); + put(producer, 2n, false); + expect(items(ordered)).toEqual(["group 0"]); + // The budget gives up on the group still owed, as stale. + clock.jump(Duration.millis(50)); + expect(items(ordered)).toEqual(["gap 1..2 (old)", "group 2"]); + drop(producer, 1n, 2n); + put(producer, 3n, false); + expect(items(ordered)).toEqual(["group 3"]); + })); + +test("a skip replaces a gap still waiting, and the rest of the drop stays one", () => + withClock(() => { + const producer = receivedTrack(); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n, false); + put(producer, 2n, false); + drop(producer, 4n, 8n); + expect(items(ordered)).toEqual(["group 0"]); + producer.skipGroups(5n, 7n); + drop(producer, 1n, 2n); + expect(items(ordered)).toEqual(["gap 1..2 (delivery timeout)", "group 2"]); + put(producer, 3n, false); + expect(items(ordered)).toEqual(["group 3", "gap 4..5 (delivery timeout)", "gap 7..8 (delivery timeout)"]); + })); + +test("a cached group the seek passes as stale is a gap", () => + withClock((clock) => { + const producer = receivedTrack(); + put(producer, 0n, false); + put(producer, 1n, false); + clock.jump(Duration.millis(100)); + put(producer, 2n, false); + const ordered = producer.subscribe({ delay: { max: Duration.millis(50) } }).ordered(); + expect(items(ordered)).toEqual(["gap 0..1 (old)", "group 1", "group 2"]); + })); + +// Rust reports it Old; JS keeps no record below the read, so its memory stays bounded. +test("a local track's late arrival is no gap", () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + put(producer, 5n); + expect(items(ordered)).toEqual(["group 0", "group 5"]); + put(producer, 3n); + expect(items(ordered)).toEqual([]); + put(producer, 6n); + expect(items(ordered)).toEqual(["group 6"]); +}); + +test("a gap is clipped to the reader's range", () => { + const producer = new TrackProducer("test"); + const ordered = producer + .subscribe({ delay: { max: null } }) + .ordered() + .withRange({ start: { included: 2n }, end: { excluded: 4n } }); + drop(producer, 0n, 6n); + put(producer, 5n); + expect(items(ordered)).toEqual(["gap 2..4 (delivery timeout)"]); +}); + +test("the frame helpers read past a gap", async () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + drop(producer, 1n, 2n); + put(producer, 2n); + expect((await ordered.readFrame())?.group).toBe(0n); + expect((await ordered.readFrame())?.group).toBe(2n); +}); + +test("a local track waits for what it asked a fill for, and reports a gap once the budget gives up", () => + withClock((clock) => { + const producer = new TrackProducer("test"); + const requests = producer.requests(); + const ordered = producer.subscribe({ delay: { max: Duration.millis(50) } }).ordered(); + put(producer, 0n); + put(producer, 2n); + expect(items(ordered)).toEqual(["group 0"]); + // Group 1 lands late, within the budget, as a group does. + put(producer, 1n); + expect(items(ordered)).toEqual(["group 1", "group 2"]); + put(producer, 4n); + expect(items(ordered)).toEqual([]); + clock.jump(Duration.millis(50)); + expect(items(ordered)).toEqual(["gap 3..4 (old)", "group 4"]); + requests.close(); + })); + +test("a gap moves the read past it, so a group landing there after is no group", () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + expect(items(ordered)).toEqual(["group 0"]); + drop(producer, 1n, 2n); + expect(items(ordered)).toEqual(["gap 1..2 (delivery timeout)"]); + put(producer, 1n); + expect(items(ordered)).toEqual([]); + put(producer, 2n); + expect(items(ordered)).toEqual(["group 2"]); +}); + +test("a group held past a gap, under a cap lowered off it, still sees the read end", async () => { + const producer = new TrackProducer("test"); + const ordered = producer.subscribe({ delay: { max: null } }).ordered(); + put(producer, 0n); + drop(producer, 1n, 2n); + put(producer, 2n); + expect((await ordered.nextItem())?.group?.start).toBe(0n); + expect((await ordered.nextItem())?.kind).toBe("gap"); + ordered.withRange({ end: { excluded: 2n } }); + expect(ordered.tryNextItem()).toBeUndefined(); + const pending = ordered.nextItem(); + ordered.close(); + const ended = await Promise.race([ + pending.then( + (item) => (item === undefined ? "ended" : `item ${item.kind}`), + (err: Error) => `rejected ${err.message}`, + ), + sleep(Duration.millis(100)).then(() => "hung"), + ]); + expect(ended).toBe("ended"); +}); + +test("a stale group passes the owed holes below it as gaps", () => + withClock((clock) => { + const producer = receivedTrack(); + const ordered = producer.subscribe({ delay: { max: Duration.millis(50) } }).ordered(); + put(producer, 0n, false); + expect(items(ordered)).toEqual(["group 0"]); + // Group 1 is owed; group 3 lands, then group 2 after it, which turns stale while the wall + // clock still spares the hole at 1. + put(producer, 3n, false); + clock.jump(Duration.millis(30)); + put(producer, 2n, false); + clock.jump(Duration.millis(30)); + expect(items(ordered)).toEqual(["gap 1..3 (old)", "group 3"]); + })); + +test("an ordered read's record holds only what lies ahead of it", () => { + const producer = new TrackProducer("test"); + const sub = producer.subscribe({ delay: { max: null } }); + const ordered = sub.ordered(); + // Keyed apart, as audio by timestamp is: each group would be a run of its own. + for (let i = 0n; i < 3000n; i++) { + put(producer, i * 20n); + expect(items(ordered)).toEqual([`group ${i * 20n}`]); + expect(hooks.reportedRuns(sub)).toBeLessThanOrEqual(1); + } + // Unlike Rust, a group a local track produces below the read is told as nothing. + put(producer, 10n); + put(producer, 2998n * 20n + 10n); + expect(items(ordered)).toEqual([]); +}); + +// A key-order read, as a route copy is read, folds at each hand-out while the keys it passed +// still wait to be reported. +test("a session-fed key-order read folds without losing the keys it passed", () => + withClock((clock) => { + const producer = receivedTrack(); + const sub = producer.subscribe({ delay: { max: Duration.millis(50) } }); + const read = () => { + const out: string[] = []; + for (let drop = hooks.tryDrop(sub); drop; drop = hooks.tryDrop(sub)) + out.push(`drop ${drop.start}..${drop.end}`); + const next = hooks.tryNextGroup(sub); + if (next.kind === "group") { + next.group.close(); + out.push(`group ${next.group.start}`); + } + for (let drop = hooks.tryDrop(sub); drop; drop = hooks.tryDrop(sub)) + out.push(`drop ${drop.start}..${drop.end}`); + return out; + }; + put(producer, 0n, false); + expect(read()).toEqual(["group 0"]); + for (let i = 1n; i <= 100n; i++) { + // Group 3i-2 never lands, and 3i-1 turns stale behind 3i: the read passes both to hand + // out 3i, and still reports them. + put(producer, 3n * i, false); + clock.jump(Duration.millis(30)); + put(producer, 3n * i - 1n, false); + clock.jump(Duration.millis(30)); + expect(read()).toEqual([`group ${3n * i}`, `drop ${3n * i - 2n}..${3n * i}`]); + } + expect(hooks.reportedRuns(sub)).toBeLessThanOrEqual(2); + sub.close(); + })); + +test("a gap the bounds left out below the read never pins its record", () => { + const producer = receivedTrack(); + const sub = producer.subscribe({ delay: { max: null } }); + const ordered = sub.ordered(); + put(producer, 0n, false); + expect(items(ordered)).toEqual(["group 0"]); + // Group 1 is owed, so the gap at 2 waits; the bounds then leave both out. + drop(producer, 2n, 3n); + expect(items(ordered)).toEqual([]); + ordered.withRange({ start: { included: 3n } }); + for (let i = 0n; i < 1000n; i++) { + const key = 3n + i * 20n; + put(producer, key, false); + producer.skipGroups(key + 1n, key + 20n); + expect(items(ordered)).toEqual([`group ${key}`]); + } + expect(hooks.reportedRuns(sub)).toBeLessThanOrEqual(1); + // The read never goes back, so widening the bounds again offers nothing below it. + ordered.withRange({ start: { included: 0n } }); + expect(items(ordered)).toEqual([]); +}); + +test("a key-order read draining its drops folds its record", () => { + const producer = new TrackProducer("test"); + const sub = producer.subscribe({ delay: { max: null } }); + // Keyed apart with the keys between skipped, so each skip is a run of its own. + for (let i = 0n; i < 3000n; i++) { + put(producer, i * 20n); + producer.skipGroups(i * 20n + 1n, i * 20n + 20n); + const read = hooks.tryNextGroup(sub); + if (read.kind === "group") read.group.close(); + while (hooks.tryDrop(sub)); + } + expect(hooks.reportedRuns(sub)).toBeLessThanOrEqual(1); + sub.close(); +}); + +test("a read returning only gaps keeps its handed-out record compact", () => { + const producer = new TrackProducer("test"); + const sub = producer.subscribe({ delay: { max: null } }); + const ordered = sub.ordered(); + put(producer, 0n); + expect(items(ordered)).toEqual(["group 0"]); + for (let i = 1n; i <= 100n; i++) { + producer.skipGroups(i * 20n - 19n, i * 20n); + drop(producer, i * 20n, i * 20n + 1n); + expect(items(ordered)).toEqual([`gap ${i * 20n}..${i * 20n + 1n} (delivery timeout)`]); + } + expect(hooks.deliveredRuns(sub)).toBe(1); +}); diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 5f27cc6ce0f..854ccc3bd1e 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1890,6 +1890,37 @@ test("a routed read right after an update picks its group under the new preferen } }); +// The copy reports the owed key it passed to take the group above, and the reader returns that +// gap before the group, as a direct read of the same track does. +test("a routed ordered read returns the gap its copy passed to take a group", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + hooks.receiveTrack(ta.producer); + front.serve(a.consume()); + const ordered = front.consumer + .track("seq") + .subscribe({ delay: { max: Duration.millis(1000) } }) + .ordered(); + try { + ta.write(4n); + expect((await ordered.nextItem())?.group?.start).toBe(4n); + ta.write(6n); + const held = ordered.nextItem(); + expect(await isPending(held)).toBe(true); + await clock.advance(Duration.millis(1000)); + expect(await held).toMatchObject({ kind: "gap", start: 5n, end: 6n }); + expect((await ordered.nextItem())?.group?.start).toBe(6n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + installed.close(); + } +}); + for (const resolution of ["arrival", "drop", "other copy", "budget"] as const) { test(`an ordered resumed reader keeps an older copy's owed hole until ${resolution}`, async () => { const clock = new Manual(); @@ -2076,6 +2107,43 @@ test("a resumed reader reports a drop once and still serves a late group", async a.close(); }); +// A copy folds its record at where the reader is, so one that only reports drops while the reader +// returns gaps holds a bounded record however long it plays. +test("a route copy's record folds while the reader returns only gaps", () => { + const copies = new Set(); + const fold = hooks.fold; + hooks.fold = (sub, at) => { + copies.add(sub); + fold(sub, at); + }; + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const ordered = front.consumer + .track("seq") + .subscribe({ delay: { max: null } }) + .ordered(); + try { + ta.write(0n); + expect(ordered.tryNextItem()).toMatchObject({ kind: "group" }); + const cut = new StreamError(StreamCode(70), { message: "cut" }); + for (let i = 1n; i <= 1000n; i++) { + // A drop and a skip apart, so each would be a run of its own in the copy's record. + hooks.dropTrackGroups(ta.producer, 2n * i - 1n, 2n * i, { error: cut }); + ta.producer.skipGroups(2n * i, 2n * i + 1n); + expect(ordered.tryNextItem()).toMatchObject({ kind: "gap", start: 2n * i - 1n, end: 2n * i }); + } + expect(copies.size).toBeGreaterThan(0); + for (const sub of copies) expect(hooks.reportedRuns(sub)).toBeLessThanOrEqual(4); + } finally { + hooks.fold = fold; + ordered.close(); + front.consumer.close(); + a.close(); + } +}); + // A skip of a run the serving copy already dropped replaces the drop, so the resumed reader // reports it again, as a skip, and only once. Mirrors Rust. test("a resumed reader reports a skip over a reported drop", () => { diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index a20d591635f..09882f1174e 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -898,11 +898,11 @@ class Reader implements ResumeSource { return hooks.exhaustedFor(serving.sub, delivered); } - tryDrop(delivered: Delivered): (Recv & { kind: "drop" }) | undefined { + tryDrop(delivered: Delivered, passing?: bigint): (Recv & { kind: "drop" }) | undefined { this.#delivered = delivered; this.sync(); this.#learnEnd(); - return this.#tryDrop(delivered); + return this.#tryDrop(delivered, passing); } /** @@ -911,8 +911,12 @@ class Reader implements ResumeSource { * is under its bound, so a run there waits until that copy no longer feeds. Mirrors the * Rust `Reader::poll_drop`. */ - #tryDrop(delivered: Delivered): (Recv & { kind: "drop" }) | undefined { + #tryDrop(delivered: Delivered, passing = 0n): (Recv & { kind: "drop" }) | undefined { if (this.#closed) return undefined; + // Below what the reader handed out or passed, no copy's drop is reported, so each copy's + // record folds there, even one that hands out nothing while the reader returns gaps. + const at = delivered.runEnd(0n) ?? 0n; + for (const copy of this.#copies) hooks.fold(copy.sub, at); const serving = this.#serving(); if (serving) { // A received stale run queued here that a new upstream subscription superseded since is @@ -944,7 +948,10 @@ class Reader implements ResumeSource { for (const run of this.#drops) { const end = min(run.end, cursor.end ?? run.end); for (const gap of delivered.gaps(max(run.start, cursor.start), end)) { - const [open] = runGaps(feeding, gap.start, gap.end); + const [open] = + gap.start < passing + ? [{ start: gap.start, end: min(gap.end, passing) }] + : runGaps(feeding, gap.start, gap.end); if (open) { // Report once without marking it delivered: a late fill may still land. this.#drops = removeRun(this.#drops, open.start, open.end); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 1c5ea22ddb9..350548760e2 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -83,6 +83,19 @@ function taken(read: () => Recv, ended: () => undefined): GroupConsumer | undefi } } +/** What {@link Ordered.nextItem} returns next. Mirrors the Rust `track::Item`. */ +export type Item = + /** A group, with a higher key than any group or gap returned before. */ + | { kind: "group"; group: GroupConsumer } + /** + * Groups `[start, end)` this reader gave up on, which it will not return, with what a read of + * them would have ended with. `group` is absent, so `item?.group` reads a group or nothing. + */ + | { kind: "gap"; start: bigint; end: bigint; error: Error; group?: never }; + +// What an ordered read found: an item, or the same idle, end, and error as a group read. +type ItemRecv = Item | { kind: "idle" } | { kind: "done" } | { kind: "error"; error: Error }; + // The unset priority, the midpoint of the signed range: moq-lite carries it as 128 and // moq-transport as 127, one urgency on both. const DEFAULT_PRIORITY = 0; @@ -971,7 +984,11 @@ let mayLand: (producer: Producer, land: Land) => boolean; // Recompute the producer's aggregates now, for a sink that just became a fetch; assigned in its // static block. let resubscribe: (producer: Producer) => void; +// Where a session's live feed starts owing groups, if known: it owes none below. +let feedFloor: (producer: Producer) => bigint | undefined; let owedRuns: (producer: Producer, start: bigint, end: bigint) => Iterable; +// Whether a wire subscription feeds the producer, so it owes every group it has not accounted for. +let wireFed: (producer: Producer) => boolean; let owedGroup: (producer: Producer, start: bigint, end: bigint) => { key: bigint; folds?: Instant } | undefined; let fillEdge: (producer: Producer) => bigint | undefined; let sourceStart: (producer: Producer) => bigint | "pending" | undefined; @@ -1066,7 +1083,8 @@ let fillsOf: (producer: Producer) => FillState; let makeOrdered: (subscriber: Subscriber) => Ordered; let ordered_: { nextGroup(subscriber: Subscriber): Promise; - tryNextGroup(subscriber: Subscriber): Recv; + nextItem(subscriber: Subscriber): Promise; + tryItem(subscriber: Subscriber): ItemRecv; ended(subscriber: Subscriber): undefined; readFrame(subscriber: Subscriber): Promise<({ group: bigint; frame: number } & Frame) | undefined>; readString(subscriber: Subscriber): Promise; @@ -1482,10 +1500,15 @@ export class Producer { } }; mayLand = (producer, land) => producer.#landable(land) !== undefined; + wireFed = (producer) => producer.#wire !== undefined; + feedFloor = (producer) => { + if (!producer.#wire) return undefined; + const feed = producer.#feedStart(); + return feed === "pending" ? producer.#wire.served?.requested : feed; + }; owedRuns = (producer, start, end) => { if (!producer.#wire) return []; - const feed = producer.#feedStart(); - const floor = feed === "pending" ? producer.#wire.served?.requested : feed; + const floor = feedFloor(producer); return producer.#landing({ start: max(start, floor ?? 0n), end, @@ -2611,6 +2634,21 @@ type Drift = { end?: bigint; }; +// An ordered read's state, mirroring the Rust `Ordered`: a group read and held back because a gap +// below it goes first, the gaps found and not returned yet (sorted and disjoint), and one past +// the highest key returned, as a group or the end of a gap. +type Items = { held?: GroupConsumer; gaps: DropRun[]; at: bigint }; + +// A group that landed after an ordered read passed its key. +function late(): Hole { + return { error: new StreamError(StreamCode.Old, { message: "group landed after the read passed it" }) }; +} + +// A group missing its head that no fill could serve. +function headless(): Hole { + return { error: new StreamError(StreamCode.NotFound, { message: "group head never served" }) }; +} + // The successor a handed-out group was last judged by, so the group stays judgeable once the // cache evicts both. Each reader keeps its own, so the memory is bounded by the groups held. type Superseded = { successor?: Successor }; @@ -2754,6 +2792,14 @@ export class Subscriber { coverage?: Coverage; }; #reported = new Runs(sameReport); + // Everything below was settled for good and folded out of #reported: handed out, told, or + // passed, so nothing there is reported again. Mirrors the Rust `Record::floor`, which folds + // under pool pressure; a key-order read folds at where it is instead (see #fold). + #floor = 0n; + // A wire publisher reads #reported as what its wire was told, so it never folds. + #wired = false; + // What an ordered read owes before its next group; unset before its first read. See #tryItem. + #items?: Items; // The group the frame-level helpers are currently draining, acquired through the // key cursor so frame reads and {@link Ordered.nextGroup} share one floor. #frameGroup?: GroupConsumer; @@ -3079,6 +3125,7 @@ export class Subscriber { const forgot = this.#delivered.runEnd(0n) ?? 0n; if (forgot > floor) this.#stoppedReading(floor, forgot); if (key >= forgot) this.#stoppedReading(key, key + 1n); + if (ordered) this.#fold(forgot); } // This reader stopped reading `[start, end)` for good: clip what it asked there, and judge @@ -3223,6 +3270,7 @@ export class Subscriber { // The publisher's view of this reader's record. #wireRecord(): WireRecord { + this.#wired = true; const reported = this.#reported; return { told: (key) => reported.at(key) !== undefined, @@ -3958,8 +4006,10 @@ export class Subscriber { return subscriber.#peekGroup(ordered); }; hooks.forgetPastBudget = (subscriber, delivered) => subscriber.#forgetPastBudget(delivered); + hooks.fold = (subscriber, at) => subscriber.#fold(at); hooks.wire = (subscriber) => subscriber.#wireRecord(); hooks.deliveredRuns = (subscriber) => subscriber.#delivered.runs; + hooks.reportedRuns = (subscriber) => subscriber.#reported.size; hooks.handedOut = (subscriber, key) => subscriber.#delivered.has(key); hooks.hasUnreadGroup = (subscriber, start, end, delivered) => subscriber.#state.groups @@ -4110,7 +4160,8 @@ export class Subscriber { // the arrival cursor); `Ordered` is the handle that reaches it. ordered_ = { nextGroup: (subscriber) => subscriber.#nextGroup(), - tryNextGroup: (subscriber) => subscriber.#tryNextGroup(), + nextItem: (subscriber) => subscriber.#nextItem(), + tryItem: (subscriber) => subscriber.#tryItem(), ended: (subscriber) => subscriber.#ended(), readFrame: (subscriber) => subscriber.#readFrame(), readString: (subscriber) => subscriber.#readString(), @@ -4382,6 +4433,9 @@ export class Subscriber { }); if (this.#frameGroup) hooks.releaseGroupReader(this.#frameGroup, abort); this.#frameGroup = undefined; + // A group an ordered read holds behind a gap is buffered too, so the read ends without it. + if (this.#items?.held) hooks.releaseGroupReader(this.#items.held, abort); + if (this.#items) this.#items.held = undefined; this.#state.timeline.length = 0; this.#wall.timer?.abort(); this.#wall.timer = undefined; @@ -4558,7 +4612,7 @@ export class Subscriber { // The first part of `[from, to)` nothing already reported blocks, now reported, with why, // replacing what it goes out over. Mirrors the Rust `Record::report`. #reportRun(from: bigint, to: bigint, hole: Hole): (Recv & { kind: "drop" }) | undefined { - const start = from; + const start = max(from, this.#floor); if (start >= to) return undefined; const blocked = this.#reported.overlapping(start, to).filter((reported) => blocksHole(reported, hole)); const [gap] = runGaps(blocked, start, to); @@ -4673,8 +4727,21 @@ export class Subscriber { this.#queueResumedDrop(recv); continue; } - if (recv.kind === "group") this.#handOut(recv.group.start, ordered); - else if (recv.kind === "idle" && this.#exhausted()) return { kind: "done" }; + if (recv.kind === "group") { + // Passing the keys below it hands them out as far as #delivered knows, so first take what + // the copies gave up there: a session-fed copy reports a key it passed to take this group. + if (ordered) { + const key = recv.group.start; + for ( + let drop = resume.tryDrop(this.#delivered, key); + drop; + drop = resume.tryDrop(this.#delivered, key) + ) { + this.#queueResumedDrop(drop); + } + } + this.#handOut(recv.group.start, ordered); + } else if (recv.kind === "idle" && this.#exhausted()) return { kind: "done" }; return recv; } } @@ -4774,12 +4841,13 @@ export class Subscriber { } // The key cursor behind {@link Ordered}, which owns the only public door to it. - async #nextGroup(): Promise { + async #nextItem(): Promise { for (;;) { - const recv = this.#tryNextGroup(); + const recv = this.#tryItem(); switch (recv.kind) { case "group": - return recv.group; + case "gap": + return recv; case "done": return this.#ended(); case "error": @@ -4789,6 +4857,215 @@ export class Subscriber { } } + // The next group in key order, past any gap. + async #nextGroup(): Promise { + for (;;) { + const item = await this.#nextItem(); + if (!item || item.kind === "group") return item?.group; + } + } + + // The synchronous half of #nextItem: the next group, or a gap the read gave up on below it, + // as the Rust `Ordered::poll_item` merges them. Every gap goes out before the end. + #tryItem(): ItemRecv { + let items = this.#items; + if (!items) { + items = this.#items = { gaps: [], at: 0n }; + // The drop record starts before the first read, so the cursor keeps what it gives up. + this.#findGaps(items); + } + const read = this.#readHeld(items); + this.#findGaps(items); + switch (read.kind) { + case "group": { + const key = read.group.start; + const lowest = this.#lowestGap(items); + // The gap lies wholly below it, or around it: the part below goes first. + if (lowest && lowest.span.start < key) { + // What lies above the group, or above the run when it ends before it, waits. + const resume = lowest.span.end <= key ? lowest.span.end : key + 1n; + const below = { start: lowest.span.start, end: min(lowest.span.end, key) }; + items.held = read.group; + return this.#takeGap(items, lowest.index, below, resume); + } + // A group handed out is no gap, even one a drop also named, as a cut group is. + if (lowest && lowest.span.start === key) this.#cutGap(items, lowest.index, key, key + 1n); + items.at = max(items.at, key + 1n); + this.#fold(this.#delivered.runEnd(0n) ?? 0n); + return read; + } + case "done": + case "error": { + const lowest = this.#lowestGap(items); + if (lowest) return this.#takeGap(items, lowest.index, lowest.span, lowest.span.end); + return read; + } + case "drop": + // Only a read with dispositions reports a drop, which this one finds through the record. + throw new Error("an ordered read took a drop"); + default: { + // What the read still owes lies above the gap, so the gap goes first. + const lowest = this.#lowestGap(items); + if (!lowest) return { kind: "idle" }; + const owed = this.nextInRange(items.at); + const ready = lowest.span.start <= items.at || owed === undefined || owed >= lowest.span.start; + return ready ? this.#takeGap(items, lowest.index, lowest.span, lowest.span.end) : { kind: "idle" }; + } + } + } + + // The group held behind a gap, while the bounds still cover it, else the cursor's next. A group + // read is settled, so no later cut or drop of it is a gap. + #readHeld(items: Items): Recv { + const held = items.held; + if (held) { + items.held = undefined; + if (this.#covered(held.start, held.start + 1n).length > 0) return { kind: "group", group: held }; + // The bounds moved off it. With nothing they cover above, it waits for them to widen again, + // still polling the cursor so the read sees the track or itself end; otherwise the read + // passes it. + if (this.#covered(held.start, UNBOUNDED).length === 0) { + const read = this.#tryNextGroup(); + // A clean end keeps it, as the buffer keeps a group past the cap, for a raise to read. + if (read.kind === "error") held.close(); + else items.held = held; + return read.kind === "done" || read.kind === "error" ? read : { kind: "idle" }; + } + held.close(); + } + const read = this.#tryNextGroup(); + if (read.kind === "group") this.#settled(read.group.start, read.group.start + 1n); + return read; + } + + // The parts of `[start, end)` this reader's ranges and bounds cover now. + #covered(start: bigint, end: bigint): Run[] { + const cursor = this.#cursor.peek(); + const from = max(start, cursor.start); + const to = cursor.end === undefined ? end : min(end, cursor.end); + if (from >= to) return []; + if (this.#resume) return this.#resume.runs(from, to) ?? [{ start: from, end: to }]; + return this.#window.runs(from, to); + } + + // Take every run the record gave up into the gaps waiting, in whatever order they come. A drop + // replaces what it overlaps, and a skip was never a group, so it removes them. Runs come lowest + // first, so one past every gap waiting is appended without copying the rest. + #findGaps(items: Items): void { + for (let dropped = this.#tryDrop(); dropped; dropped = this.#tryDrop()) { + const last = items.gaps.at(-1); + if (!last || last.end <= dropped.start) { + if (dropped.skip) continue; + const gap = { start: dropped.start, end: dropped.end, error: dropped.error }; + if (last && last.end === gap.start && sameHole(last, gap)) last.end = gap.end; + else items.gaps.push(gap); + continue; + } + items.gaps = dropped.skip + ? removeRun(items.gaps, dropped.start, dropped.end) + : insertDrop(items.gaps, dropped.start, dropped.end, { error: dropped.error }); + } + } + + // The lowest gap waiting that the bounds cover, as its index and the first part of it they cover, + // since they may have moved since it was found. A gap they exclude waits, in case they widen. + #lowestGap(items: Items): { index: number; span: Run } | undefined { + for (let index = 0; index < items.gaps.length; index++) { + const gap = items.gaps[index]; + const [span] = this.#covered(gap.start, gap.end); + if (span) return { index, span }; + } + return undefined; + } + + // Take `given` from the gap at `index` as an item, which the read has now moved past. What lies + // below `given`, which the bounds leave out, and from `resume` on stay gaps. + #takeGap(items: Items, index: number, given: Run, resume: bigint): Item { + const error = this.#cutGap(items, index, given.start, resume); + items.at = max(items.at, given.end); + // Told once: no later drop of these keys is a gap again. + this.#settled(given.start, given.end); + // Passed for good, as a group handed out is: an ordered read never goes back, so no group + // below its end is handed out after it, whether it lands late here or on a resumed + // reader's copy, and a fill no longer owes it there. + const floor = this.#delivered.runEnd(0n) ?? 0n; + this.#delivered.forgetBelow(given.end); + const forgot = this.#delivered.runEnd(0n) ?? 0n; + if (forgot > floor) this.#stoppedReading(floor, forgot); + if (!this.#resume) this.#nextKey = max(this.#nextKey, given.end); + this.#fold(forgot); + return { kind: "gap", start: given.start, end: given.end, error }; + } + + // Remove `[from, resume)` from the gap at `index`, returning its error. + #cutGap(items: Items, index: number, from: bigint, resume: bigint): Error { + const gap = items.gaps[index]; + const kept: DropRun[] = []; + if (gap.start < from) kept.push({ ...gap, end: from }); + if (gap.end > resume) kept.push({ ...gap, start: resume }); + items.gaps.splice(index, 1, ...kept); + return gap.error; + } + + // Record `[start, end)` as settled: handed out, or told as a gap. + #settled(start: bigint, end: bigint): void { + start = max(start, this.#floor); + if (start < end) this.#reported.set(start, end, { start, end, kind: "settled" }); + } + + // Fold the record below `at`, where a key-order read is, into the floor, held below every run + // still to report there: a run given up and not reported yet, and a gap found and not returned + // that the read may still reach. + // The record then holds only what lies ahead of the read, bounded however long it plays. A + // session-fed read loses nothing, since every key it passed was handed out or reported; a group + // a local track produces below the floor is told as nothing, where Rust, folding only under pool + // pressure, would report it Old. A wire publisher's record is what its wire was told, which it + // keeps. + #fold(at: bigint): void { + if (this.#wired) return; + const items = this.#items; + if (items) { + // A gap the bounds left out below where the read is can never come in key order, and an + // ordered read never goes back, so it is dropped rather than held for a widening. + if ((items.gaps[0]?.start ?? items.at) < items.at) items.gaps = removeRun(items.gaps, 0n, items.at); + at = min(at, items.at); + for (const gap of items.gaps) at = min(at, gap.start); + } + for (const run of this.#pending) if (run.end > this.#floor) at = min(at, run.start); + if (at <= this.#floor) return; + this.#floor = at; + this.#reported.set(0n, at); + } + + // The key cursor passes `[start, end)` to hand out the group at `end`. Each key there that the + // track owed and no group or drop accounts for was given up, whether the budget convicted it or + // the tail's grace folded it, so it is stale. A wire-fed copy's feed owes every key from where it + // starts, and a fill what this reader asked it for. Below the first group passed is where the + // read joined, which is no gap. + #passOwed(start: bigint, end: bigint, drift: Drift): void { + const producer = this.#state.producer; + if (start >= end || this.#nextKey === 0n || !this.#dropsSeen || !producer) return; + const owed = [...intersect(this.#asked, start, end), ...intersect(this.#givenUp, start, end)]; + if (wireFed(producer)) owed.push(...this.#window.runs(max(start, feedFloor(producer) ?? 0n), end)); + // Only a drop that settles this reader reports the key; a stale verdict under a tighter budget + // does not, so the key is still this reader's to give up. + const { drops, refused } = this.#state; + const coverage = this.#coverage(); + for (const run of owed) { + const settled = givenUp(drops, refused.peek(), run.start, run.end, (drop) => + settles(drop, drift.budget, coverage), + ).filter((drop) => settles({ ...drop, start: max(drop.start, run.start) }, drift.budget, coverage)); + for (const passed of runGaps(settled, run.start, run.end)) { + this.#giveUp(passed, staleHole(drift.budget)); + } + } + } + + // Keep a run the key cursor gave up, once the drop record started, so a read reports it. + #giveUp(run: Run, hole: Hole): void { + if (this.#dropsSeen) this.#pending = insertDrop(this.#pending, run.start, run.end, hole); + } + // A missing group the subscription still owes holds sequence order until it lands, // is accounted for, or the same drift budget as a buffered group gives up on it. #waitOwed(start: bigint, end: bigint, drift: Drift): boolean { @@ -4850,7 +5127,13 @@ export class Subscriber { const groups = this.#state.groups.peek(); const cursor = this.#cursor.peek(); let start = max(cursor.start, this.#nextKey); - while (groups.length > 0 && groups[0].start < start) groups.shift()?.close(); + // A group landing after the read passed its key is old: the read never goes back. + for (let group = groups[0]; group && group.start < start; group = groups[0]) { + groups.shift(); + group.close(); + // The record tells nothing twice, so a key already handed out or given up stays told. + this.#giveUp({ start: group.start, end: group.start + 1n }, late()); + } // One anchor for the whole pass, so walking a backlog off stays linear. const drift = this.#drift(); @@ -4866,6 +5149,9 @@ export class Subscriber { // Every frame this group could still hold is past the budget, so keep // scanning: one pass walks a whole backlog off rather than replaying it. if (this.#staleAt(group.start, drift)) { + // Passing it passes the owed keys below it too, which the budget gives up with it. + this.#passOwed(start, group.start, drift); + this.#giveUp({ start: group.start, end: group.start + 1n }, staleHole(drift.budget)); const passed = this.#nextKey; this.#pass(groups, index); // The cursor passed what lies below it, which a fill no longer owes this reader. @@ -4885,7 +5171,9 @@ export class Subscriber { this.#fillWait = true; return { kind: "idle" }; } - // Given up: past it, as past any group no fill can serve. + // Given up: past it, as past any group no fill can serve, and the owed keys below it. + this.#passOwed(start, group.start, drift); + this.#giveUp({ start: group.start, end: group.start + 1n }, headless()); const passed = this.#nextKey; this.#pass(groups, index); this.#stoppedReading(passed, this.#nextKey); @@ -4894,6 +5182,7 @@ export class Subscriber { group.close(); continue; } + this.#passOwed(start, group.start, drift); this.#pass(groups, index); this.#handOut(group.start, true); return { kind: "group", group: this.#guard(group) }; @@ -5031,7 +5320,8 @@ export class Subscriber { * Created by {@link Subscriber.ordered}, which takes the subscription over: the two * cursors draw from one buffer, so a track is read one way or the other and never both. * Every group this returns has a higher key than the last, so a late arrival is - * skipped rather than delivered out of turn. + * reported as a gap rather than delivered out of turn, as is every group the read gives up on + * (see {@link Ordered.nextGroup}). * * `delay.max` applies as this cursor reads, exactly as it does on the arrival cursor: a * group is skipped once its reach, where its immediate successor begins, is that far @@ -5097,7 +5387,8 @@ export class Ordered { /** * Limit subsequent reads to these groups and return this reader for chaining; see * {@link Subscriber.withRange}. An ordered read never goes back, so lowering the floor - * below a group it already returned offers nothing. + * below a group it already returned offers nothing, not even a gap it passed while the + * range left it out. */ withRange(groups: Groups): this { this.#subscriber.withRange(groups); @@ -5115,7 +5406,8 @@ export class Ordered { } /** - * Return the next group with a strictly-greater sequence number than the last returned. + * Return the next group with a strictly-greater sequence number than the last returned, + * passing any gap {@link nextItem} would report below it. Mirrors the Rust `next_group`. * * Late arrivals (key at or below the last returned) are silently skipped, as is a * group whose every frame is further behind the live edge than `delay.max` (the default @@ -5134,10 +5426,47 @@ export class Ordered { * `poll_next_group`. */ tryNextGroup(): GroupConsumer | undefined { - return taken( - () => ordered_.tryNextGroup(this.#subscriber), - () => ordered_.ended(this.#subscriber), - ); + for (;;) { + const item = this.tryNextItem(); + if (item?.kind !== "gap") return item?.group; + } + } + + /** + * The next group, as {@link nextGroup} returns it, or a gap this reader gave up on below it. + * Mirrors the Rust `next_item` and its `Item`. + * + * A gap is every group the publisher or a relay dropped, the transport evicted, `delay.max` + * aged out, a fill could not find, or that landed after the read passed its key. It carries + * the keys it spans and is returned once, after any group the read still owes below it and + * before the next group above it, or when the track ends. Keys the publisher declared skipped + * were never groups, so they are no gap, and nor is a key a local track never produced. + * Unlike Rust, nor is a group a local track produces after the read moved past its key: the + * read keeps no record below where it is, so its memory stays bounded. A session-fed track + * reports such a key at the latest when the read passes it. + */ + nextItem(): Promise { + return ordered_.nextItem(this.#subscriber); + } + + /** + * Take the next item as {@link nextItem} would, without waiting: `undefined` when nothing + * is ready right now, which is not by itself the end of the track. Mirrors the Rust + * `poll_next_item`. + */ + tryNextItem(): Item | undefined { + const recv = ordered_.tryItem(this.#subscriber); + switch (recv.kind) { + case "group": + case "gap": + return recv; + case "error": + throw recv.error; + case "done": + return ordered_.ended(this.#subscriber); + default: + return undefined; + } } /** diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 7b3663b69d1..59d657e8200 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -6,6 +6,15 @@ import type { Clock, Data, End, Init, Latency, Playing, Reading, Received, Reset // The stretch's 30 ms processing window, in microseconds. @internal export const WINDOW = 30_000; +/** + * How long an ordered reader waits for a missing group before giving it up as a gap: what the ring + * holds of the delay, plus how far the ring lets audio trail the clock before skipping it, never less + * than one stretch window. A group landing any later could no longer play. + */ +export function holeWait(held: Duration, bound: Duration): Duration { + return held.add(bound.max(Duration.micros(WINDOW))); +} + // The least move, in milliseconds, of how far the worklet renders ahead of what sounds that is reported. const LEAD_STEP = 5; diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 74f92a69bb8..353b9b88921 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1596,14 +1596,18 @@ async function livePlayback(burst: number) { let frame = 0; let sequence = 0n; let held: number[] = []; + let owed: Moq.Track.Requests | undefined; const ring = () => worklets.findLast((worklet) => worklet.ring && insert.mock.contexts.includes(worklet.ring)); return { sync, error: decoder.out.error, // Lose one whole group without changing the following media timestamps or arrival cadence. + // An open fill nobody serves stands in for a session's feed, which owes the group until the + // reader gives up on it. async hole() { if (held.length) throw new Error("a hole needs an empty delivery burst"); + owed ??= track.requests(); sequence++; frame++; await clock.advance(FRAME); @@ -1660,6 +1664,7 @@ async function livePlayback(burst: number) { counts.inserted = insert.mock.calls.reduce((sum, [, data]) => sum + (data as Float32Array[])[0].length, 0); }, async close() { + owed?.close(); decoder.close(); source.close(); sync.close(); @@ -1682,8 +1687,8 @@ it("a missing live group re-primes without clearing decoded PCM or resetting Syn const buffered = ring?.length; await playback.hole(); expect(ring?.length).toBe(buffered); - // A missing group is waited on for the subscription's max delay, as late audio could still - // arrive inside it, and skipped once the frames past it span that much. + // A missing group is waited on for what the ring holds, as late audio could still play inside + // it, and the ordered reader then reports the gap the decoder re-primes on. await playback.run(4000); expect(prime).toHaveBeenCalledTimes(1); expect(reset).not.toHaveBeenCalled(); @@ -2053,26 +2058,29 @@ it.each([false, true])("releases shared startup across a hole in queued audio (C }); // Audio may trail the clock by the stretch, or a buffered player's whole lookahead when that is longer, -// and a zero stretch turns the time stretch off in either mode. The stretch sizes nothing else: the -// subscription reaches the estimator's 2 s ceiling past the delay whatever it is. +// and a zero stretch turns the time stretch off in either mode. The subscription reaches the +// estimator's 2 s ceiling past the delay whatever the stretch is, while the ordered reader that +// delivers the audio waits a hole out for as long as the ring could still play it: the delay it holds +// plus that bound, never less than one stretch window. for (const buffered of [false, true]) { it(`bounds ${buffered ? "buffered" : "live"} audio's lag by the stretch alone`, async () => { - // The max delay each subscribe and update asked the publisher for, in order. - const asked: (Duration | null | undefined)[] = []; + // The max delay each subscription last asked for, in the order they opened. + const asked = new Map(); const subscribe = Moq.Track.Consumer.prototype.subscribe; const update = Moq.Track.Subscriber.prototype.update; const subscribes = spyOn(Moq.Track.Consumer.prototype, "subscribe").mockImplementation(function ( this: Moq.Track.Consumer, ...args: Parameters ) { - asked.push(args[0]?.delay?.max); - return subscribe.apply(this, args); + const subscriber = subscribe.apply(this, args); + asked.set(subscriber, args[0]?.delay?.max); + return subscriber; }); const updates = spyOn(Moq.Track.Subscriber.prototype, "update").mockImplementation(function ( this: Moq.Track.Subscriber, ...args: Parameters ) { - asked.push(args[0]?.delay?.max); + asked.set(this, args[0]?.delay?.max); return update.apply(this, args); }); const playback = await play(Duration.millis(100), buffered); @@ -2080,15 +2088,16 @@ for (const buffered of [false, true]) { await playback.play(12); const bound = () => worklets.at(-1)?.terms?.bound; expect(bound()).toBe(buffered ? 1000 : 200); - const reach = Duration.millis(2100); - expect(asked.at(-1)).toEqual(reach); + const budgets = (hole: number) => [Duration.millis(2100), Duration.millis(hole)]; + expect([...asked.values()].slice(-2)).toEqual(budgets(buffered ? 1100 : 300)); playback.stretch.set(Duration.millis(1500)); await microtasks(); expect(bound()).toBe(1500); + expect([...asked.values()].slice(-2)).toEqual(budgets(1600)); playback.stretch.set(Duration.ZERO); await microtasks(); expect(bound()).toBe(0); - expect(asked.at(-1)).toEqual(reach); + expect([...asked.values()].slice(-2)).toEqual(budgets(130)); } finally { subscribes.mockRestore(); updates.mockRestore(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 4112a4c5783..30041a8cb2a 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -19,7 +19,7 @@ import { base64ToBytes } from "../base64"; import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; import { DELAY_MAX, type Sync, type Track } from "../sync"; -import { AudioBuffer } from "./buffer"; +import { AudioBuffer, holeWait } from "./buffer"; import { type DecoderConfig, decoderDescription, @@ -31,7 +31,6 @@ import { import { Handover } from "./handover"; import { bound, maxDelay, ringHold, target } from "./latency"; import { Order } from "./order"; -import { playable } from "./playhead"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -136,7 +135,14 @@ const EMPTY: Stats = { }; // What a decode attempt reads, and where its counts go. -type Attempt = { config: DecoderConfig; failure: Signal; path: Moq.Path.Valid; key: Key }; +type Attempt = { + config: DecoderConfig; + failure: Signal; + path: Moq.Path.Valid; + key: Key; + // The same track read in key order, which decides every hole. + ordered: Moq.Track.Ordered; +}; // The newest container consumer, the path it reads, and its codec, whose estimate a replacement // continues. @@ -191,8 +197,6 @@ export class Decoder { // Audio ring bridging the main thread and the worklet. #ring = new Signal(undefined); - // How far ahead of the clock the ring renders, so what it renders sounds when due. - #output: Getter; // The context, worklet, and ring are keyed on this alone. It outlives a rendition's absence, so // the queued tail plays out and a return with the same shape reuses the graph. @@ -212,6 +216,9 @@ export class Decoder { // The subscription's min delay: the playout delay plus the warmup, where the publisher starts a // live join. #subscribeMinDelay = new Signal(Duration.ZERO); + // The ordered reader's max delay, how long it waits for a missing group before reporting a gap: + // as long as the ring could still play it, plus the warmup the subscription adds. + #readMaxDelay = new Signal(Duration.ZERO); // The container consumer's delay: the playout delay to start at and the shared budget. Its warmup // reaches the start further back, to the media the decoder primes on. #delay = new Signal({}); @@ -286,10 +293,6 @@ export class Decoder { complete, }; }); - this.#output = this.#signals.computed((effect) => { - const ring = effect.get(this.#ring); - return ring ? effect.get(ring.lead) : undefined; - }); this.#track = this.sync.track({ target: playout, startup }); this.#signals.cleanup(() => this.#track.close()); this.#identity = this.#signals.computed((effect) => { @@ -367,20 +370,25 @@ export class Decoder { // The subscription reaches the estimator's 2 s ceiling past the delay, whatever the stretch, since a // group the relay expires is never observed: a subscription cut to the target would leave the - // estimate rising one bucket at a time. The container - // consumer drops aged media at the same max delay, since it observes each frame before applying - // it, and carves out the warmup the subscription adds once it has warmed, as Rust's does. It waits - // on a missing group only for the delay, its min: past that the group would play late, and a - // reordered backlog's holes waited out to the max would read as jitter to the estimator. - // See doc/concept/audio-jitter.md. + // estimate rising one bucket at a time. The container consumer reads it only to observe every + // arrival, and drops aged media at the same max delay. + // + // The ordered reader delivering the audio decides each hole, giving up on a missing group once the + // ring could no longer play it: past what the ring holds of the delay and the stretch bound it still + // plays late audio within, which follow the delay, the stretch, and the output latency. Both carry + // the warmup, so the start reaches the media the decoder primes on. See doc/concept/audio-jitter.md. #runSubscribeMaxDelay(effect: Effect): void { const budget = this.#budget(effect); const delay = budget.delay; const max = maxDelay(budget); const warmup = Duration.millis(effect.get(this.source.out.config)?.warmup ?? 0).min(DELAY_MAX); + const ring = effect.get(this.#ring); + if (ring) effect.get(ring.output); + const held = ring ? ring.held(delay) : delay; this.#delay.set({ min: delay, max }); this.#subscribeMinDelay.set(delay.add(warmup)); this.#subscribeMaxDelay.set(max.add(warmup)); + this.#readMaxDelay.set(holeWait(held, bound(budget)).add(warmup)); } // The shared delay and max delay, and the stretch threshold the ring skips past. @@ -654,14 +662,22 @@ export class Decoder { priority: Catalog.PRIORITY.audio, delay: new Derived([this.#subscribeMinDelay, this.#subscribeMaxDelay], (min, max) => ({ min, max })), }); - if (!sub) { + // A second subscription to the same track, read in key order, delivers the audio; `sub` + // only feeds the arrival estimate. Both share one upstream request, at the wider budget. + const read = subscribeMedia(run, { + broadcast: active, + track: Catalog.track(track, identity), + priority: Catalog.PRIORITY.audio, + delay: new Derived([this.#subscribeMinDelay, this.#readMaxDelay], (min, max) => ({ min, max })), + }); + if (!sub || !read) { this.#retire(); return; } // A config the probe accepted can still fail to parse (a malformed init segment or Opus // header). Report it like a decode failure, which also retires the outgoing estimate. - const props: Attempt = { config, failure, path: active.path, key: rendition }; + const props: Attempt = { config, failure, path: active.path, key: rendition, ordered: read.ordered() }; spawnFatal(run, failure, async () => { if (config.container.kind === "cmaf") await this.#runCmafDecoder(run, sub, props); else this.#runLegacyDecoder(run, sub, props); @@ -669,7 +685,11 @@ export class Decoder { }); } - #runLegacyDecoder(effect: Effect, sub: Moq.Track.Subscriber, { config, failure, path, key }: Attempt): void { + #runLegacyDecoder( + effect: Effect, + sub: Moq.Track.Subscriber, + { config, failure, path, key, ordered }: Attempt, + ): void { const preSkip = config.codec === "opus" && config.description ? Util.Opus.preSkip(Util.Hex.toBytes(config.description)) : 0; this.#terminal.clear( @@ -682,7 +702,7 @@ export class Decoder { config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); // Create consumer with slightly less latency than the render worklet to avoid underflowing. // TODO include JITTER_UNDERHEAD - const { consumer, read } = this.#consume(effect, sub, format, { config, path }); + const { consumer, read } = this.#consume(effect, sub, format, { config, path, ordered }); spawnFatal(effect, failure, async () => { const loaded = await Util.Libav.polyfill(hosted.url); @@ -752,7 +772,7 @@ export class Decoder { async #runCmafDecoder( effect: Effect, sub: Moq.Track.Subscriber, - { config, failure, path, key }: Attempt, + { config, failure, path, key, ordered }: Attempt, ): Promise { if (config.container.kind !== "cmaf") return; // just to help typescript @@ -776,7 +796,7 @@ export class Decoder { config.description ? Util.Hex.toBytes(config.description) : init.description, ); - const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); + const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path, ordered }); spawnFatal(effect, failure, async () => { const loaded = await Util.Libav.polyfill(hosted.url); @@ -935,14 +955,15 @@ export class Decoder { })); } - // Reorders groups and frames up to the shared max delay. A rename is not a cold start: the path's - // arrivals are what they were, so the estimate continues from whatever read it before. Another - // path's arrivals have nothing to do with what this one learned. + // Reads `ordered` in key order, playing each gap it reports as the hole, and observes every arrival + // on `sub`. A rename is not a cold start: the path's arrivals are what they were, so the estimate + // continues from whatever read it before. Another path's arrivals have nothing to do with what + // this one learned. #consume( effect: Effect, sub: Moq.Track.Subscriber, format: Container.Format, - { config, path }: { config: DecoderConfig; path: Moq.Path.Valid }, + { config, path, ordered }: { config: DecoderConfig; path: Moq.Path.Valid; ordered: Moq.Track.Ordered }, ): { consumer: Container.Consumer; read: Signal } { const phase = this.#phase.peek(); if (phase.kind !== "running") throw new Error("audio: a consumer opened outside a running rendition"); @@ -953,7 +974,7 @@ export class Decoder { delay: this.#delay, warmup: Duration.millis(config.warmup ?? 0).min(DELAY_MAX), predecessor: continues ? outgoing.consumer : undefined, - playhead: playable(this.sync, this.#output), + ordered, }); effect.cleanup(() => consumer.close()); this.#phase.set({ ...phase, consumer: { path, codec: config.codec, consumer } }); diff --git a/js/watch/src/audio/playhead.test.ts b/js/watch/src/audio/playhead.test.ts deleted file mode 100644 index 4a3f1048e18..00000000000 --- a/js/watch/src/audio/playhead.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { expect, it } from "bun:test"; -import { Duration } from "@moq/clock"; -import { Manual } from "@moq/clock/test"; -import { Signal } from "@moq/signals"; -import { Sync } from "../sync"; -import { playable } from "./playhead"; - -it("reads the clock the ring renders at, less the stretch it plays late audio within", async () => { - const clock = new Manual(); - const installed = clock.install(); - const sync = new Sync({ delay: Duration.millis(300), stretch: Duration.millis(200) }); - const span = new Signal<{ start: Duration; end: Duration } | null>(null); - const audio = sync.track({ target: new Signal(Duration.millis(300)), startup: span }); - const output = new Signal(Duration.millis(40)); - const playhead = playable(sync, output); - try { - await clock.advance(Duration.ZERO); - audio.received(Duration.ZERO); - span.set({ start: Duration.ZERO, end: Duration.millis(300) }); - await clock.advance(Duration.millis(1000)); - const now = sync.now(); - if (now === undefined) throw new Error("the clock never started"); - expect(playhead.now()).toEqual(now.add(Duration.millis(40)).saturatingSub(Duration.millis(200))); - - // The trail moving is a jump, which wakes a consumer waiting on a hole. - let jumped = false; - const dispose = playhead.anchor.subscribe(() => { - jumped = true; - }); - await clock.advance(Duration.ZERO); - jumped = false; - output.set(Duration.millis(60)); - await clock.advance(Duration.ZERO); - expect(jumped).toBe(true); - expect(playhead.now()).toEqual(now.add(Duration.millis(60)).saturatingSub(Duration.millis(200))); - dispose(); - } finally { - audio.close(); - sync.close(); - installed.close(); - } -}); - -it("applies the output latency before media zero's turn, when the ring already renders it", async () => { - const clock = new Manual(); - const installed = clock.install(); - const sync = new Sync({ delay: Duration.millis(300), lookahead: Duration.millis(1000), stretch: Duration.ZERO }); - const span = new Signal<{ start: Duration; end: Duration; lead?: Duration } | null>(null); - const audio = sync.track({ target: new Signal(Duration.millis(300)), startup: span }); - const playhead = playable(sync, new Signal(Duration.millis(120))); - try { - await clock.advance(Duration.ZERO); - audio.received(Duration.ZERO); - span.set({ start: Duration.ZERO, end: Duration.millis(180), lead: Duration.millis(120) }); - await clock.advance(Duration.millis(50)); - // 70 ms short of media zero's turn, which the clock reads as zero, the ring renders 50 ms, and - // plays audio trailing it by its 30 ms window even with the stretch off. - expect(sync.now()).toEqual(Duration.ZERO); - expect(sync.position()).toBe(-70); - expect(playhead.now()).toEqual(Duration.millis(20)); - } finally { - audio.close(); - sync.close(); - installed.close(); - } -}); diff --git a/js/watch/src/audio/playhead.ts b/js/watch/src/audio/playhead.ts deleted file mode 100644 index 900233d0fe8..00000000000 --- a/js/watch/src/audio/playhead.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Duration } from "@moq/clock"; -import type * as Container from "@moq/hang/container"; -import { Derived, type Getter, Signal } from "@moq/signals"; -import type { Sync } from "../sync"; -import { WINDOW } from "./buffer"; -import { bound } from "./latency"; - -/** - * The oldest media the ring still plays, which the audio consumer reads as its clock. The ring renders - * `output` (none until known) ahead of the shared clock, so what it renders now sounds when the clock - * reaches it, and stretches audio trailing that within its {@link bound} back onto it, at least one stretch window. So a missing - * group can still play until the group past it trails the ring by more than the bound, as when a late - * burst fills the hole a moment after the group past it lands. - * - * @internal Exported for the replay harness. - */ -export function playable(sync: Sync, output: Getter = new Signal(undefined)): Container.Playhead { - // How far the oldest playable media trails the clock, in ms; negative when the output leads by more. - const trail = new Derived( - [sync.out.delay, sync.out.maxDelay, sync.in.stretch, output], - (delay, maxDelay, stretch, output) => - // The ring skips only past its bound, never less than one stretch window, even with the stretch off. - Math.max(bound({ delay, maxDelay, stretch }).asMillis(), WINDOW / 1000) - (output?.asMillis() ?? 0), - ); - return { - now: () => { - // The signed position, since before media zero's turn the ring may already render it. - const position = sync.position(); - if (position === undefined) return undefined; - return Duration.millis(Math.max(0, position - trail.peek())); - }, - anchor: new Derived([sync.anchor, trail], (anchor, trail) => ({ anchor, trail })), - }; -} diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index b40a259bd26..6d9b6d3096f 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -1,8 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; import { Duration } from "@moq/clock"; import type * as Catalog from "@moq/hang/catalog"; -import { Track } from "@moq/net"; -import { type Arrival, answering, type Options, QUANTUM, type Quantum, replay } from "./replay"; +import { type Arrival, type Options, QUANTUM, type Quantum, replay } from "./replay"; const RATE = 48000; const CONFIG = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; @@ -71,9 +70,10 @@ describe("replay", () => { }); it("counts each run-dry the way the player's stats do", async () => { - // One frame 200 ms late against a 100 ms target: the ring runs dry once and re-stalls to refill, - // which counts the short quantum it ran dry in and the stall after it. - const trace = paced(10, (i) => (i === 250 ? 230 : 30)).sort((a, b) => a.at - b.at); + // Ten frames held and released together, the first 200 ms late against a 100 ms target: the ring + // runs dry once and re-stalls to refill, which counts the short quantum it ran dry in and the + // stall after it. + const trace = paced(10, (i) => (i >= 250 && i < 260 ? 30 + (260 - i) * 20 : 30)).sort((a, b) => a.at - b.at); const counted = async (arrivals: Arrival[]) => { let last = 0; for await (const { underruns } of replay(arrivals, { @@ -90,19 +90,17 @@ describe("replay", () => { expect(await counted(trace)).toBeLessThanOrEqual(2); }); - it("holds newer groups behind a missing one until the max delay gives up on it", async () => { + it("holds newer groups behind a missing one until the ring could no longer play it", async () => { const trace = paced(10, () => 30).filter((arrival) => arrival.group !== 250); - // The consumer delivers in group order, so the ring runs dry for longer than the missing frame - // while the groups behind it wait. Writing arrivals straight into the ring plays straight through. - const frame = Math.ceil((20 / 1000) * (RATE / QUANTUM)); - expect( - await underruns(trace, { - rate: RATE, - config: CONFIG, - delay: Duration.millis(100), - duration: observed(10), - }), - ).toBeGreaterThan(frame); + // The consumer delivers in group order, so the ring runs dry while the groups behind the missing + // one wait, and the ordered reader gives up on it as the ring's stretch bound passes it. + const run = await underruns(trace, { + rate: RATE, + config: CONFIG, + delay: Duration.millis(100), + duration: observed(10), + }); + expect(run).toBeGreaterThan(0); expect(warn).toHaveBeenCalled(); }); @@ -449,23 +447,3 @@ describe("buffered replay against the shared clock", () => { expect(warn).not.toHaveBeenCalled(); }); }); - -describe("answering", () => { - it("names the publisher's largest group first, then each newer group, and delegates the rest", async () => { - const track = new Track.Producer("audio"); - const subscriber = answering(track.subscribe(), 5n); - try { - // A group at or below the answer is no news: the track's own largest is older than it. - track.createGroup(3n); - expect(await subscriber.largest()).toEqual({ group: 5n, frame: 0 }); - const next = subscriber.largest(); - track.createGroup(5n); - track.createGroup(9n); - expect((await next)?.group).toBe(9n); - expect(subscriber.name).toBe("audio"); - } finally { - subscriber.close(); - track.close(); - } - }); -}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index c059501b774..b6b484f9238 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -2,8 +2,8 @@ * Replay a recorded arrival trace through the player's consumer and ring on a simulated clock. * * Every frame is written into its group at the instant the trace says it arrived, and the player's - * real `Container.Consumer` decides what to deliver, wait for, and skip, at the hole wait, max delay, and - * clock the decoder gives it. Delivered frames go into the real ring, which carries on through a gap the way the decoder's + * real `Container.Consumer` decides what to deliver and skip, at the max delay the decoder gives it, + * reading the ordered reader that decides each hole at the budget the decoder gives that. Delivered frames go into the real ring, which carries on through a gap the way the decoder's * does: a gap restarts only the codec. One render quantum is read every quantum's worth of that same * clock, which is what the AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder * registers, measured by that same consumer from the recorded catalog config, and the ring and the @@ -22,11 +22,10 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { type Group, Time, Track } from "@moq/net"; import { Derived, Effect, Signal } from "@moq/signals"; -import { type Delay, STRETCH, Sync } from "../sync"; -import { Backpressure } from "./buffer"; +import { DELAY_MAX, type Delay, STRETCH, Sync } from "../sync"; +import { Backpressure, holeWait } from "./buffer"; import { frameDuration } from "./config"; import { maxDelay as audioMaxDelay, bound, ringHold, target } from "./latency"; -import { playable } from "./playhead"; import { AudioRingBuffer } from "./ring-buffer"; /** An AudioWorklet render quantum, in frames. */ @@ -66,12 +65,6 @@ export interface Options { lookahead?: number; /** The configured stretch, in ms, as the element takes it (default: the element's). */ stretch?: number; - /** - * The publisher's largest group as the subscription began, which the consumer learns as a wire - * subscriber learns it from the publisher's answer. Omitted, the replay's track names its own: the - * first group to land. - */ - largest?: number; /** The rendition as the catalog described it: its advertised jitter, delay, and codec. */ config: Catalog.AudioConfig; /** @@ -184,33 +177,6 @@ const micros = (ms: number) => Math.round(ms * 1000); */ const settle = () => new Promise((resolve) => setImmediate(resolve)); -/** - * `track` answering `largest` the way a wire subscriber does once the publisher has answered: first - * the publisher's largest group as the subscription began, then each newer group as it lands. - * Everything else is the local subscriber's. - */ -export function answering(track: Track.Subscriber, largest: bigint): Track.Subscriber { - let answered = false; - return new Proxy(track, { - get(target, prop) { - if (prop !== "largest") { - const value = Reflect.get(target, prop, target); - return typeof value === "function" ? value.bind(target) : value; - } - return async () => { - if (!answered) { - answered = true; - return { group: largest, frame: 0 }; - } - for (;;) { - const next = await target.largest(); - if (!next || next.group > largest) return next; - } - }; - }, - }); -} - /** * Play `trace` in simulated real time, yielding every render quantum from the first arrival to the * end of the observation. @@ -252,15 +218,16 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const maxDelay = new Derived([sync.out.delay, sync.out.maxDelay], (delay, max) => audioMaxDelay({ delay, maxDelay: max }), ); + // Fills nobody serves stand in for the recorder's feed: the ordered reader waits for a missing + // group as it would for a session's, until the group lands or its budget gives up on it. + const requests = track.requests(); const subscriber = track.subscribe({ delay: { max: WIRE_MAX_DELAY } }); - const consumer = new Container.Consumer( - options.largest === undefined ? subscriber : answering(subscriber, BigInt(options.largest)), - { - format: new Container.Legacy.Format("audio"), - delay: new Derived([sync.out.delay, maxDelay], (min, max) => ({ min, max })), - playhead: playable(sync), - }, - ); + const ordered = track.subscribe({ delay: { min: sync.out.delay.peek() } }).ordered(); + const consumer = new Container.Consumer(subscriber, { + format: new Container.Legacy.Format("audio"), + delay: new Derived([sync.out.delay, maxDelay], (min, max) => ({ min, max })), + ordered, + }); // Match the decoder: a codec constant until a delivered packet states its own duration. const signals = new Effect(); @@ -287,6 +254,15 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato : undefined; const audio = sync.track({ target: playout, startup }); signals.cleanup(() => audio.close()); + // The ordered reader waits a hole out as the decoder's does, the warmup included. A replay's ring + // holds the whole delay: it has no output. + const warmup = Duration.millis(config.warmup ?? 0).min(DELAY_MAX); + signals.run((effect) => { + const delay = effect.get(sync.out.delay); + const max = holeWait(delay, bound({ delay, maxDelay: effect.get(sync.out.maxDelay), stretch })); + ordered.update({ delay: { min: delay.add(warmup), max: max.add(warmup) } }); + }); + signals.cleanup(() => requests.close()); await settle(); let delay = sync.out.delay.peek(); diff --git a/quest/m2/audio-playout/browser-audio-hole-skip.md b/quest/m2/audio-playout/browser-audio-hole-skip.md index 94a78d40555..55d07dfeccb 100644 --- a/quest/m2/audio-playout/browser-audio-hole-skip.md +++ b/quest/m2/audio-playout/browser-audio-hole-skip.md @@ -1,71 +1,47 @@ -# [L] Browser audio takes its holes from the ordered reader +# [M] Browser audio's ordered reader grades no worse than before #8799 ## Goal -`@moq/watch`'s audio reads its track through `@moq/net`'s ordered reader with -a reader budget equal to what the ring holds, and plays each `Gap` the reader -reports as the hole: the audio container consumer keeps no hole wait of its -own and no longer passes `playhead`. Video keeps the playhead. The Chromium -`bursty` and `congestion` rows of `just test audio-quality` grade no worse -than before [moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799) -gave the audio consumer the player's clock, and the Firefox `after resume` -window that PR fixed stays audible. +The Chromium `bursty` and `congestion` rows of `just test audio-quality`, with +browser audio reading its holes from `@moq/net`'s ordered reader, grade no +worse than before [moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799) +gave the audio consumer the player's clock, measured on the nightly runner, +and the one full starvation seen locally is explained or fixed. ## Plan Decided 2026-11-16 (maintainer, /quest-run): ✅ the transport decides an audio hole, through an ordered reader whose budget is the ring's size (rejected: first measure the playhead rule against a hand-written wait in -the container consumer, as this quest planned before; keep the playhead rule -and the separate wait). - -Interim rule, decided 2026-11-17 (maintainer, /quest-run, -[moq-dev/moq#8982](https://github.com/moq-dev/moq/pull/8982)): ✅ browser -audio ends a hole once the group past it trails the clock by the ring's -stretch bound (`playable(sync, output)` in `js/watch/src/audio/playhead.ts`), -the rule #8982 ships (rejected: main's bare clock with only the replay harness -fixed and re-budgeted; the estimate wait, which brought back the Firefox resume -silence). It is the only option with no measured regression in the browser, -and native audio keeps its estimate wait until -[moq-dev/moq#8984](https://github.com/moq-dev/moq/pull/8984) settles one rule. -The work below replaces `playable` with the ordered reader's `Gap`, so -`playable` and the audio consumer's clock go with it. - -Why the playhead rule goes: natively, ending an audio hole once the clock -reaches the group past it took `bursty` from 3 to 155 underruns a minute -(`just test audio-quality --runtimes native --codecs opus --duration 40 ---seed 7`). A burst's groups land late together, so the group past a hole is -already due when the missing one lands a moment later in the same burst. -Native audio instead waits a hole out for its playout estimate from the -arrival of the group past it (`rs/moq-audio/src/decode/consumer.rs`), and -[moq-dev/moq#8984](https://github.com/moq-dev/moq/pull/8984) makes that wait -the subscription's `Delay::min`. Rust `track::Ordered` -([moq-dev/moq#8910](https://github.com/moq-dev/moq/pull/8910)) gives up on a -missing group by the same two clocks: the live edge's timestamp a budget past -it, or its successor's arrival a budget ago. So the rule lives once, in the -transport. +the container consumer; keep the playhead rule and the separate wait). + +[moq-dev/moq#9068](https://github.com/moq-dev/moq/pull/9068) moved browser +audio onto the ordered reader: `Ordered.nextItem` returns a gap for each +half-open range of keys it gives up on, audio reads through a second +subscription whose budget is what the ring holds plus the stretch bound and +the warmup, and the audio container consumer plays each gap as its hole. Firefox's +`after resume` window stays audible. + +Its local Chromium measurement, three runs each of the PR, its merge base, +and `cf23b5ae04~1` (before #8799), could not grade the rows: the machine sat +at a load average of 30 to 70, and render underruns reached 580 a minute on +the merge base. The player-side numbers were level with the merge base on +`congestion`, which also trimmed about half the audio it did before #8799. +On `bursty` they were mixed, and one Opus `bursty` run of seven starved for +its last 7.5 s: the playhead held still while the render clock ran, which no +merge-base or pre-#8799 run did, and four reruns did not repeat it. Work: -- Mirror `track::Ordered`'s `Item::{Group, Gap { range, error }}` in - `@moq/net`, whose `Ordered` handle (`js/net/src/track.ts`) skips a stale or - dropped group silently today, with the reader's own budget as in Rust. - This is #8910's follow-up "mirror `Item` in `@moq/net` once a consumer - needs it"; the bindings follow when a binding consumer needs it. -- The budget is what the ring holds (`ring.held(delay)` in - `js/watch/src/audio/decoder.ts`), and follows it as the delay estimate, - stretch, and output latency move. It is the reader's, not the - subscription's: the subscription keeps reaching the estimator's 2 s - ceiling past the delay, since a group the relay expires is never observed - and the estimate could not rise after underruns. -- The audio container consumer (`js/hang/src/container/consumer.ts`, used - from `js/watch/src/audio/decoder.ts` and `replay.ts`) reads ordered for - audio, turns each `Gap` into its hole, and drops its own hole wait and the - `playhead` it is passed. -- Measure the Chromium `bursty` and `congestion` rows before and after, a few - runs each, since `congestion` is noisy. Confirm the resume hole with - #8799's repro, `taskset -c 4-5 nix develop --command just test media - --media-engines firefox --media-cases rename,pause`. +- Grade the Chromium `bursty` and `congestion` rows from the nightly + runner's `audio-quality-run` artifacts after #9068 lands, against the + nights before it and before #8799. +- Find what starved that run. The audio container consumer + (`js/hang/src/container/consumer.ts`) in ordered mode has nothing that + passes an active group that opened but never delivers a frame until the + max buffer fills, which #8799's playhead covered. A trace recorded with + `--trace` holds only the frames that arrived, so replaying such a group + also needs the trace to record when each group opens. ## Related diff --git a/quest/m2/outcome-enums/ordered-gaps.md b/quest/m2/outcome-enums/ordered-gaps.md index 62f552e93a8..6c016aabab4 100644 --- a/quest/m2/outcome-enums/ordered-gaps.md +++ b/quest/m2/outcome-enums/ordered-gaps.md @@ -26,7 +26,8 @@ points the container reader at `next_item`. Public API: breaking in moq-net; callers of `next_group` that matched `Item` drop the match, and the gap-reading one calls `next_item`. No JS or binding -change: js/net's ordered reads and moq-ffi already hide gaps. Wire: none. +change: js/net's `Ordered` already has `nextGroup` and `nextItem`, and moq-ffi +hides gaps. Wire: none. ## Related diff --git a/quest/m2/unused-api/net.md b/quest/m2/unused-api/net.md index c93775ba40d..fafd832ccc3 100644 --- a/quest/m2/unused-api/net.md +++ b/quest/m2/unused-api/net.md @@ -82,5 +82,4 @@ Wire: none. ## Related - [Ordered gaps](/quest/m2/outcome-enums/ordered-gaps.md) - splits `Ordered` into `next_group` and the `next_item` this reader uses - -- [Browser audio holes](/quest/m2/audio-playout/browser-audio-hole-skip.md) - the browser counterpart of the reader moving onto `Gap` +- [Browser audio holes](/quest/m2/audio-playout/browser-audio-hole-skip.md) - grading the browser counterpart, which already reads `Gap`s diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index ea75e7544a5..83b97d56642 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -226,8 +226,9 @@ receives it, with the same transport, container, and codec-duration readers over on the delay, so the replay decides those again. `replay.ts` plays a trace through the player's own `Container.Consumer` and ring on a simulated -clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), whose consumer is told -the trace's `largest` as a wire subscriber is told by the publisher, at the "auto" delay a +clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), whose consumer reads +the groups through an ordered reader and observes every arrival through a second subscription, as +the browser player does, at the "auto" delay a real `Sync` resolves from the playout target that consumer measures, and reads each quantum through the tap's classifier. The consumer makes the player's group ordering, max delay skips, and discontinuity resets again at that delay, and the ring follows the delay as it moves; a group's stream is taken to finish with its last recorded frame. Every @@ -256,8 +257,8 @@ microphone, instead of publishing Chromium's fake capture device. Name the brows the trace's `source` before checking it in. Version 4 adds `largest`, the publisher's largest group as the subscription began, which a wire -subscriber learns from the publisher's answer and the replay's consumer needs to tell a group it -skips unread as the join's backlog. The recorder and the `--capture` observer read it the way +subscriber learns from the publisher's answer. The replay no longer reads it: its consumer observes +every arrival, the join's backlog included, through its second subscription. The recorder and the `--capture` observer read it the way `js/hang`'s consumer does: once the track's info or first group shows the subscription established and its start resolved, so a cached group racing the answer, or live groups landing after it, do not set it. No earlier recording kept the answer, and re-recording needs a person at the microphone and diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 445d4a410fc..19fdb4937c9 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows. Also on 2026-11-13, the estimator began counting one outage, the intervals its stalled frames span and the empty ones before them, as one observation, and every replay row was re-measured on top of the snap: only relay-mic-stall moved. Its single stall no longer raises the target, so its target p95 fell from 1540 to 80 ms and the trail the outage leaves is skipped onto the clock at once rather than through the target's falls: 2 skips a minute where it took 6.1, 5122.3 trimmed ms a minute where it trimmed 4678.2, and its A/V lag p95 from 118.6 to 19.7 ms. With 80 ms of buffer rather than 1540, more of the outage plays as silence: its stalled share rose from 0.067 to 0.092 and silence from 0.085 to 0.092. On 2026-11-14 every player began following the clock by its lag alone: a continuous WSOLA time stretch for any content, at a rate proportional to the lag and capped at 10%, which leaves a lag within half a 10 ms hop alone, and slows down while the audio received ahead of the playhead is about to run out, replacing NetEq's buffer-level decision and pitch-period splices. Every replay row was re-measured. Audio now rests within half a hop of the clock rather than within a pitch period: the A/V lag p50 fell from 10 to 0.2 ms on bursty-aac, whose p95 fell from 49.2 to 0.2, from 1.7 to -1.3 on local-aac, 2.2 to 1.8 on relay-mic and 0.9 to 0.6 on relay-mic-rtt40, and moved from -2 to 2.7 on relay-bbb and 2.2 to 3.1 on relay-mic-stall, each within the half hop the rule leaves alone. The p95 rose from 3.6 to 6.2 ms on relay-bbb, 4.5 to 6.9 on relay-mic, 2.9 to 6.9 on relay-mic-rtt40 and 19.7 to 32.1 on relay-mic-stall, and fell from 15.1 to 14.3 on local-aac: a lag under a hop is corrected only once its pace adds up to one. player_underruns_per_min rose from 4.4 to 6.6 on bursty-aac and 4 to 8.1 on local-aac, whose recorded outages outlast what slowing down at 10% bridges, and fell from 3.1 to 1 on relay-mic-rtt40, whose tap's underruns fell from 0.7 to 0.3 a minute; bursty-aac's silence rose from 0.007 to 0.008. Nothing more is trimmed on any row. The browser and native rows were measured on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row) against main the same day and left to the nightly runner: every browser row's A/V lag p50 now sits between 27 and 39 ms, where the fixed-250 control does on that host, where wide and bursty sat up to 58 ms behind, and every native row's within 15 ms of the clock. Also on 2026-11-14 the estimator stopped folding the intervals a subscription's join spans, those holding a frame due before its backlog finished arriving, and every replay row was re-measured: only relay-mic and relay-mic-stall moved. Their first frame arrives slower than the path after it, so their first interval folds nothing and the target leaves the prior 500 ms later, which moves where audio rests within the half hop the rule leaves alone: relay-mic's A/V lag p50 from 1.8 to -0.3 ms and p95 from 6.9 to 9, and relay-mic-stall's p50 from 3.1 to 1.7. relay-mic-rtt40 did not move: the browser consumer closes its join's stale groups below the cursor unread, so the estimator never sees when that backlog ended, and its target p95 held at 320 ms. Later on 2026-11-14 both container consumers began reporting when each group they close unread arrived, if it is at or below the publisher's largest group as the subscription began, which extends a join still open. On 2026-11-15 the traces gained that largest group (version 4), so the replay's consumer learns it as a wire subscriber does, and relay-mic-rtt40 was re-measured: its target p95 fell from 320 to 120 ms, underruns per minute rose from 0.3 to 0.7, player skips per minute fell from 0.3 to 0, trimmed ms per minute from 83 to 0, A/V lag p50 rose from 0.6 to 2.3 ms and p95 from 6.9 to 8.6, the trade of reporting the join's backlog. No other replay row moved.", + "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows. Also on 2026-11-13, the estimator began counting one outage, the intervals its stalled frames span and the empty ones before them, as one observation, and every replay row was re-measured on top of the snap: only relay-mic-stall moved. Its single stall no longer raises the target, so its target p95 fell from 1540 to 80 ms and the trail the outage leaves is skipped onto the clock at once rather than through the target's falls: 2 skips a minute where it took 6.1, 5122.3 trimmed ms a minute where it trimmed 4678.2, and its A/V lag p95 from 118.6 to 19.7 ms. With 80 ms of buffer rather than 1540, more of the outage plays as silence: its stalled share rose from 0.067 to 0.092 and silence from 0.085 to 0.092. On 2026-11-14 every player began following the clock by its lag alone: a continuous WSOLA time stretch for any content, at a rate proportional to the lag and capped at 10%, which leaves a lag within half a 10 ms hop alone, and slows down while the audio received ahead of the playhead is about to run out, replacing NetEq's buffer-level decision and pitch-period splices. Every replay row was re-measured. Audio now rests within half a hop of the clock rather than within a pitch period: the A/V lag p50 fell from 10 to 0.2 ms on bursty-aac, whose p95 fell from 49.2 to 0.2, from 1.7 to -1.3 on local-aac, 2.2 to 1.8 on relay-mic and 0.9 to 0.6 on relay-mic-rtt40, and moved from -2 to 2.7 on relay-bbb and 2.2 to 3.1 on relay-mic-stall, each within the half hop the rule leaves alone. The p95 rose from 3.6 to 6.2 ms on relay-bbb, 4.5 to 6.9 on relay-mic, 2.9 to 6.9 on relay-mic-rtt40 and 19.7 to 32.1 on relay-mic-stall, and fell from 15.1 to 14.3 on local-aac: a lag under a hop is corrected only once its pace adds up to one. player_underruns_per_min rose from 4.4 to 6.6 on bursty-aac and 4 to 8.1 on local-aac, whose recorded outages outlast what slowing down at 10% bridges, and fell from 3.1 to 1 on relay-mic-rtt40, whose tap's underruns fell from 0.7 to 0.3 a minute; bursty-aac's silence rose from 0.007 to 0.008. Nothing more is trimmed on any row. The browser and native rows were measured on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row) against main the same day and left to the nightly runner: every browser row's A/V lag p50 now sits between 27 and 39 ms, where the fixed-250 control does on that host, where wide and bursty sat up to 58 ms behind, and every native row's within 15 ms of the clock. Also on 2026-11-14 the estimator stopped folding the intervals a subscription's join spans, those holding a frame due before its backlog finished arriving, and every replay row was re-measured: only relay-mic and relay-mic-stall moved. Their first frame arrives slower than the path after it, so their first interval folds nothing and the target leaves the prior 500 ms later, which moves where audio rests within the half hop the rule leaves alone: relay-mic's A/V lag p50 from 1.8 to -0.3 ms and p95 from 6.9 to 9, and relay-mic-stall's p50 from 3.1 to 1.7. relay-mic-rtt40 did not move: the browser consumer closes its join's stale groups below the cursor unread, so the estimator never sees when that backlog ended, and its target p95 held at 320 ms. Later on 2026-11-14 both container consumers began reporting when each group they close unread arrived, if it is at or below the publisher's largest group as the subscription began, which extends a join still open. On 2026-11-15 the traces gained that largest group (version 4), so the replay's consumer learns it as a wire subscriber does, and relay-mic-rtt40 was re-measured: its target p95 fell from 320 to 120 ms, underruns per minute rose from 0.3 to 0.7, player skips per minute fell from 0.3 to 0, trimmed ms per minute from 83 to 0, A/V lag p50 rose from 0.6 to 2.3 ms and p95 from 6.9 to 8.6, the trade of reporting the join's backlog. No other replay row moved. On 2026-11-17 the browser player began reading audio through the ordered reader, which waits for a hole as long as the ring could still play it and reports each half-open range of keys it gives up on as one gap, and the replay dropped the trace's largest group: a second subscription now observes every arrival for the estimate, the join's backlog included. Every replay row was re-measured. bursty-aac's player underruns per minute fell from 6.6 to 4.4 and silence share from 0.008 to 0.007; audio rests at 0.1 ms until a stall at 19.5 s, from which the stretch catches up to 4.2 ms behind the clock, inside the half hop it leaves alone, so its A/V lag p50 and p95 rose from 0.2 to 4.2 ms. relay-bbb's A/V lag p50 fell from 2.7 to 1.4 ms and p95 rose from 6.2 to 7.6. relay-mic-rtt40's underruns per minute fell from 0.7 to 0.3, as did its player underruns per minute, underrun ms per minute from 0.8 to 0.2, A/V lag p50 from 2.3 to 0.6 ms, and p95 rose from 8.6 to 9.7, its target p95 holding at 120 ms; 0.6 ms of that rise is the replay's ordered reader taking the catalog's warmup into its budget, as the browser's does. No other replay row moved.", "rows": [ { "runtime": "chromium", @@ -426,16 +426,16 @@ "codec": "aac", "rate": 44100, "profile": "bursty-aac", - "player_underruns_per_min": 6.6, + "player_underruns_per_min": 4.4, "player_skips_per_min": 1.1, "underruns_per_min": 0, "underrun_ms_per_min": 0, "trimmed_ms_per_min": 312.7, "stalled_share": 0.005, - "silence_share": 0.008, + "silence_share": 0.007, "target_ms_p95": 343.2, - "av_lag_ms_p50": 0.2, - "av_lag_ms_p95": 0.2 + "av_lag_ms_p50": 4.2, + "av_lag_ms_p95": 4.2 }, { "runtime": "replay", @@ -466,8 +466,8 @@ "stalled_share": 0, "silence_share": 0, "target_ms_p95": 223.2, - "av_lag_ms_p50": 2.7, - "av_lag_ms_p95": 6.2 + "av_lag_ms_p50": 1.4, + "av_lag_ms_p95": 7.6 }, { "runtime": "replay", @@ -490,16 +490,16 @@ "codec": "opus", "rate": 48000, "profile": "relay-mic-rtt40", - "player_underruns_per_min": 0.7, + "player_underruns_per_min": 0.3, "player_skips_per_min": 0, - "underruns_per_min": 0.7, - "underrun_ms_per_min": 0.8, + "underruns_per_min": 0.3, + "underrun_ms_per_min": 0.2, "trimmed_ms_per_min": 0, "stalled_share": 0, "silence_share": 0, "target_ms_p95": 120, - "av_lag_ms_p50": 2.3, - "av_lag_ms_p95": 8.6 + "av_lag_ms_p50": 0.6, + "av_lag_ms_p95": 9.7 }, { "runtime": "replay", diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index 6cb92752b6c..f0f20b0e1c7 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -141,7 +141,6 @@ for (const { row, trace } of rows) { rate, delay: "auto", config: trace.config as unknown as Catalog.AudioConfig, - largest: trace.largest, duration: trace.duration, }); let last: Reading | undefined;