diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 1e23ad2ef91..50b9fe395de 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -174,7 +174,7 @@ MPEG-TS export is covered in the [MPEG-TS guide](/bin/ts#export). ```bash moq --connect https://relay.example.com/anon --broadcast my-stream.hang play moq ... play --delay 500ms # fix the delay instead of measuring it -moq ... play --stretch 200ms # let audio trail the clock by at most 200 ms +moq ... play --stretch 500ms # let audio trail the clock by up to 500 ms before it skips moq ... play --video off # audio only, without a window moq ... play --video off --metrics play.ndjson # log what the speaker played ``` @@ -192,9 +192,9 @@ default, `auto`, sizes the buffer to how unevenly audio arrives, with the same delay instead. Audio and video play from one clock and stay in sync while both arrive on time. After a stall, video shows each late frame as it arrives, while audio slows down and falls behind the clock rather than dropping what arrives -late, then speeds up until it is back on it. `--stretch` (default 2 s) bounds -how far audio may trail before it skips ahead, and a stalled group is waited on -for the delay plus the stretch before it is skipped. `--stretch 0` plays +late, then speeds up until it is back on it. Audio that trails by more than +`--stretch` (default 200 ms) skips onto the clock, as video does, and audio +that leads it by as much waits in silence. `--stretch 0` plays received audio unaltered: late audio is skipped, an underrun is silence, and drift against the publisher's clock is corrected by a small skip or pad. Each role follows the catalog, switching rendition when the publisher retires diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index d52fff6e7f5..ee1595d0d1c 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -47,8 +47,8 @@ no frame, is waited for only the playout delay plus any lookahead, since past th it would play late: a reordered backlog's holes waited out to the whole budget would delay the frames behind them, which the estimator would measure as jitter. A caller that estimates keeps the budget past the target, which is why -`moq play` waits on a stalled group for the delay plus its `--stretch`, 2 s by -default. The observation is also only as honest as +`moq play` waits on a stalled audio group for the delay plus the estimate's 2 s +ceiling. The observation is also only as honest as the caller is prompt: `read()` is timed when it resolves, so a player drains it as packets come and lets the playback buffer hold the target, rather than pacing reads to the speaker. @@ -59,15 +59,12 @@ before any container sees them, at the relay and in the receiver's own transport subscriber, so a subscription cut to the target hides every frame later than the target: the estimate creeps up one bucket at a time while the frames it should have measured go unplayed. Both implementations subscribe with the delay -plus the player's `stretch` setting: `moq play` through its age budget, and the -browser through the audio subscription alone, since its container consumer -observes each frame before applying the local budget. The 2 s default reaches -the estimate's own ceiling, so any lateness the estimator can measure is -observed at once. A smaller stretch hides frames later than the target plus the -stretch, so a jump in lateness larger than the stretch is learned in steps of -about the stretch; a zero stretch cuts a live subscription to the target and -accepts the one-bucket creep. A buffered browser player keeps its subscription -at least the delay plus its lookahead, whatever the stretch. +plus the estimate's own 2 s ceiling, whatever the player's `stretch` setting: +`moq play` through its age budget, and the browser through the audio +subscription alone, since its container consumer observes each frame before +applying the local budget. So any lateness the estimator can measure is +observed at once. A buffered browser player keeps its subscription at least the +delay plus its lookahead. Browser video keeps the shared budget, because a longer one fetches whole stale groups, so a broadcast with no audio measures only the lateness that budget lets through. @@ -681,10 +678,10 @@ At each window boundary a player decides on five inputs: clock from speeding into it. - `lag`: how far the playhead trails the shared clock, negative when it plays ahead of it. -- `bound`: how far audio may trail the clock, which is the subscription's max - delay less the playout delay: the player's `stretch` setting (2 s by - default), or for a buffered player the larger of it and the lookahead. A zero - stretch turns the stretch off; see [below](#skips-underruns-and-silence). +- `bound`: how far audio may trail the clock: the player's `stretch` setting + (200 ms by default), or for a buffered player the larger of it and the + lookahead. It sizes nothing else. A zero stretch turns the stretch off; see + [below](#skips-underruns-and-silence). `level` passes through NetEq's `BufferLevelFilter` once per window: three of its 10 ms steps at the factor the target selects, 251, 252, 253, or 254 in 256ths @@ -724,8 +721,8 @@ playhead ahead of the clock back onto it. These are NetEq's thresholds and rates, untuned. A moved target moves the clock, so a rise leaves audio ahead of it and a fall leaves audio behind, and both become ordinary slowing down and speeding up: nothing parks in silence on a rise, and nothing skips buffered -audio on a fall shorter than the bound and one window. A fall past that skips -to the bound, as any lag past it does. +audio on a fall, within the bound. Past it, a fall skips onto the clock and a +rise pads silence until the clock arrives, as any lag past it does. Without a clock reading the target is where the playhead belongs, so `lag = level - target`. The browser then never skips; native playback bounds @@ -733,9 +730,12 @@ the lag at three quarters of its ring's capacity above the target. ### Skips, underruns, and silence -A playhead that trails the clock by more than `bound` and one window skips -ahead to `bound` behind it, dropping the staged window and the audio it passes. -Less than that is the stretch's to catch up. A join is the exception: until +A playhead that trails the clock by more than `bound`, or one window when that +is larger, skips onto the clock, dropping the staged window and the audio it +passes, as video skips the frames a stall made late. One that leads the clock +by as much, as a target rise past the bound leaves it, plays silence until the +clock reaches it, dropping nothing. Less than that either way is the stretch's +to catch up. A join is the exception: until the player has read once on the clock, its bound is zero, so the backlog a subscription opens on is skipped rather than played out at the stretch's pace, and playback starts on the clock. A shared clock whose reference moves earlier, diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 7ed0fd69b1f..45648fe9855 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -34,7 +34,7 @@ in sync at the latency you ask for. | `paused`, `muted`, `volume` | The usual player controls. | | `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames arrive, at most 10 s; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint video as it decodes, with no audio. | | `lookahead` | How far ahead of the live edge media may run; see [buffered playback](#buffered-playback). | -| `stretch` | How far audio may trail the clock before it skips ahead: a duration like `"2s"` (the default), or `"0"` to turn the time stretch off; see [catching up](#catching-up). | +| `stretch` | How far audio may trail the clock before it skips onto it: a duration like `"200ms"` (the default), or `"0"` to turn the time stretch off; see [catching up](#catching-up). | | `captions` | The caption track to show. | | `visible` | Only download video while the element is on screen (or near it). | | `backend` | How the canvas is drawn; see [@moq/video](/lib/js/video). `auto` (default), `webgpu`, or `2d`. | @@ -224,11 +224,10 @@ leave the connection's estimate unclaimed. Without a claim, video selects the be After a stall, video shows each late frame as it arrives and is live again at once. Audio cannot skip without losing what arrives late, so it slows down slightly when the audio received ahead of it runs low, falls behind the clock, -and speeds back up once it recovers until it is back on it. `stretch` bounds -how far it may trail: only audio later than that is skipped, and the audio -subscription waits on a stalled group for `delay + stretch`, so the late audio -still arrives. A voice app might ask for `stretch="200ms"`, a sports stream for -the default 2 s, whether `delay` is `auto` or fixed. +and speeds back up once it recovers until it is back on it. Audio that trails +by more than `stretch`, 200 ms by default, skips onto the clock as video does, +so a long stall costs one skip instead of seconds of lag, and audio that leads +it by as much, as when the delay rises, waits for it in silence. `stretch="0"` turns the time stretch off, for content where a pitch-period splice is audible, such as music: received audio plays unaltered, late audio diff --git a/js/watch/README.md b/js/watch/README.md index 3def3a2dba6..c6032397762 100644 --- a/js/watch/README.md +++ b/js/watch/README.md @@ -74,7 +74,7 @@ The simplest way to watch a stream: | `follow` | boolean | true | Follow the broadcast's announcements: wait for one before subscribing, and move to each restart. Ignored when the relay does not support broadcast discovery. | | `delay` | `auto`, duration, `instant` | `auto` | Distance from the live edge. `instant` paints frames as they decode and disables audio. | | `lookahead` | duration | `0ms` | How far future-dated media may run ahead of the live edge. Not a jitter buffer: live streams tune `delay`. | -| `stretch` | duration | `2000ms` | How far audio may trail the clock after a stall before it skips ahead. `0` turns the time stretch off, so received audio plays unaltered. | +| `stretch` | duration | `200ms` | How far audio may trail the clock after a stall before it skips onto it. `0` turns the time stretch off, so received audio plays unaltered. | | `captions` | string | off | Text rendition to render. | | `catalog-format` | hang, msf, manual | auto-detected | The catalog format; detected from the name suffix unless set. | | `echo` | path segment | - | The viewer's feedback name: once the catalog solicits feedback, publish `/.echo` reporting what each rendition received and played. | diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 8cacc93fec7..d5703462e79 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1774,21 +1774,45 @@ it.each([false, true])("releases shared startup across a hole in queued audio (C }); // Audio may trail the clock by the stretch, or a buffered player's whole lookahead when that is longer, -// and a zero stretch turns the time stretch off in either mode. +// and a zero stretch turns the time stretch off in either mode. The stretch sizes nothing else: the +// subscription reaches the estimator's 2 s ceiling past the delay whatever it is. for (const buffered of [false, true]) { - it(`bounds ${buffered ? "buffered" : "live"} audio's lag by the stretch`, async () => { + it(`bounds ${buffered ? "buffered" : "live"} audio's lag by the stretch alone`, async () => { + // The max delay each subscribe and update asked the publisher for, in order. + const asked: (Duration | null | undefined)[] = []; + const subscribe = Moq.Track.Consumer.prototype.subscribe; + const update = Moq.Track.Subscriber.prototype.update; + const subscribes = spyOn(Moq.Track.Consumer.prototype, "subscribe").mockImplementation(function ( + this: Moq.Track.Consumer, + ...args: Parameters + ) { + asked.push(args[0]?.delay?.max); + return subscribe.apply(this, args); + }); + const updates = spyOn(Moq.Track.Subscriber.prototype, "update").mockImplementation(function ( + this: Moq.Track.Subscriber, + ...args: Parameters + ) { + asked.push(args[0]?.delay?.max); + return update.apply(this, args); + }); const playback = await play(Duration.millis(100), buffered); try { await playback.play(12); const bound = () => worklets.at(-1)?.terms?.bound; - expect(bound()).toBe(2000); - playback.stretch.set(Duration.millis(300)); + expect(bound()).toBe(buffered ? 1000 : 200); + const reach = Duration.millis(2100); + expect(asked.at(-1)).toEqual(reach); + playback.stretch.set(Duration.millis(1500)); await microtasks(); - expect(bound()).toBe(buffered ? 1000 : 300); + expect(bound()).toBe(1500); playback.stretch.set(Duration.ZERO); await microtasks(); expect(bound()).toBe(0); + expect(asked.at(-1)).toEqual(reach); } finally { + subscribes.mockRestore(); + updates.mockRestore(); playback.close(); await microtasks(); } @@ -1816,7 +1840,7 @@ for (const cmaf of [false, true]) { // Either ring keeps what it receives and plays onto the clock, which audio may trail by the // larger of the lookahead and the default stretch, even at a fixed delay. - expect(worklets.at(-1)?.terms).toEqual({ latency: 100, target: 100, bound: 2000 }); + expect(worklets.at(-1)?.terms).toEqual({ latency: 100, target: 100, bound: buffered ? 1000 : 200 }); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 242c438c17d..ab219d5a7b6 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -188,7 +188,7 @@ export class Decoder { // The container consumer's arrival estimate, unset while nothing is subscribed. #measured = new Signal(undefined); - // The subscription's max delay: the shared budget, raised to the delay plus the stretch. + // The subscription's max delay: the shared budget, raised to the delay plus the estimator's ceiling. #subscribeMaxDelay = new Signal(Duration.ZERO); // The container consumer's delay: the playout delay to start at and the shared budget. Its warmup // reaches the start further back, to the media the decoder primes on. @@ -321,9 +321,9 @@ export class Decoder { }); } - // The subscription reaches the stretch past the delay, so audio the ring could still catch up on - // arrives. A group the relay expires is never observed, so a zero stretch, which cuts the - // subscription to the target, leaves the estimate rising one bucket at a time. The container + // The subscription reaches the estimator's 2 s ceiling past the delay, whatever the stretch, since a + // group the relay expires is never observed: a subscription cut to the target would leave the + // estimate rising one bucket at a time. The container // consumer drops aged media at the same max delay, since it observes each frame before applying // it, and carves out the warmup the subscription adds once it has warmed, as Rust's does. It waits // on a missing group only for the delay plus the lookahead: past that the group would play late, @@ -338,7 +338,7 @@ export class Decoder { this.#subscribeMaxDelay.set(max.add(warmup)); } - // The terms of the subscription's max delay: the shared delay and max delay, and the stretch. + // The shared delay and max delay, and the stretch threshold the ring skips past. #budget(effect: Effect): { delay: Duration; maxDelay: Duration; stretch: Duration } { return { delay: effect.get(this.sync.out.delay), @@ -511,7 +511,7 @@ export class Decoder { if (!ring) return; // The ring plays onto the shared clock, slowing down onto a rise and speeding up onto a fall, - // and skips only audio trailing it past what the subscription still delivers. + // and skips onto it only audio trailing it past the stretch threshold. const budget = this.#budget(effect); ring.setLatency(budget.delay, bound(budget)); } diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 8ee53995323..060b010ba6e 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -26,20 +26,26 @@ export function target(props: Target): Duration { return floored.add(props.frame ?? Duration.ZERO).add(props.delay ?? Duration.ZERO); } +// The arrival estimate's ceiling, `BUCKET * BUCKETS` in @moq/hang's jitter estimator: no lateness past +// it moves the target, so a subscription reaching that far past the delay observes all it can measure. +const REACH = Duration.secs(2); + /** - * The audio subscription's max delay, less warmup: the shared `maxDelay`, raised to `delay + stretch` - * so audio trailing the clock by up to the stretch still arrives for the ring to catch up on. + * The audio subscription's max delay, less warmup: the shared `maxDelay`, raised to the delay plus the + * estimator's 2 s ceiling, so every frame late enough to move the target is still observed. The + * `stretch` setting does not size it. */ -export function maxDelay(props: { delay: Duration; maxDelay: Duration; stretch: Duration }): Duration { - return props.maxDelay.max(props.delay.add(props.stretch)); +export function maxDelay(props: { delay: Duration; maxDelay: Duration }): Duration { + return props.maxDelay.max(props.delay.add(REACH)); } /** - * How far the ring lets audio trail the clock before it skips ahead: the audio max delay less the - * playout delay, or zero, which turns the time stretch off, when `stretch` is zero. + * How far the ring lets audio trail the clock before it skips onto it: the `stretch` threshold, or + * for a buffered player the larger of it and the lookahead (`maxDelay` less the delay). Zero, which + * turns the time stretch off, when `stretch` is zero. */ export function bound(props: { delay: Duration; maxDelay: Duration; stretch: Duration }): Duration { - return props.stretch.isZero() ? Duration.ZERO : maxDelay(props).saturatingSub(props.delay); + return props.stretch.isZero() ? Duration.ZERO : props.stretch.max(props.maxDelay.saturatingSub(props.delay)); } // An AudioWorkletProcessor renders in fixed 128-sample quanta, so a ring shallower than one can diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index 8477904a5b3..aeb3a6ec9c9 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -127,12 +127,12 @@ async function measure(t: Arrival[], budget: (target: Duration) => Duration): Pr return result; } -/** The subscription "auto" asks for at the default stretch: the target plus the stretch. */ -const stretched = (target: Duration) => maxDelay({ delay: target, maxDelay: target, stretch: STRETCH }); +/** The subscription "auto" asks for: the target plus the estimator's 2 s ceiling. */ +const reached = (target: Duration) => maxDelay({ delay: target, maxDelay: target }); /** The target the estimator settles on by the end of `t`, given the subscription "auto" asks for. */ async function settled(t: Arrival[]): Promise { - const { targets } = await measure(t, stretched); + const { targets } = await measure(t, reached); return targets[targets.length - 1]; } @@ -244,7 +244,7 @@ describe("subscription budget", () => { } it("reaches a flush that starts after the target settled", async () => { - const { targets, delivered } = await measure(t, stretched); + const { targets, delivered } = await measure(t, reached); expect(targets[PACED - 1]).toBe(2 * CHUNK_MS); expect(delivered).toBe(t.length); // Every flush lands whole, so its 100ms reaches the 95th percentile within a few intervals. diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 575d731981a..a5d05a03832 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -216,7 +216,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const track = new Track.Producer("audio"); // The subscription's max delay, as the decoder sets it, which the consumer drops aged media at too. const maxDelay = new Derived([sync.out.delay, sync.out.maxDelay], (delay, max) => - audioMaxDelay({ delay, maxDelay: max, stretch }), + audioMaxDelay({ delay, maxDelay: max }), ); const consumer = new Container.Consumer(track.subscribe({ delay: { max: WIRE_MAX_DELAY } }), { format: new Container.Legacy.Format("audio"), diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index ff957bebab6..ff20b7df278 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -1237,28 +1237,61 @@ describe("time stretch", () => { expect(buffer.underruns).toBe(0); }); - // Audio the stretch could still catch up on plays; only what trails the clock past the bound and - // one window skips, back to the bound. + // A playhead ahead of the clock within the bound is the stretch's to slow down onto; past it, as a + // target rise leaves it, it pads silence until the clock reaches it, dropping nothing. + it("pads a playhead ahead of the clock past its bound onto it", () => { + const buffer = live(100, { bound: 200 }); + write(buffer, 0, frames(800), { value: 0.5 }); + buffer.clock(0); + read(buffer, QUANTUM, 1); + const quantum = (QUANTUM * 1_000_000) / RATE; + + // 100 ms ahead: within the bound, so audio keeps playing. + let clock = buffer.timestamp - 100_000; + buffer.clock(clock); + expect(read(buffer, QUANTUM, 1)[0].some((sample) => sample !== 0)).toBe(true); + clock += quantum; + + // 300 ms ahead: silence until the clock reaches the playhead. + clock = buffer.timestamp - 300_000; + const playhead = buffer.timestamp; + let padded = 0; + for (;;) { + buffer.clock(clock); + const [output] = read(buffer, QUANTUM, 1); + clock += quantum; + if (output.some((sample) => sample !== 0)) break; + padded++; + } + expect(padded).toBe(Math.floor(300_000 / quantum)); + expect(buffer.timestamp).toBeGreaterThan(playhead); + expect(buffer.skips).toBe(0); + expect(buffer.underruns).toBe(0); + }); + + // Audio the stretch could still catch up on plays; what trails the clock past the bound, or one + // window when that is larger, skips onto the clock. for (const bound of [0, 200]) { - it(`skips only audio trailing the clock past a ${bound} ms bound and a window`, () => { + it(`skips audio trailing the clock past a ${bound} ms bound onto it`, () => { + const threshold = Math.max(bound, 30); const buffer = live(100, { bound }); write(buffer, 0, frames(800), { value: 0.5 }); // Joined on the clock. buffer.clock(0); read(buffer, QUANTUM, 1); - buffer.clock(buffer.timestamp + (bound + 30) * 1000); + buffer.clock(buffer.timestamp + threshold * 1000); read(buffer, QUANTUM, 1); expect(buffer.skips).toBe(0); - const clock = buffer.timestamp + (bound + 31) * 1000; + const clock = buffer.timestamp + (threshold + 1) * 1000; buffer.clock(clock); read(buffer, QUANTUM, 1); expect(buffer.skips).toBe(1); - // Back to the bound, then a quantum on, which a stretch may have sped up. + // Onto the clock, then a quantum on. const quantum = (QUANTUM * 1_000_000) / RATE; - expect(buffer.timestamp).toBeGreaterThanOrEqual(clock - bound * 1000 + quantum); - expect(buffer.timestamp).toBeLessThan(clock - bound * 1000 + 2 * quantum + 1); + expect(buffer.timestamp).toBeGreaterThanOrEqual(clock + quantum - 1); + expect(buffer.timestamp).toBeLessThan(clock + quantum + 1); }); } }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 5ed35381afa..a7a07fed31d 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -11,8 +11,8 @@ const QUANTUM = 128; * * Playback follows the decision rule in doc/concept/audio-jitter.md: each window of received PCM may * play faster to catch up with the shared clock once the buffer holds more than the target, or slower - * while the buffer runs low or the playhead runs ahead of the clock. Only audio trailing the clock past - * the bound is skipped. A zero bound turns the stretch off: received audio plays unaltered, and a + * while the buffer runs low or the playhead runs ahead of the clock. Audio trailing the clock past the + * bound skips onto it, and a playhead ahead of it past the bound pads silence until the clock reaches it. A zero bound turns the stretch off: received audio plays unaltered, and a * playhead more than a window off the clock skips ahead onto it or waits for it in silence. */ export class AudioRingBuffer { @@ -500,11 +500,12 @@ export class AudioRingBuffer { return 0; } - // Skip what trails the clock past its bound and one window, back to the bound, or the jumps of - // its reference past one window. Less than that is the stretch's to catch up. A join has the - // bound of nothing heard yet, so it lands on the clock past whatever backlog it was handed, and a - // jump before it was never heard. A finite tail plays as received. With the stretch off, a - // playhead more than a window ahead of the clock pads silence until the clock reaches it. + // Skip what trails the clock past its bound, or one window when that is larger, onto the clock, + // or the jumps of its reference past one window. Less than that is the stretch's to catch up. A + // join has the bound of nothing heard yet, so it lands on the clock past whatever backlog it was + // handed, and a jump before it was never heard. A finite tail plays as received. A playhead ahead of + // the clock past the bound, or one window when that is larger, as a target rise leaves it, pads + // silence until the clock reaches it. let skip = 0; let pad = 0; if (this.#clock !== undefined) { @@ -517,13 +518,11 @@ export class AudioRingBuffer { this.#incoming = 0; this.#joined = true; if (!this.#ending()) { - if (inputs.lag > bound + this.#stretch.window) skip = inputs.lag - bound; + if (inputs.lag > Math.max(bound, this.#stretch.window)) skip = inputs.lag; if (this.#jumped > this.#stretch.window) skip = Math.max(skip, this.#jumped); - if (this.#bound === 0) { - const lead = Math.round(-inputs.lag); - this.#padding = lead > (this.#padding ? 0 : this.#stretch.window); - if (this.#padding) pad = Math.min(lead, output[0].length); - } + const lead = Math.round(-inputs.lag); + this.#padding = lead > (this.#padding ? 0 : Math.max(this.#bound, this.#stretch.window)); + if (this.#padding) pad = Math.min(lead, output[0].length); } } if (skip > 0) { diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index 6691768f068..7a5d5450944 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -113,7 +113,7 @@ function parseStretch(value: string | null): Duration { if (value === null) return STRETCH; const parsed = parseDuration(value); if (parsed !== undefined) return parsed; - throw invalidAttribute("stretch", value, 'a duration like "2s", or "0" to turn it off'); + throw invalidAttribute("stretch", value, 'a duration like "200ms", or "0" to turn it off'); } // Parse the `echo` attribute: the viewer's feedback name, one visible path segment, or undefined when absent. @@ -218,7 +218,7 @@ export default class MoqWatch extends HTMLElement { delay: new Signal("auto"), // Future-dated media held beyond the live edge before playback skips ahead. lookahead: new Signal(Duration.ZERO), - // How far audio may trail the clock before it skips ahead; zero turns the time stretch off. + // How far audio may trail the clock before it skips onto it; zero turns the time stretch off. stretch: new Signal(STRETCH), video: { // The desired video rendition (resolution/bitrate cap). @@ -622,8 +622,8 @@ export default class MoqWatch extends HTMLElement { * Zero (the default) minimizes latency. A larger value enables buffered playback: build up * future-dated frames (e.g. TTS written faster than real-time) and only skip ahead * once they would sit further than `delay + lookahead` past the playhead. Audio that falls - * behind, as after a stall, catches up by playing faster, and skips only what trails by more - * than the larger of the lookahead and the {@link stretch}. Call `reset()` at each utterance boundary. Not a jitter buffer: live + * behind, as after a stall, catches up by playing faster, and skips onto the clock only past + * the larger of the lookahead and the {@link stretch}. Call `reset()` at each utterance boundary. Not a jitter buffer: live * streams leave it at zero and tune `delay`. */ get lookahead(): Duration { @@ -637,13 +637,12 @@ export default class MoqWatch extends HTMLElement { } /** - * How far audio may trail the clock before it skips ahead (default: 2 s), mirrored by the + * How far audio may trail the clock before it skips onto it (default: 200 ms), mirrored by the * `stretch` attribute. * - * After a stall, audio slows down and falls behind the clock rather than dropping what arrives - * late, then speeds up until it is back on it. A voice app might ask for 200 ms, a sports stream - * for 2 s. Zero turns the time stretch off: received audio plays unaltered and late audio is - * skipped. + * Within it, audio behind the clock, as after a stall, speeds up until it is back on it; past it, + * audio skips onto the clock as video does. Zero turns the time stretch off: received audio plays + * unaltered and late audio is skipped. */ get stretch(): Duration { return this.controls.stretch.peek(); diff --git a/js/watch/src/player.ts b/js/watch/src/player.ts index e238b491032..9bd0a4f2d76 100644 --- a/js/watch/src/player.ts +++ b/js/watch/src/player.ts @@ -30,7 +30,7 @@ export type PlayerInput = { delay: Getter; /** Future-dated media held beyond the live edge. */ lookahead: Getter; - /** How far audio may trail the clock before it skips ahead; zero turns the time stretch off. */ + /** How far audio may trail the clock before it skips onto it; zero turns the time stretch off. */ stretch: Getter; /** Video rendition and canvas output. */ video: { diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 176a7fb6d66..a162d5a3304 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -16,8 +16,8 @@ import { Derived, Effect, type Getter, getter, type Inputs, type Readonlys, read */ export type Delay = "instant" | "auto" | Duration; -/** The default {@link SyncInput.stretch}: how far live audio may trail the clock before it skips. @internal */ -export const STRETCH = Duration.secs(2); +/** The default {@link SyncInput.stretch}: how far live audio may trail the clock before it skips onto it. @internal */ +export const STRETCH = Duration.millis(200); /** The most an `"auto"` delay holds, and how far catalog `warmup` reaches back. @internal */ export const DELAY_MAX = Container.DELAY_MAX; @@ -33,8 +33,8 @@ export type SyncInput = { * Zero minimizes latency, the live default: playback re-anchors as soon as anything arrives * early. A larger value lets faster-than-real-time frames (a TTS response with future * timestamps) build up instead of being skipped. Audio that falls behind, as after a stall, - * catches up by playing faster, and skips only what trails by more than the larger of the - * lookahead and {@link SyncInput.stretch}. Always + * catches up by playing faster, and skips onto the clock only when it trails by more than the + * larger of the lookahead and {@link SyncInput.stretch}. Always * finite, so worst case the audio ring drops its oldest samples rather than exhausting memory. * With a nonzero lookahead, the first video * frame is a poster until every enabled media track holds the resolved delay; `lookahead` is @@ -43,12 +43,13 @@ export type SyncInput = { lookahead: Getter; /** - * How far audio may trail the clock before it skips ahead (default: 2 s). + * How far audio may trail the clock before it skips onto it (default: 200 ms). * - * After a stall, audio slows down and falls behind the clock rather than dropping what arrives - * late, then speeds up until it is back on it; only what trails by more than this is skipped. The - * audio subscription asks for `delay + stretch`, so that late audio still arrives. A buffered - * player already keeps its whole lookahead, so audio skips only past the larger of the two. + * Within this, audio that falls behind the clock, as after a stall, speeds up until it is back on + * it; past it, audio skips onto the clock, as video does. Audio that leads the clock by more than + * this, as a rise in the delay leaves it, waits for it in silence. A buffered player already keeps its + * whole lookahead, so audio skips only past the larger of the two. This sizes nothing else: the + * audio subscription reaches 2 s past the delay whatever it is set to. * * Zero turns the time stretch off: received audio plays unaltered, late audio is skipped, an * underrun is silence, and drift against the publisher's clock is corrected by a small skip or diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 5c4dd07f667..d2852608302 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -56,6 +56,5 @@ audit ([moq-dev/moq#8648](https://github.com/moq-dev/moq/pull/8648)), the size report's output ([moq-dev/moq#8654](https://github.com/moq-dev/moq/pull/8654)), the firefox resume ([moq-dev/moq#8635](https://github.com/moq-dev/moq/pull/8635)), and the chromium `av_lag` rows -([output latency](/quest/m2/audio-playout/output-latency.md) and -[congestion catch-up](/quest/m2/audio-playout/congestion-catch-up.md)). +([output latency](/quest/m2/audio-playout/output-latency.md)). diff --git a/quest/m2/audio-playout/README.md b/quest/m2/audio-playout/README.md index e944482a3e3..c65a7104a84 100644 --- a/quest/m2/audio-playout/README.md +++ b/quest/m2/audio-playout/README.md @@ -87,8 +87,8 @@ zero underruns (2026-11-08); the target formula and the playout as the largest `delay + target` among subscribed renditions; one algorithm with shared vectors for both players. -Order: [congestion catch-up](/quest/m2/audio-playout/congestion-catch-up.md) -gives the stall its snap on today's splices, the +Order: the 200 ms snap both ways is on `main` on today's splices (the +congestion catch-up), the [lag rule](/quest/m2/audio-playout/lag-rule.md) replaces the controller under it, and the [clock step](/quest/m2/audio-playout/clock-step.md) builds on the lag rule. [Target fall](/quest/m2/audio-playout/target-fall.md) and diff --git a/quest/m2/audio-playout/congestion-catch-up.md b/quest/m2/audio-playout/congestion-catch-up.md deleted file mode 100644 index 76ffef2125d..00000000000 --- a/quest/m2/audio-playout/congestion-catch-up.md +++ /dev/null @@ -1,62 +0,0 @@ -# [M] Audio recovers from a congestion stall quickly - -## Goal - -After a congestion stall, audio is back on the shared clock within about 2 s -of arrivals resuming, in the browser and natively, instead of trailing by up -to 1.4 s for 17 to 28 s. Audio that trails the clock by more than the -`stretch` threshold, now 200 ms by default, skips onto the clock, and the -`stretch` setting sizes nothing but that threshold. - -## Plan - -Found by the triage of Nightly run -[43447849289](https://github.com/moq-dev/moq/actions/runs/43447849289) -(2026-11-13). Caused by [moq-dev/moq#8507](https://github.com/moq-dev/moq/pull/8507), -which stopped trimming audio that trails the clock, and -[moq-dev/moq#8532](https://github.com/moq-dev/moq/pull/8532), whose `stretch` -bound defaults to 2 s. Chromium `opus-48000-congestion` `av_lag_ms` p95 is -1434 ms against a 32 ms ceiling; trims fell from about 1000 ms/min to 0. Both -players catch up at about 82 ms of lag a second, and each later fall of the -target adds lag back. - -Decided 2026-11-13 (maintainer, /quest-plan): ✅ a regression to fix -(rejected: accept it and let calibrate budget the congestion rows). Then the -same day, in the [audio playout](/quest/m2/audio-playout/README.md) plan: -✅ a 200 ms default threshold past which audio skips onto the clock -(rejected: the implementer measuring a lower bound against NetEq's fast -accelerate). This is the first step of that rule, on today's splices; the -[lag rule](/quest/m2/audio-playout/lag-rule.md) replaces the controller -under it. - -Scope: - -- `stretch` defaults to 200 ms (`js/watch/src/sync.ts` `STRETCH`, `moq play - --stretch`); zero still turns the stretch off. -- A lag past the threshold skips onto the clock, not back to the bound, in - `js/watch/src/audio/ring-buffer.ts` and `rs/moq-audio/src/ring.rs`. A - buffered player skips only past the larger of its lookahead and the - threshold, as today. -- The setting sizes nothing else. The audio subscription keeps the delay - plus the larger of the lookahead and the estimator's 2 s ceiling, so late - frames are still observed and a buffered player keeps its lookahead. - Since [moq-dev/moq#8611](https://github.com/moq-dev/moq/pull/8611) that max - delay is also where buffered media past a missing group skips it, so the - threshold must not shrink it. Natively, the decode cap, the hole policy (`fill_max`), and video's wait on - a missing group (`args.rs` `video_max_delay`) stop following it, so video - waits its own delay. That also fixes `--delay auto --stretch 0`, where any - hole over 50 ms drops the sink and its queued audio (`play/media.rs` - `cap`). -- The audio-quality harness reports, per row, how long after a disturbance - audio is back within a frame of where it rested before it. Today's - `recover_ms` measures against the window's p50 and is not tied to the - disturbance. -- `doc/concept/audio-jitter.md`'s observation point and skip rules say the - same. - -Put the before and after rows of `just test audio-quality` for chromium and -native in the PR. - -## Related - -- [moq-dev/moq#8635](https://github.com/moq-dev/moq/pull/8635) - snaps resumed audio to the clock, the same rule for a return from absence diff --git a/quest/m2/audio-playout/lag-rule.md b/quest/m2/audio-playout/lag-rule.md index 18946c89f9a..36f3d2e2d14 100644 --- a/quest/m2/audio-playout/lag-rule.md +++ b/quest/m2/audio-playout/lag-rule.md @@ -34,7 +34,8 @@ Settled: gain and reports the time from 200 ms back to within a frame. - Snap: past the threshold, or for a buffered player past the larger of its lookahead and the threshold, skip onto the clock when behind and pad - silence when ahead. A join snaps. + silence when ahead. A join snaps. Both rings already do this on NetEq's + controller (`ring-buffer.ts` `read`, `ring.rs` `skip` and `pad`); keep it. - Bridge: slow down while the received depth is below a cushion sized from the stretch's window and the device read, and the lag is under the threshold. The depth, not NetEq's level at arrival, so a batched publisher @@ -60,10 +61,6 @@ Recommendations: Public API: `Sink::due` and `bus::Positioned` lose `jumped`; both are `main`-only, so nothing published migrates. Wire: none. -## Required - -- [Congestion catch-up](/quest/m2/audio-playout/congestion-catch-up.md) - the 200 ms threshold and the snap this controller keeps - ## Related - [WebRTC vendor sync](/quest/m3/vendor-sync/webrtc.md) - tracks the NetEq port this deletes diff --git a/rs/moq-audio/src/bus.rs b/rs/moq-audio/src/bus.rs index 39df96fec94..1eb6a7781d9 100644 --- a/rs/moq-audio/src/bus.rs +++ b/rs/moq-audio/src/bus.rs @@ -88,10 +88,10 @@ pub const MAX_DELAY: Duration = Duration::from_secs(2); pub const REACH: Duration = LATENCY.saturating_add(HEADROOM); /// The longest stretch bound [`Bus::positioned`] accepts, which bounds how -/// much audio a positioned channel holds. Matches the players' default. +/// much audio a positioned channel holds. pub const MAX_STRETCH: Duration = Duration::from_secs(2); -/// The time stretch's window, past which audio trailing the bound is skipped. +/// The time stretch's window, which a ring may hold past its bound. const WINDOW: Duration = Duration::from_millis(30); /// What every mix is mixed to. @@ -1098,7 +1098,7 @@ impl Positioned { /// With one, a write plays right after the audio the channel holds, and the /// channel speeds up or slows down onto its position, as the playback sink /// does onto its clock. Audio trailing its positions by more than the bound - /// plus one 30 ms window is skipped back to the bound. Only a write to a + /// skips onto them. Only a write to a /// channel that has run dry is placed exactly, behind silence, since /// nothing audible plays before it. A write fails whole if the channel has /// no room for it. @@ -2415,7 +2415,7 @@ mod tests { /// A stretched channel that has run dry places a write exactly, since /// silence plays until it anyway; one trailing its positions plays late, - /// up to the bound plus a window, past which it skips back to the bound. + /// up to the bound, past which it skips onto its positions. #[test] fn a_stretched_channel_places_onto_dry_and_skips_past_its_bound() { let (bus, mut driver) = Bus::new(Config::default()).unwrap(); @@ -2444,8 +2444,8 @@ mod tests { assert_eq!(left.iter().position(|sample| *sample != 0.0), Some(0)); assert_eq!(stretched(&bus, &channel, |cons| cons.trimmed()), 0); - // Trailing by 300 ms, past the bound and a window: skips the 200 ms - // past the bound. + // Trailing by 300 ms, past the bound: skips all 300 ms onto its + // positions. mixed(&mut driver, &mut reader, &[BLOCK], 48_000); let next = driver.position(); channel.write(next - 14_400, &tone(0, 15_360)).unwrap(); @@ -2459,7 +2459,7 @@ mod tests { driver.advance(BLOCK); ready(&mut reader); - assert_eq!(stretched(&bus, &channel, |cons| cons.trimmed()), 9_600); + assert_eq!(stretched(&bus, &channel, |cons| cons.trimmed()), 14_400); channel.write(next, &tone(0, 960)).unwrap(); } diff --git a/rs/moq-audio/src/playback/sink.rs b/rs/moq-audio/src/playback/sink.rs index 988b22c18c3..b107b7c3094 100644 --- a/rs/moq-audio/src/playback/sink.rs +++ b/rs/moq-audio/src/playback/sink.rs @@ -279,8 +279,9 @@ impl Sink { /// The sink then plays onto that clock rather than at its latency alone. /// Behind it, it speeds up once it holds more than its latency; ahead of /// it, it slows down. Its buffer running low slows it down too, falling - /// behind the clock by at most `stretch`, and audio trailing the clock by - /// more than `stretch` plus one 30 ms window is skipped. A zero `stretch` + /// behind the clock by at most `stretch`. Audio trailing the clock by more + /// than `stretch` skips onto it, and audio leading it by as much waits in + /// silence for it. A zero `stretch` /// turns the time stretch off: written audio plays unaltered, and a sink /// more than a window off the clock skips onto it or pads silence until /// the clock reaches it. `jumped` is how far the diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index c6934a35ea6..1027a20a94c 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -5,8 +5,9 @@ //! It opens primed with silence to its latency, can pad back up to it when the //! writer falls behind, and can trim back down to it when the writer runs ahead. //! Native playback instead enables received-audio stretch, which plays the -//! queue onto the writer's clock (see [`Producer::due`]) and skips only audio -//! that trails it too far; an actual underrun refills with emergency silence, +//! queue onto the writer's clock (see [`Producer::due`]), skips audio that +//! trails it too far onto it, and pads silence ahead of audio that leads it +//! too far until the clock arrives; an actual underrun refills with emergency silence, //! which never enters the pitch search, until one window is queued again. A //! clock with a zero stretch turns the stretch off: the queue plays as written, //! and a reader more than a window off the clock skips onto it or pads silence @@ -375,8 +376,8 @@ impl Producer { /// Anchor what was pushed so far to the writer's clock: its end is due to /// play at `at`. The stretch then plays the queue onto that clock, and the - /// reader skips audio trailing it by more than `stretch` plus one window. A - /// zero `stretch` turns the stretch off. + /// reader skips audio trailing it by more than `stretch` onto it. A zero + /// `stretch` turns the stretch off. /// `jumped` is how far the clock's reference moved earlier since the last /// anchor, which the reader skips too, as far as it trails the clock. /// Without an anchor, the latency is the clock. @@ -758,8 +759,10 @@ impl Consumer { } } - /// Skip what trails the clock by more than its bound plus one window, back - /// to the bound, or the jumps of its reference past one window, returning + /// Skip what trails the clock by more than its bound, or one window when + /// that is larger, onto the clock (without a clock, what passes the bound + /// by a window, back to it), or the jumps of its reference past one window, + /// returning /// the frames skipped. Less than that is the stretch's to catch up, as a /// writer's clock drifting fast moves the reference. A join has the bound /// of nothing heard yet, so it lands on the clock past whatever backlog it @@ -786,7 +789,11 @@ impl Consumer { 0 }; self.joined |= clocked && self.prime == 0; - let late = if inputs.lag > bound.saturating_add(window as i64) { + // Onto the clock past its bound, or without one, a window past the + // bound back to it, so a level hovering there skips no scraps. + let late = if clocked && inputs.lag > bound.max(window as i64) { + inputs.lag as usize + } else if !clocked && inputs.lag > bound.saturating_add(window as i64) { (inputs.lag - bound) as usize } else { 0 @@ -818,17 +825,22 @@ impl Consumer { !self.shared.clocked.load(Ordering::Acquire) || self.shared.anchor.load(Ordering::Acquire) >> 32 != 0 } - /// With the stretch off, pad silence ahead of the queue once the reader is - /// more than a window ahead of the clock, until the clock reaches it, - /// returning the samples padded. The stretch would slow down instead. + /// Pad silence ahead of the queue once the reader is ahead of the clock by + /// more than its bound, or one window when that is larger, as a target rise + /// leaves it, until the clock reaches it, returning the samples padded. + /// Less than that is the stretch's to slow down onto. fn pad(&mut self, output: &mut [f32]) -> usize { let Some(window) = self.stretch.as_ref().map(|stretch| stretch.window) else { return 0; }; - let lead = -self.inputs(0).lag; - let off = - !self.stretching() && !self.refill && !self.shared.finished.load(Ordering::Relaxed) && !self.declared(); - self.padding = off && lead > if self.padding { 0 } else { window as i64 }; + let inputs = self.inputs(0); + let lead = -inputs.lag; + let eligible = self.shared.clocked.load(Ordering::Acquire) + && !self.refill + && !self.shared.finished.load(Ordering::Relaxed) + && !self.declared(); + let threshold = inputs.bound.max(window as i64); + self.padding = eligible && lead > if self.padding { 0 } else { threshold }; if !self.padding { return 0; } @@ -1409,8 +1421,8 @@ mod tests { } } - /// Audio trailing the clock by more than its bound and a window skips back - /// to the bound; less than that is the stretch's to catch up. + /// Audio trailing the clock by more than its bound skips onto it; less than + /// that is the stretch's to catch up. #[test] fn only_audio_past_the_bound_skips() { let rate = 48_000; @@ -1422,22 +1434,23 @@ mod tests { for _ in 0..100 { assert_eq!(prod.push(&pcm), Push::Ok); } - prod.due( - base + Duration::from_millis(2010), - Duration::from_millis(1970), - Duration::ZERO, - ); + let bound = Duration::from_millis(300); + prod.due(base + Duration::from_millis(2010), bound, Duration::ZERO); assert_eq!(pull(&mut cons, &mut out, base), Read::Ok); assert_eq!(pull(&mut cons, &mut out, base + Duration::from_millis(10)), Read::Ok); - // Now all due at 10 ms: the reader trails by 2000 ms. - let due = base + Duration::from_millis(10); - prod.due(due, Duration::from_millis(1970), Duration::ZERO); + // Now due 500 ms earlier: the reader trails by 500 ms. + let due = base + Duration::from_millis(1510); + prod.due(due, Duration::from_millis(510), Duration::ZERO); assert_eq!(pull(&mut cons, &mut out, base + Duration::from_millis(20)), Read::Ok); - assert_eq!(cons.skips(), 0, "a window past the bound is the stretch's"); - prod.due(due, Duration::from_millis(1960), Duration::ZERO); + assert_eq!(cons.skips(), 0, "within the bound is the stretch's"); + prod.due(due, Duration::from_millis(490), Duration::ZERO); let read = pull(&mut cons, &mut out, base + Duration::from_millis(30)); - assert_eq!(read, Read::Trimmed { frames: 1920 }); - assert_eq!(cons.trimmed(), 1920, "back to the bound"); + let Read::Trimmed { frames } = read else { + panic!("{read:?}"); + }; + // Onto the clock, not back to the bound. + assert!((23_520..=24_480).contains(&frames), "{frames} frames trimmed"); + assert!(cons.inputs(0).lag.abs() <= 480, "lag {}", cons.inputs(0).lag); // A zero stretch has no bound: whatever trails the clock goes, here // what is due before the second the end is due at. prod.due(base + Duration::from_secs(1), Duration::ZERO, Duration::ZERO); @@ -1449,6 +1462,87 @@ mod tests { assert!((940.0..960.0).contains(&left), "{left} ms left"); } + /// Without a clock, a level up to a window past the bound is the stretch's + /// to play down; past that it skips back to the bound, so a writer hovering + /// at the bound never skips a few frames every callback. + #[test] + fn without_a_clock_only_a_window_past_the_bound_skips() { + let rate = 48_000; + let (mut prod, mut cons, pcm) = tone(rate, Duration::from_millis(10)); + let base = prod.base; + let window = cons.stretch.as_ref().unwrap().window; + let mut out = vec![0.0; 480 * 2]; + // Fill to `lag` frames past the target, in slices of the tone. + let fill = |prod: &mut Producer, cons: &Consumer, lag: i64| { + let mut left = (lag - cons.inputs(0).lag) as usize; + while left > 0 { + let frames = left.min(pcm.len() / 2); + assert_eq!(prod.push(&pcm[..frames * 2]), Push::Ok); + left -= frames; + } + }; + let bound = cons.inputs(0).bound; + fill(&mut prod, &cons, bound + window as i64 / 2); + assert_eq!(pull(&mut cons, &mut out, base), Read::Ok); + assert_eq!(cons.skips(), 0, "within a window past the bound is the stretch's"); + + fill(&mut prod, &cons, bound + 2 * window as i64); + let read = pull(&mut cons, &mut out, base + Duration::from_millis(10)); + assert!(matches!(read, Read::Trimmed { .. }), "{read:?}"); + assert_eq!(cons.skips(), 1); + let lag = cons.inputs(0).lag; + assert!( + (bound - 480..=bound).contains(&lag), + "back to the bound {bound}, at {lag}" + ); + } + + /// A reader ahead of the clock within its bound is the stretch's to slow + /// down onto; past it, as a target rise leaves it, it pads silence until the + /// clock reaches it, dropping nothing. + #[test] + fn audio_ahead_past_the_bound_pads_onto_the_clock() { + let rate = 48_000; + let (mut prod, mut cons, pcm) = tone(rate, Duration::from_millis(10)); + let base = prod.base; + let mut out = vec![0.0; 480 * 2]; + for _ in 0..100 { + assert_eq!(prod.push(&pcm), Push::Ok); + } + let bound = Duration::from_millis(200); + prod.due(base + Duration::from_millis(2010), bound, Duration::ZERO); + assert_eq!(pull(&mut cons, &mut out, base), Read::Ok); + assert_eq!(pull(&mut cons, &mut out, base + Duration::from_millis(10)), Read::Ok); + let mut now = base + Duration::from_millis(20); + + // 100 ms ahead: within the bound, so audio keeps playing. + prod.due(base + Duration::from_millis(2110), bound, Duration::ZERO); + assert_eq!(pull(&mut cons, &mut out, now), Read::Ok); + assert!(out.iter().any(|sample| *sample != 0.0)); + now += Duration::from_millis(10); + + // 300 ms ahead: silence until the clock reaches the reader. + prod.due(base + Duration::from_millis(2310), bound, Duration::ZERO); + let available = cons.available(); + let lead = -cons.inputs(0).lag; + // The 10 ms it played since add to the 300. + assert!((14_400..=15_360).contains(&lead), "{lead} frames ahead"); + let mut padded = 0; + loop { + assert_eq!(pull(&mut cons, &mut out, now), Read::Ok); + now += Duration::from_millis(10); + if out.iter().any(|sample| *sample != 0.0) { + break; + } + padded += 1; + assert!(padded <= 32, "never caught up"); + } + assert!(padded >= 30, "padded {padded} reads"); + assert!(cons.available() <= available, "padding plays nothing written"); + assert_eq!(cons.skips(), 0); + assert!(cons.inputs(0).lag.abs() <= 480, "lag {}", cons.inputs(0).lag); + } + /// A zero stretch plays what was written unaltered: a lag under a window /// that the stretch would speed through stays, more skips onto the clock, /// and a reader more than a window ahead of it pads silence until the clock diff --git a/rs/moq-cli/src/play/args.rs b/rs/moq-cli/src/play/args.rs index 62e16704adf..c8583d00490 100644 --- a/rs/moq-cli/src/play/args.rs +++ b/rs/moq-cli/src/play/args.rs @@ -15,6 +15,11 @@ use moq_mux::container::DELAY_MAX; /// estimate, as when there is none: video has no estimator of its own. const AUTO_VIDEO_DELAY: Duration = Duration::from_millis(100); +/// How far past the shared delay audio is still waited on and observed: the +/// arrival estimate's 2 s ceiling, so every frame late enough to move the +/// target is measured. `--stretch` does not size it. +pub(super) const REACH: Duration = Duration::from_secs(2); + /// How far playback trails the live edge. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Delay { @@ -64,15 +69,14 @@ pub struct Args { #[usage(long, default = "auto")] pub delay: Delay, - /// How far audio may trail the clock before it skips ahead, or `0` to turn the time stretch off. + /// How far audio may trail the clock before it skips onto it, or `0` to turn the time stretch off. /// /// After a stall, audio slows down and falls behind the clock rather than dropping what - /// arrives late, then speeds up until it is back on it; only what trails by more than this - /// is skipped. Every subscription waits on a stalled group for the delay plus this, so the - /// late audio still arrives. Zero plays received audio unaltered: late audio is skipped, an - /// underrun is silence, and drift against the publisher's clock is corrected by a small - /// skip or pad. Video stays on the clock either way. - #[usage(long, default = "2s")] + /// arrives late, then speeds up until it is back on it; audio that trails by more than this + /// skips onto the clock, as video does. Zero plays received audio unaltered: late audio is + /// skipped, an underrun is silence, and drift against the publisher's clock is corrected by + /// a small skip or pad. Video stays on the clock either way. + #[usage(long, default = "200ms")] pub stretch: moq_tokio::cli::Duration, /// Rendition selection by track name or codec. @@ -119,13 +123,13 @@ impl Args { } /// How long a stalled audio group is waited on before it is skipped, while - /// the shared delay is still the configured one: the delay plus the stretch. + /// the shared delay is still the configured one: the delay plus [`REACH`]. /// A shared delay past it grows the budget by the excess. pub(super) fn max_delay(&self) -> Duration { - self.fixed_delay().unwrap_or_default() + self.stretch() + self.fixed_delay().unwrap_or_default() + REACH } - /// How far audio may trail the clock before it skips ahead; zero turns the time stretch off. + /// How far audio may trail the clock before it skips onto it; zero turns the time stretch off. pub(super) fn stretch(&self) -> Duration { self.stretch.into() } @@ -135,18 +139,6 @@ impl Args { self.fixed_delay().unwrap_or(AUTO_VIDEO_DELAY) } - /// How long a stalled video group is waited on, given the catalog's audio: - /// video's delay plus the stretch, as audio's. - /// - /// With no enabled speaker to follow, the playhead is video's own, so waiting - /// on audio's budget would only freeze the picture. - pub(super) fn video_max_delay(&self, audio: &hang::catalog::Audio) -> Duration { - match audio.enabled().next() { - Some(_) => self.video_delay() + self.stretch(), - None => self.video_delay(), - } - } - /// Reject a codec the local decoders can't open. /// /// The selection flags are shared with the stdout exports, which pass bytes @@ -242,7 +234,8 @@ mod tests { } } - /// A fixed delay is the playout offset, and with the stretch the staleness budget. + /// A fixed delay is the playout offset, and audio's staleness budget reaches + /// the estimator's ceiling past it, whatever the stretch. #[test] fn a_fixed_delay_sets_the_staleness_budget() { let args = parse(&["--delay", "500ms"]); @@ -250,40 +243,26 @@ mod tests { assert_eq!(args.max_delay(), Duration::from_millis(2500)); assert_eq!(args.video_delay(), Duration::from_millis(500)); - let args = parse(&["--delay", "500ms", "--stretch", "0"]); - assert_eq!(args.max_delay(), Duration::from_millis(500)); - let args = parse(&["--delay", "500ms", "--stretch", "200ms"]); - assert_eq!(args.max_delay(), Duration::from_millis(700)); + for stretch in ["0", "200ms", "5s"] { + let args = parse(&["--delay", "500ms", "--stretch", stretch]); + assert_eq!(args.max_delay(), Duration::from_millis(2500), "{stretch}"); + } } - /// The default measures the delay, and waits on a stalled group for the stretch past it. + /// The default measures the delay, skips audio past 200 ms onto the clock, + /// and waits on a stalled audio group for the estimator's ceiling. #[test] fn the_delay_defaults_to_the_estimate() { let args = parse(&[]); assert_eq!(args.delay, Delay::Auto); assert_eq!(args.fixed_delay(), None); - assert_eq!(args.stretch(), Duration::from_secs(2)); + assert_eq!(args.stretch(), Duration::from_millis(200)); assert_eq!(args.max_delay(), Duration::from_secs(2)); assert_eq!(parse(&["--delay", "auto"]).delay, Delay::Auto); assert_eq!(Delay::Auto.to_string(), "auto"); assert_eq!(Delay::Fixed(Duration::from_millis(250)).to_string(), "250ms"); } - /// Disabled audio sends nothing to follow, so video keeps its own budget. - #[test] - fn disabled_audio_leaves_video_its_own_budget() { - let args = parse(&[]); - let mut audio = hang::catalog::Audio::default(); - assert_eq!(args.video_max_delay(&audio), AUTO_VIDEO_DELAY); - - let config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); - audio.renditions.insert("audio".into(), config); - assert_eq!(args.video_max_delay(&audio), AUTO_VIDEO_DELAY + args.stretch()); - - audio.renditions.get_mut("audio").unwrap().enabled = false; - assert_eq!(args.video_max_delay(&audio), AUTO_VIDEO_DELAY); - } - /// The playout budget has one spelling because it always controls both /// presentation delay and media staleness. #[test] diff --git a/rs/moq-cli/src/play/headless.rs b/rs/moq-cli/src/play/headless.rs index 534a0d4889b..56bdabd08ea 100644 --- a/rs/moq-cli/src/play/headless.rs +++ b/rs/moq-cli/src/play/headless.rs @@ -149,7 +149,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: super::super::args::Delay::Auto, - stretch: std::time::Duration::from_secs(2).into(), + stretch: std::time::Duration::from_millis(200).into(), select: crate::subscribe::SelectArgs { video: Some(crate::subscribe::Mode::Off), ..Default::default() @@ -177,7 +177,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: super::super::args::Delay::Auto, - stretch: std::time::Duration::from_secs(2).into(), + stretch: std::time::Duration::from_millis(200).into(), select: crate::subscribe::SelectArgs { video: Some(crate::subscribe::Mode::Off), ..Default::default() @@ -208,7 +208,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: super::super::args::Delay::Auto, - stretch: std::time::Duration::from_secs(2).into(), + stretch: std::time::Duration::from_millis(200).into(), select: crate::subscribe::SelectArgs { video: Some(crate::subscribe::Mode::Off), ..Default::default() diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 288c2a7f465..4dd1cec0001 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -11,7 +11,7 @@ use moq_mux::catalog::{self, Stream}; // playout clock itself. use kio::clock::Instant; -use super::args::Args; +use super::args::{Args, REACH}; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, Track, joined}; use super::timeline::{AudioTimeline, Playhead}; @@ -34,7 +34,7 @@ const AUDIO_BUFFER_MIN: Duration = Duration::from_millis(50); /// How far past the latency it opened on the sink may be asked to hold. Its ring /// reaches three seconds past that latency, less a 30 ms processing window -/// (`Sink::set_latency`), so this leaves margin, and it covers the default stretch. +/// (`Sink::set_latency`), so this leaves margin, and it covers audio's reach past the delay. const SINK_ROOM: Duration = Duration::from_secs(2); /// Maximum sample frames per silence write, independent of the catalog rate. @@ -267,7 +267,7 @@ impl Media { }; // Nothing older than the playhead is worth presenting, so the delay // doubles as the staleness budget on the wire. - let max_delay = self.args.video_max_delay(&snapshot.audio); + let max_delay = self.args.video_delay(); // A decoder and its subscription, starting `lookback` groups before the live edge. let open = async |delay: moq_net::track::Delay, lookback: u64| { // The window's renderer refuses a GPU surface it cannot import, so @@ -629,8 +629,7 @@ async fn play_audio( let mut silence = Vec::new(); // The most the sink holds, within what its ring was sized for. A shared delay - // past it, such as a slow video encoder's or a measured target past a long - // stretch, stays encoded in the consumer until it is due instead, so the + // past it, such as a slow video encoder's, stays encoded in the consumer until it is due instead, so the // ring never grows with the delay. let cap = max_delay .max(AUDIO_BUFFER_MIN) @@ -675,8 +674,7 @@ async fn play_audio( let target = || schedule.delay().clamp(AUDIO_BUFFER_MIN, cap); // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches - // onto that clock, and skips only what trails it past the stretch, which - // the subscription still delivers. + // onto that clock, and skips onto it what trails it past the stretch. // The last anchor, as when its end was due, the end, and the delay. The // clock's reference is when media zero arrived, `at - end - delay`: one that // moved earlier jumped the clock ahead, which the sink skips rather than @@ -734,13 +732,14 @@ async fn play_audio( // Media went missing since the last frame, so the next one does not continue it. let mut lost = false; let mut ends = VecDeque::new(); - // The subscription reaches the stretch past the shared delay wherever that moves, never - // below what it opened with, so audio the sink may still catch up on arrives. + // The subscription reaches the estimator's ceiling past the shared delay wherever that + // moves, never below what it opened with, so every frame late enough to move the target + // is observed, whatever the stretch. let floor = consumer.options().delay.max.unwrap_or_default(); let mut asked = floor; loop { let seen = schedule.delay(); - let max = seen.saturating_add(stretch).max(floor); + let max = seen.saturating_add(REACH).max(floor); if max != asked { consumer.set_max_delay(max); asked = max; @@ -1282,7 +1281,7 @@ mod tests { let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); let mut options = moq_audio::decode::Options::new(); let jitter = Duration::from_millis(100); - // A budget past the ring, as a long `--stretch` asks for. + // A budget past the ring the sink was sized for. options.delay = moq_audio::decode::Delay { min: Some(jitter), max: Some(Duration::from_secs(5)), @@ -1658,10 +1657,9 @@ mod tests { recorder.played().iter().any(|part| part.sample == 0.5), "audio never resumed" ); - assert!( - recorder.latencies().len() >= 2, - "the late burst never started a fresh sink" - ); + // The burst lands within the reach, so it continues the sink rather than + // aging out into a hole; the real ring skips it onto the clock. + assert_eq!(recorder.latencies().len(), 1, "the late burst replaced the sink"); assert!(shown.len() > 50, "video stalled: {} pictures", shown.len()); for (timestamp, at) in shown { let due = start.saturating_add(delay).saturating_add(Duration::from(timestamp)); @@ -2053,12 +2051,13 @@ mod tests { producer.write(packet(index, 0.25)).unwrap(); kio::clock::sleep(PACKET_DURATION).await; } - // Video declares an end two seconds out, holding the audio replacement back. + // Video declares an end six seconds out, holding the audio replacement + // back past its age budget, the delay plus the estimator's 2 s reach. let video = presentation.track(moq_mux::container::Kind::Video); video.declare(); - video.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + video.end(moq_net::Timestamp::from_millis(6_000).unwrap()).unwrap(); producer.discontinuity().unwrap(); - const REPLACEMENT: u64 = 100; + const REPLACEMENT: u64 = 300; for index in 0..REPLACEMENT { producer.write(packet(5 + index, 0.5)).unwrap(); kio::clock::sleep(PACKET_DURATION).await; diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 687f76d877b..a6d936d9b81 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -324,10 +324,10 @@ player's own count, which its stats panel shows, is read alongside: early. Its `p50` is where audio rests, so latency spent to buy fewer underruns shows there; its `p95` includes audio trailing through a stall. Natively, the presentation clock `moq play` paces video from (`sync::Consumer::position`). -- `recover_ms`: the longest audio spent off its resting point, from the ring stalling, or audio - trailing the shared clock by more than one 20 ms bucket past the window's `av_lag_ms` p50, to - audio back within it. Reported, not budgeted: it is how long a stall costs, which a player may - trade against the audio it keeps. +- `recover_ms`: the longest audio took to return after a disturbance: from the ring stalling, or + audio more than one codec frame off where it rested (its median lag over the 5 s before), either + way, to audio back within a frame of that rest. Reported, not budgeted: it is how long a stall + costs, which a player may trade against the audio it keeps. - `converge_ms`: from first audio to the last time the target moved more than one 20 ms bucket from its final value. - `render_load`: Chromium's render capacity, the share of each quantum's budget the graph used. diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index dce4c96be65..1b763eda39f 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8).", + "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows.", "rows": [ { "runtime": "chromium", @@ -189,7 +189,7 @@ "player_skips_per_min": 24, "underruns_per_min": 270, "underrun_ms_per_min": 720, - "trimmed_ms_per_min": 2300, + "trimmed_ms_per_min": 3800, "stalled_share": 0.11, "silence_share": 0.11, "target_ms_p95": 850, @@ -204,7 +204,7 @@ "player_skips_per_min": 36, "underruns_per_min": 280, "underrun_ms_per_min": 740, - "trimmed_ms_per_min": 2400, + "trimmed_ms_per_min": 3700, "stalled_share": 0.078, "silence_share": 0.082, "target_ms_p95": 1300, @@ -399,7 +399,7 @@ "player_skips_per_min": 7.6, "underruns_per_min": 1200, "underrun_ms_per_min": 3200, - "trimmed_ms_per_min": 3000, + "trimmed_ms_per_min": 3900, "stalled_share": 0.01, "silence_share": 0.062, "target_ms_p95": 2600, @@ -411,10 +411,10 @@ "codec": "opus", "rate": 48000, "profile": "congestion", - "player_skips_per_min": 7.6, + "player_skips_per_min": 15, "underruns_per_min": 1400, "underrun_ms_per_min": 3700, - "trimmed_ms_per_min": 2500, + "trimmed_ms_per_min": 4600, "stalled_share": 0.01, "silence_share": 0.067, "target_ms_p95": 2600, @@ -427,15 +427,15 @@ "rate": 44100, "profile": "bursty-aac", "player_underruns_per_min": 4.4, - "player_skips_per_min": 0, + "player_skips_per_min": 1.1, "underruns_per_min": 0, "underrun_ms_per_min": 0, - "trimmed_ms_per_min": 0, + "trimmed_ms_per_min": 312.7, "stalled_share": 0.005, "silence_share": 0.007, "target_ms_p95": 343.2, - "av_lag_ms_p50": 11, - "av_lag_ms_p95": 102.7 + "av_lag_ms_p50": 10, + "av_lag_ms_p95": 49.2 }, { "runtime": "replay", @@ -482,8 +482,8 @@ "stalled_share": 0, "silence_share": 0, "target_ms_p95": 80, - "av_lag_ms_p50": 1.1, - "av_lag_ms_p95": 3.4 + "av_lag_ms_p50": 2.2, + "av_lag_ms_p95": 4.5 }, { "runtime": "replay", @@ -491,15 +491,15 @@ "rate": 48000, "profile": "relay-mic-rtt40", "player_underruns_per_min": 3.1, - "player_skips_per_min": 0, + "player_skips_per_min": 0.3, "underruns_per_min": 0.7, "underrun_ms_per_min": 0.3, - "trimmed_ms_per_min": 0, + "trimmed_ms_per_min": 83.3, "stalled_share": 0, "silence_share": 0, "target_ms_p95": 320, "av_lag_ms_p50": 0.9, - "av_lag_ms_p95": 3.5 + "av_lag_ms_p95": 2.9 }, { "runtime": "replay", @@ -507,15 +507,15 @@ "rate": 48000, "profile": "relay-mic-stall", "player_underruns_per_min": 4, - "player_skips_per_min": 0, + "player_skips_per_min": 6.1, "underruns_per_min": 0, "underrun_ms_per_min": 0, - "trimmed_ms_per_min": 0, - "stalled_share": 0.05, - "silence_share": 0.049, + "trimmed_ms_per_min": 4678.2, + "stalled_share": 0.067, + "silence_share": 0.085, "target_ms_p95": 1540, - "av_lag_ms_p50": 1.1, - "av_lag_ms_p95": 484.9 + "av_lag_ms_p50": 2.2, + "av_lag_ms_p95": 118.6 } ] } diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts index 9dd3c279d28..b5179e56bd7 100644 --- a/test/audio-quality/clients/js/src/analyze.test.ts +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { analyze, type Input, longestAway, percentile } from "./analyze.ts"; +import { analyze, type Input, percentile, recovery } from "./analyze.ts"; import type { Sample, Shaper } from "./schema.ts"; const row = { runtime: "chromium", codec: "opus", rate: 48000, profile: "mild" } as const; @@ -33,12 +33,28 @@ const input = (samples: Sample[], extra: Partial = {}): Input => ({ ...extra, }); -test("the longest time away runs from the first away sample to the next one back", () => { - const at = (away: boolean[]) => away.map((a, i) => ({ at: i * 250, away: a })); - expect(longestAway(at([false, true, true, false, true, false]))).toBe(500); - // Still away at the end: it lasts to the last sample. - expect(longestAway(at([false, false, true, true, true]))).toBe(500); - expect(longestAway(at([false, false]))).toBe(0); +test("recovery runs from a disturbance to audio back within a frame of where it rested before it", () => { + const at = (lags: (number | "stalled")[]) => + lags.map((lag, i) => ({ + at: i * 250, + lag: lag === "stalled" ? undefined : lag, + stalled: lag === "stalled", + })); + // Resting 30 ms behind, jittering within a frame, then a stall and a trail that recovers. + const rest = [30, 40, 25, 35, 30, 30]; + expect(recovery(at([...rest, "stalled", 400, 200, 60, 45, 30]), 20)).toBe(1000); + // Ahead of where it rested counts too, as after a target rise. + expect(recovery(at([...rest, -70, -20, 15, 30]), 20)).toBe(500); + // Settling somewhere new never returns: it lasts to the last sample. + expect(recovery(at([...rest, 200, 200, 200]), 20)).toBe(500); + expect(recovery(at(rest), 20)).toBe(0); + // Nothing before the first sample to rest on. + expect(recovery(at(["stalled"]), 20)).toBeNull(); + // What precedes `from` only sets the rest, so a disturbance right at it is still measured. + expect(recovery(at([...rest, "stalled", 200, 30]), 20, 1500)).toBe(500); + expect(recovery(at(["stalled", 30, ...rest]), 20, 500)).toBe(0); + // Settling somewhere new before `from` is the tune-in: the rest at `from` is where it settled. + expect(recovery(at([0, 0, 100, 100, 100, 100, 100, 100, "stalled", 300, 100]), 20, 1500)).toBe(500); }); test("percentile is nearest rank", () => { @@ -125,7 +141,7 @@ describe("analyze", () => { expect(analyze(input(samples)).metrics.underruns_total).toBe(0); }); - test("recovery is the longest audio spent stalled or trailing its resting lag", () => { + test("recovery is the longest audio spent off where it rested before a disturbance", () => { // At rest 20 ms ahead of the clock; samples 80 to 83 stall, and audio then trails until 91. const samples = clean().map((s, i) => { const timestamp = s.timestamp ?? 0; @@ -133,7 +149,7 @@ describe("analyze", () => { const lag = i >= 84 && i < 92 ? 300 - (i - 84) * 30 : -20; return { ...s, clock: timestamp + lag }; }); - // 300 down to 90 ms trail by more than a bucket past rest; 92 is back. + // 300 down to 90 ms trail by more than a frame past rest; 92 is back. expect(analyze(input(samples)).metrics).toMatchObject({ av_lag_ms_p50: -20, recover_ms_max: 12 * 250 }); expect(analyze(input(clean())).metrics.recover_ms_max).toBeNull(); }); diff --git a/test/audio-quality/clients/js/src/analyze.ts b/test/audio-quality/clients/js/src/analyze.ts index 8f443e1df9f..ec043fd1eb9 100644 --- a/test/audio-quality/clients/js/src/analyze.ts +++ b/test/audio-quality/clients/js/src/analyze.ts @@ -76,22 +76,47 @@ export function slope(xs: number[], ys: number[]): number | null { return den === 0 ? null : num / den; } +/** How far before a disturbance audio's resting point is read from. */ +export const REST_MS = 5000; + +/** One codec frame of a row, the tolerance for audio being back where it rested. */ +export const frameMs = (row: Row): Ms => (row.codec === "aac" ? (1024 / row.rate) * 1000 : BUCKET_MS); + /** - * The longest run of samples spent away from rest, in ms: from the first away sample to the next - * one back. A run still open at the end lasts to the last sample. + * The longest time audio took to return after a disturbance, in ms, or null when it never had a + * resting point to leave. Audio rests at the median lag of the {@link REST_MS} before each sample, + * leaving out earlier disturbances; one starts at the first sample stalled or more than `frame` off + * that rest, either way, and ends at the next sample back within `frame` of it. One still open at + * the end lasts to the last sample, so a later one, always shorter, cannot raise the longest. Samples + * before `from` only set the rest, so where audio settled during a tune-in is its rest at `from`. */ -export function longestAway(samples: { at: Ms; away: boolean }[]): Ms { - let longest = 0; - let since: Ms | undefined; - for (const { at, away } of samples) { - if (away) since ??= at; - else if (since !== undefined) { - longest = Math.max(longest, at - since); - since = undefined; - } +export function recovery( + samples: { at: Ms; lag: Ms | undefined; stalled: boolean }[], + frame: Ms, + from: Ms = Number.NEGATIVE_INFINITY, +): Ms | null { + type Point = (typeof samples)[number]; + // Samples inside a disturbance, which say nothing about where audio rests. + const disturbed = new Set(); + let longest: Ms | null = null; + for (let i = samples.findIndex((s) => s.at >= from); i >= 0 && i < samples.length; i++) { + const onset = samples[i] as Point; + const before = samples.flatMap((s, j) => + s.at >= onset.at - REST_MS && j < i && !disturbed.has(j) && !s.stalled && s.lag !== undefined + ? [s.lag] + : [], + ); + const rest = median(before); + if (rest === undefined) continue; + longest ??= 0; + const back = (s: Point) => !s.stalled && s.lag !== undefined && Math.abs(s.lag - rest) <= frame; + if (back(onset) || (onset.lag === undefined && !onset.stalled)) continue; + let end = samples.findIndex((s, j) => j > i && back(s)); + if (end < 0) end = samples.length - 1; + for (let j = i; j < end; j++) disturbed.add(j); + longest = Math.max(longest, (samples[end] as Point).at - onset.at); + i = end; } - const end = samples.at(-1)?.at; - if (since !== undefined && end !== undefined) longest = Math.max(longest, end - since); return longest; } @@ -216,18 +241,17 @@ export function analyze(input: Input): Summary { const lag = lagOf(s); return lag === undefined ? [] : [lag]; }); - // Rest is the window's median lag, not zero: a player may settle a little either side of the clock, - // and how far is `av_lag_ms_p50`'s to grade. A stalled ring is off the clock however far it ran. - const rest = percentile(behind, 50); - const recover = - rest === null - ? null - : longestAway( - window.map((s) => { - const lag = s.stalled ? undefined : lagOf(s); - return { at: s.at, away: s.stalled === true || (lag !== undefined && lag > rest + BUCKET_MS) }; - }), - ); + // Measured against where audio rested before each disturbance, not zero: a player may settle a + // little either side of the clock, and how far is `av_lag_ms_p50`'s to grade. + // The warmup's last `REST_MS` sets where audio rests going into the window, so a disturbance at its + // start is measured too. + const recover = recovery( + samples + .filter((s) => s.at >= t0 - REST_MS && s.at <= t1) + .map((s) => ({ at: s.at, lag: s.stalled ? undefined : lagOf(s), stalled: s.stalled === true })), + frameMs(row), + t0, + ); const stalledShare = window.length > 0 ? window.filter((s) => s.stalled).length / window.length : null; // ── the target ────────────────────────────────────────────────────────── diff --git a/test/audio-quality/clients/js/src/capture.test.ts b/test/audio-quality/clients/js/src/capture.test.ts index 5d53de50f41..38f8bb1a9d7 100644 --- a/test/audio-quality/clients/js/src/capture.test.ts +++ b/test/audio-quality/clients/js/src/capture.test.ts @@ -90,18 +90,18 @@ test("the observer shares playback demand and closes without ending playback", a } as Catalog.AudioConfig), delay: new Signal(Duration.millis(250)), maxDelay: new Signal(Duration.millis(250)), - stretch: new Signal(Duration.ZERO), }); try { while (!track.subscription.peek()) await track.subscription.changed(); + // The player's reach: the delay plus the estimator's 2 s ceiling. expect(track.subscription.peek()).toMatchObject({ priority: Catalog.PRIORITY.audio, - delay: { max: Duration.millis(250) }, + delay: { max: Duration.millis(2250) }, }); const player = published .consume() .track("audio") - .subscribe({ priority: Catalog.PRIORITY.audio, delay: { max: Duration.millis(250) } }); + .subscribe({ priority: Catalog.PRIORITY.audio, delay: { max: Duration.millis(2250) } }); await capture.close(); expect(player.closed.peek()).toBeUndefined(); player.close(); @@ -138,7 +138,6 @@ test("the observer asks for the decoder's delay plus its stretch and warmup, not } as Catalog.AudioConfig), delay: new Signal(Duration.millis(100)), maxDelay: new Signal(Duration.millis(100)), - stretch: new Signal(Duration.secs(2)), }); try { while (!track.subscription.peek()) await track.subscription.changed(); @@ -169,7 +168,6 @@ async function captureGroupEnd(end: (group: Group.Producer) => void): Promise; maxDelay: Getter; - stretch: Getter; }; /** Record frames on the player's connection with its subscription settings. */ @@ -50,7 +49,6 @@ export class Capture { max: maxDelay({ delay: input.delay.peek(), maxDelay: input.maxDelay.peek(), - stretch: input.stretch.peek(), }).add(Duration.millis(config.warmup ?? 0)), }, }); @@ -66,7 +64,6 @@ export class Capture { max: maxDelay({ delay: inner.get(input.delay), maxDelay: inner.get(input.maxDelay), - stretch: inner.get(input.stretch), }).add(Duration.millis(inner.get(input.config)?.warmup ?? 0)), }, }), diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 60d416e3074..6d95173576e 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -58,7 +58,6 @@ export function probe(watch: MoqWatch, capture = false): Probe { config: audio.source.out.config, delay: sync.out.delay, maxDelay: sync.out.maxDelay, - stretch: sync.in.stretch, }) : undefined; diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index 488cfdf8ec7..c4bd7fc2b89 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -138,7 +138,6 @@ const capture = new Capture({ config: audio.source.out.config, delay: watch.sync.out.delay, maxDelay: watch.sync.out.maxDelay, - stretch: watch.sync.in.stretch, }); const arrivals: Arrival[] = []; let attached: AudioNode | undefined; diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index a28b3af79ef..0028602aa98 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -150,7 +150,7 @@ export const METRICS = { clock: "viewer", aggregations: ["max"], description: - "The longest audio spent off its resting point: from the ring stalling, or audio trailing the shared clock by more than one bucket past the window's median lag, to audio back within it.", + "The longest audio took to return after a disturbance: from the ring stalling, or audio more than one codec frame off the median lag of the 5 s before, to audio back within a frame of it.", }, target_ms: { unit: "ms",