Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions doc/bin/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,11 @@ a track that stalls plays late and catches up by playing faster, rather than
holding the other one back.
Each role follows the catalog, switching rendition when the publisher retires
the one playing. With `--video off`, `--metrics <PATH>` writes a JSON line every
250 ms with the playhead, the delay, and every gap and stall the speaker hit, in
250 ms with the playhead, the delay, every gap and stall the speaker hit, and
the media it trimmed to bound its delay or shed by playing faster, in
the schema the
[audio quality harness](https://github.com/moq-dev/moq/tree/main/test/audio-quality)
reduces to underruns, skips, and silence. A silence the publisher declares, such
reduces to underruns, trims, and silence. A silence the publisher declares, such
as a push-to-talk mute, plays as comfort noise at the level of the quietest audio
heard before it, and counts as no gap or stall. The metrics total that comfort
noise and name the media timestamp each utterance after it resumes from.
Expand Down
2 changes: 2 additions & 0 deletions js/watch/api/index.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ export namespace Audio {
track: Signals.Signal<string | undefined>;
};
export interface Stats {
accelerated: Clock.Milli;
arrival?: {
timestamp: Clock.Milli;
wall: Clock.Wall;
Expand All @@ -94,6 +95,7 @@ export namespace Audio {
skips: number;
stalled: Clock.Milli;
stalls: number;
trimmed: Clock.Milli;
underruns: number;
}
export type Supported = (config: Hang.Catalog.AudioConfig) => Promise<boolean>;
Expand Down
4 changes: 4 additions & 0 deletions js/watch/src/audio/buffer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ function open(): {
anchored: timestamp !== undefined,
underruns: 0,
skips: 0,
trimmed: Milli.zero,
accelerated: Milli.zero,
silent: false,
epoch,
};
Expand Down Expand Up @@ -441,6 +443,8 @@ function fake(delay: Milli, buffered = false) {
anchored: true,
underruns: 0,
skips: 0,
trimmed: Milli.zero,
accelerated: Milli.zero,
silent: false,
epoch,
};
Expand Down
10 changes: 10 additions & 0 deletions js/watch/src/audio/buffer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,14 @@ export class AudioBuffer {
/** How many times the ring skipped ahead, dropping queued audio, cumulative. */
readonly skips: Getter<number> = this.#skips;

readonly #trimmed = new Signal<Milli>(Milli.zero);
/** Media the ring's skips dropped unplayed, less declared silence, cumulative. */
readonly trimmed: Getter<Milli> = this.#trimmed;

readonly #accelerated = new Signal<Milli>(Milli.zero);
/** Media the stretch removed by playing faster, cumulative. */
readonly accelerated: Getter<Milli> = this.#accelerated;

readonly #silent = new Signal<boolean>(false);
/** Whether playback is at a declared end, reached or announced before any audio, and nothing has played since. */
readonly silent: Getter<boolean> = this.#silent;
Expand Down Expand Up @@ -338,6 +346,8 @@ export class AudioBuffer {
this.#stalled.set(data.stalled);
this.#underruns.set(data.underruns);
this.#skips.set(data.skips);
this.#trimmed.set(data.trimmed);
this.#accelerated.set(data.accelerated);
this.#silent.set(data.silent);
if (data.epoch === this.#epoch) {
this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled);
Expand Down
2 changes: 2 additions & 0 deletions js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ class FakeWorklet {
anchored: ring.anchored,
underruns: ring.underruns,
skips: ring.skips,
trimmed: ring.trimmed,
accelerated: ring.accelerated,
silent: ring.silent,
epoch: this.#epoch,
};
Expand Down
33 changes: 32 additions & 1 deletion js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ export interface Stats {
underruns: number;
/** How many times playback skipped ahead, dropping buffered audio rather than playing it faster. */
skips: number;
/** Media those skips dropped unplayed, less declared silence. */
trimmed: Milli;
/** Media the time stretch removed by playing faster. */
accelerated: Milli;
/** Frames that arrived after their playout time. */
late: number;
/** Renditions that failed to decode. */
Expand All @@ -106,7 +110,17 @@ export interface Stats {
}

// The running totals before anything is counted.
const EMPTY: Stats = { bytesReceived: 0, underruns: 0, skips: 0, late: 0, errors: 0, stalls: 0, stalled: Milli.zero };
const EMPTY: Stats = {
bytesReceived: 0,
underruns: 0,
skips: 0,
trimmed: Milli.zero,
accelerated: Milli.zero,
late: 0,
errors: 0,
stalls: 0,
stalled: Milli.zero,
};

// What a decode attempt reads, and where its counts go.
type Attempt = { config: DecoderConfig; failure: Signal<Error | undefined>; path: Moq.Path.Valid; key: Key };
Expand Down Expand Up @@ -427,6 +441,23 @@ export class Decoder {
if (added <= 0) return;
this.#count(undefined, {}, (stats) => ({ skips: stats.skips + added }));
});
// Each ring's totals start at zero too, so carry each forward by its rise.
let trimmed = Milli.zero;
let accelerated = Milli.zero;
effect.run((inner) => {
const total = inner.get(ring.trimmed);
const added = total - trimmed;
trimmed = total;
if (added > 0) this.#count(undefined, {}, (stats) => ({ trimmed: Milli(stats.trimmed + added) }));
});
effect.run((inner) => {
const total = inner.get(ring.accelerated);
const added = total - accelerated;
accelerated = total;
if (added > 0) {
this.#count(undefined, {}, (stats) => ({ accelerated: Milli(stats.accelerated + added) }));
}
});
Comment thread
kixelated marked this conversation as resolved.
Resolved

effect.set(this.#out.root, worklet);
});
Expand Down
4 changes: 4 additions & 0 deletions js/watch/src/audio/render-worklet.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ describe("render dispatch", () => {
anchored: true,
underruns: 0,
skips: 0,
trimmed: Milli.zero,
accelerated: Milli.zero,
silent: false,
},
]);
Expand Down Expand Up @@ -115,6 +117,8 @@ describe("render dispatch", () => {
anchored: true,
underruns: 0,
skips: 0,
trimmed: Milli.zero,
accelerated: Milli.zero,
silent: false,
},
]);
Expand Down
2 changes: 2 additions & 0 deletions js/watch/src/audio/render-worklet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ class Render extends AudioWorkletProcessor {
anchored: ring.anchored,
underruns: ring.underruns,
skips: ring.skips,
trimmed: ring.trimmed,
accelerated: ring.accelerated,
silent: ring.silent,
epoch: this.#epoch,
};
Expand Down
4 changes: 4 additions & 0 deletions js/watch/src/audio/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ export interface State {
underruns: number;
// How many times the ring skipped ahead, dropping queued audio, cumulative.
skips: number;
// Media those skips dropped unplayed, less declared silence, cumulative.
trimmed: Milli;
// Media the stretch removed by playing faster, cumulative.
accelerated: Milli;
// Whether playback is at a declared end, reached or announced before any audio, and nothing has played since.
silent: boolean;
// The epoch of the last reset or park the worklet applied.
Expand Down
6 changes: 6 additions & 0 deletions js/watch/src/audio/replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ export interface Quantum {
underruns: number;
/** The ring's cumulative skips, as the player's stats count them: each jump that dropped queued audio. */
skips: number;
/** The received audio those skips dropped, cumulative, in ms. */
trimmed: number;
/** The received audio the stretch removed by playing faster, cumulative, in ms. */
accelerated: number;
Comment thread
kixelated marked this conversation as resolved.
Resolved
}

/** Samples as the milliseconds `AudioBuffer` posts to the worklet. */
Expand Down Expand Up @@ -285,6 +289,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
delay,
underruns: ring.underruns,
skips: ring.skips,
trimmed: ring.trimmed,
accelerated: ring.accelerated,
};
}
} finally {
Expand Down
51 changes: 51 additions & 0 deletions js/watch/src/audio/ring-buffer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,9 @@ describe("time stretch", () => {
settled(peaks, 100);
// Every sample written either played or is still buffered, so nothing was skipped.
expect(buffer.skips).toBe(0);
expect(buffer.trimmed).toBe(Milli.zero);
// At least the 200 ms it started above the floor, less the period it may rest above it.
expect(buffer.accelerated).toBeGreaterThan(200 - 2.5);
expect(Math.round((Milli.fromMicro(buffer.timestamp) * RATE) / 1000) + buffer.length).toBeGreaterThan(
written - frames(30),
);
Expand Down Expand Up @@ -964,11 +967,59 @@ describe("time stretch", () => {
expect(buffer.skips).toBe(1);
expect(buffer.length).toBe(frames(100));
expect(Milli.fromMicro(buffer.timestamp)).toBeCloseTo(400, 6);
// Everything before the new playhead but the quantum that played.
expect(buffer.trimmed).toBeCloseTo(400 - (QUANTUM * 1000) / RATE, 6);
expect(buffer.accelerated).toBe(Milli.zero);
const [output] = read(buffer, QUANTUM, 1);
expect(output.length).toBe(QUANTUM);
expect(output.every((sample) => sample === 0.5)).toBe(true);
});

it("trims a gap's zeros as media", () => {
const buffer = live(100);
write(buffer, 0 as Milli, frames(100), { value: 0.25 });
read(buffer, QUANTUM, 1);
// A 100 ms gap, then enough to cross the 480 ms fast threshold and re-anchor at 410 ms.
write(buffer, 200 as Milli, frames(10), { value: 0.5 });
write(buffer, 210 as Milli, frames(300), { value: 0.5 });
expect(buffer.skips).toBe(1);
expect(Milli.fromMicro(buffer.timestamp)).toBeCloseTo(410, 6);
expect(buffer.trimmed).toBeCloseTo(410 - (QUANTUM * 1000) / RATE, 6);
});

it("trims no declared silence", () => {
const buffer = live(100);
write(buffer, 0 as Milli, frames(100), { value: 0.25 });
read(buffer, QUANTUM, 1);
// A declared silence over [100, 200) ms, then more than the ring's 1 s capacity, which skips to
// 250 ms through the silence and the incoming chunk's first 50 ms.
buffer.end(Micro(100_000));
buffer.next();
write(buffer, 200 as Milli, frames(1050), { value: 0.5 });
expect(buffer.skips).toBe(1);
expect(Milli.fromMicro(buffer.timestamp)).toBeCloseTo(250, 6);
expect(buffer.trimmed).toBeCloseTo(250 - 100 - (QUANTUM * 1000) / RATE, 6);
});

it("trims only what the ring held, never the gap a re-anchor jumps", () => {
const buffer = live(100);
write(buffer, 0 as Milli, frames(100), { value: 0.25 });
read(buffer, QUANTUM, 1);
// A group past its age budget: the next audio lands 2 s on, past the ring's capacity.
write(buffer, 2000 as Milli, frames(20), { value: 0.5 });
expect(buffer.skips).toBe(1);
expect(buffer.trimmed).toBeCloseTo(100 - (QUANTUM * 1000) / RATE, 6);
});

it("counts nothing when an empty ring re-anchors", () => {
const buffer = live(100);
write(buffer, 0 as Milli, frames(100), { value: 0.25 });
while (buffer.length > 0) read(buffer, QUANTUM, 1);
write(buffer, 3000 as Milli, frames(20), { value: 0.5 });
expect(buffer.skips).toBe(0);
expect(buffer.trimmed).toBe(Milli.zero);
});

it("counts a re-anchor that drops only the staged window as a skip", () => {
const buffer = live(100);
// Four whole 30 ms windows, so once the playhead passes 90 ms the ring is empty and only the
Expand Down
47 changes: 41 additions & 6 deletions js/watch/src/audio/ring-buffer.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Micro, type Milli, Second } from "@moq/clock";
import { Micro, Milli, Second } from "@moq/clock";
import { ringCapacity } from "./latency";
import { Stretch } from "./stretch";

Expand All @@ -14,6 +14,10 @@ export class AudioRingBuffer {
#parked = false;
#underruns = 0;
#skips = 0;
// Samples of media a skip dropped, and that the stretch removed by accelerating, gap zeros
// included and declared silence excluded, as moq-audio counts them.
#trimmed = 0;
#accelerated = 0;

// Where the publisher declared its audio ends, as a sample index, until the next reset.
#end?: number;
Expand Down Expand Up @@ -105,6 +109,16 @@ export class AudioRingBuffer {
return this.#skips;
}

/** Media the ring's skips dropped unplayed, less declared silence, cumulative. */
get trimmed(): Milli {
return Milli.fromSecond((this.#trimmed / this.rate) as Second);
}

/** Media the stretch removed by playing faster, cumulative. */
get accelerated(): Milli {
return Milli.fromSecond((this.#accelerated / this.rate) as Second);
}

/** Whether playback is at a declared end, reached or announced before any audio, and nothing has played since. */
get silent(): boolean {
return this.#silent;
Expand Down Expand Up @@ -227,7 +241,7 @@ export class AudioRingBuffer {
const latency = this.#latencySamples;
const queued = end - this.#readIndex + this.#stretch.available;
if (end - this.#readIndex > this.capacity) {
this.#skip(end - this.capacity);
this.#skip(end - this.capacity, start);
} else if (band && pause && pause.start <= this.#readIndex && queued > latency + this.#chunk) {
// Skip only the silence past the queued floor, which the staged window counts toward, so the
// read index only moves forward and never re-stages PCM the window already holds.
Expand All @@ -241,7 +255,7 @@ export class AudioRingBuffer {
latency + Math.floor(((this.capacity - latency) * 3) / 4),
)
) {
this.#skip(end - latency);
this.#skip(end - latency, start);
}

// Fill gaps with zeros if there's a discontinuity
Expand Down Expand Up @@ -328,9 +342,14 @@ export class AudioRingBuffer {
if (!this.#anchored || (this.#stalled && this.length <= 0 && this.#ended())) this.#silent = true;
}

// Move the read index forward to `index`, dropping any queued audio it passes, staged included.
#skip(index: number): void {
if (index > this.#readIndex && this.length > 0) this.#skips++;
// Move the read index forward to `index`, dropping any queued audio it passes, staged included,
// and the front of the incoming chunk at `start` when `index` lies past it. An empty ring only
// re-anchors over a gap it never held: neither counter moves, as in moq-audio.
#skip(index: number, start: number): void {
if (index > this.#readIndex && this.length > 0) {
this.#skips++;
this.#trimmed += this.#discarded(index, start);
}
// The staged window lies behind the new index, so it is dropped with the rest, as moq-audio's
// trim discards staged output first.
const length = this.length;
Expand All @@ -343,6 +362,21 @@ export class AudioRingBuffer {
this.#prune();
}

// Media a skip to `index` drops: the unplayed share of the staged window, what the ring holds up
// to `index`, and the incoming chunk's front below it, less declared silence. The gap between the
// write index and the chunk at `start` was never held, so it is no trim.
#discarded(index: number, start: number): number {
const stretch = this.#stretch;
const held = Math.min(index, this.#writeIndex);
const front = Math.max(start, this.#writeIndex);
let dropped = held - this.#readIndex + Math.max(0, index - front);
if (stretch.available > 0 && !this.#windowQuiet) dropped += (stretch.available * this.#window) / stretch.length;
for (const pause of this.#pauses) {
dropped -= Math.max(0, Math.min(held, pause.end) - Math.max(this.#readIndex, pause.start));
}
return dropped;
}

// Forget the declared silences the playhead has passed.
#prune(): void {
const pauses = this.#pauses;
Expand Down Expand Up @@ -462,6 +496,7 @@ export class AudioRingBuffer {
const stretch = this.#stretch;
stretch.process(frames, this.#peak - this.#latencySamples, enabled);
this.#peak -= frames - stretch.length;
this.#accelerated += Math.max(0, frames - stretch.length);
this.#readIndex += frames;
this.#window = frames;
this.#windowQuiet = quiet;
Expand Down
6 changes: 6 additions & 0 deletions quest/m1/audio-jitter-target/browser-ceilings.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ it, as is an `av_lag_ms` p50 above zero, which is audio settling behind the
clock. Rewrite the note's browser sentences to the new nights and say the stretch
is in them.

[moq-dev/moq#8308](https://github.com/moq-dev/moq/pull/8308) adds the
player's own `trimmed_ms` and `accelerated_ms` to every row. The native
calibration budgets `trimmed_ms_per_min` beside `player_skips_per_min`, as the
maintainer decided there; budgeting it here too keeps the two runtimes graded
alike.

## Related

- [Capped budgets](/quest/m1/audio-jitter-target/capped-budgets.md) - re-measures the AAC browser rows from nightlies too; the same seven nights can serve both
Expand Down
Loading
Loading