Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 77 additions & 2 deletions js/watch/src/audio/replay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ const CONFIG = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catal

/** Twenty ms frames for `seconds`, one per group, each arriving `lateness(i)` ms after it was captured. */
function paced(seconds: number, lateness: (i: number) => number): Arrival[] {
return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20, group: i }));
return Array.from({ length: seconds * 50 }, (_, i) => ({
at: i * 20 + lateness(i),
timestamp: i * 20,
group: i,
duration: 20,
}));
}

/** How long {@link paced} observes `seconds` of frames: the frames, plus one more. */
Expand Down Expand Up @@ -137,7 +142,7 @@ describe.each(["shared", "post"] as const)("%s ring at auto", (ring) => {

describe.each(["shared", "post"] as const)("%s ingress diagnostics", (ring) => {
it("keeps recorded arrival and lateness when delivery is held", async () => {
const trace = [0, 20, 40].map((timestamp) => ({ timestamp, at: timestamp + 1000, group: 0 }));
const trace = [0, 20, 40].map((timestamp) => ({ timestamp, at: timestamp + 1000, group: 0, duration: 20 }));
const run = async (decodeAfter?: number) => {
let window: Quantum["arrivals"] = [];
for await (const quantum of replay(trace, {
Expand All @@ -164,3 +169,73 @@ describe.each(["shared", "post"] as const)("%s ingress diagnostics", (ring) => {
);
});
});

describe.each(["shared", "post"] as const)("%s recorded frame durations", (ring) => {
it("keeps missing packets out of the codec term even when most packets are absent", async () => {
const trace = Array.from({ length: 600 }, (_, i) => ({
at: i * 100,
timestamp: i * 100,
group: 0,
duration: 20,
}));
let delay = 0;
for await (const quantum of replay(trace, {
ring,
rate: RATE,
config: CONFIG,
delay: "auto",
duration: 60_000,
}))
delay = quantum.delay;
expect(delay).toBe(40);
});

it("updates the codec term when the packet duration changes", async () => {
const trace = Array.from({ length: 600 }, (_, i) => ({
at: i * 100,
timestamp: i * 100,
group: 0,
duration: i < 300 ? 20 : 10,
}));
const targets: number[] = [];
for await (const quantum of replay(trace, {
ring,
rate: RATE,
config: CONFIG,
delay: "auto",
duration: 60_000,
})) {
if (quantum.at >= 29_000 && quantum.at <= 29_100) targets.push(quantum.delay);
if (quantum.at >= 59_000) targets.push(quantum.delay);
}
expect(new Set(targets)).toEqual(new Set([40, 30]));
});

it("plays a single packet for its recorded duration", async () => {
const trace = [{ at: 0, timestamp: 0, group: 0, duration: 20 }];
let heard = 0;
for await (const { output } of replay(trace, {
ring,
rate: RATE,
config: CONFIG,
delay: Milli(20),
duration: 100,
}))
heard += output.filter((sample) => sample !== 0).length;
expect(heard).toBe(960);
});
});

it("replay refuses missing, non-finite or non-positive recorded durations", async () => {
for (const duration of [undefined, Number.NaN, Number.POSITIVE_INFINITY, 0, -20]) {
const trace = [{ at: 0, timestamp: 0, group: 0, duration }] as Arrival[];
const reader = replay(trace, {
ring: "shared",
rate: RATE,
config: CONFIG,
delay: "auto",
duration: 100,
});
await expect(reader.next()).rejects.toThrow("invalid recorded frame duration");
}
});
49 changes: 17 additions & 32 deletions js/watch/src/audio/replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { Manual, type Micro, Milli, Nano, type Second } from "@moq/clock";
import type * as Catalog from "@moq/hang/catalog";
import * as Container from "@moq/hang/container";
import { Group, Time, Track } from "@moq/net";
import { Effect } from "@moq/signals";
import { Effect, Signal } from "@moq/signals";
import { type Arrival as Diagnostic, gap } from "../arrivals";
import { type Delay, Sync } from "../sync";
import { frameDuration } from "./config";
Expand Down Expand Up @@ -49,6 +49,8 @@ export interface Arrival {
timestamp: number;
/** The group that carried it. */
group: number;
/** The encoded frame's duration, in ms, measured before recording discards its payload. */
duration: number;
}

/** What a replay plays through. */
Expand Down Expand Up @@ -140,29 +142,6 @@ function post(rate: number, latency: number): Ring {
/** A trace's media timestamp as the container carries it. */
const micros = (ms: number) => Math.round(ms * 1000) as Micro;

/**
* Samples each frame carries, by its timestamp: the trace's typical frame, the median spacing.
*
* A frame within half a frame of that runs up to where the next one starts instead, so consecutive
* frames tile the timeline exactly despite rounding. Any wider spacing is a frame that never
* arrived, which stays missing audio rather than stretching the frame before it.
*/
function frameSamples(trace: Arrival[], rate: number): { spans: Map<Micro, number>; typical: number } {
const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b);
const sample = (ms: number) => Math.round((ms * rate) / 1000);
const gaps = starts.slice(1).map((next, i) => sample(next) - sample(starts[i]));
const typical = [...gaps].sort((a, b) => a - b)[Math.floor(gaps.length / 2)];
if (typical === undefined) throw new Error("a trace needs two frames");

const spans = new Map<Micro, number>();
starts.forEach((start, i) => {
const gap = gaps[i];
const tiles = gap !== undefined && Math.abs(gap - typical) * 2 <= typical;
spans.set(micros(start), tiles ? gap : typical);
});
return { spans, typical };
}

/**
* Run every microtask already queued, and every one those queue, before returning.
*
Expand All @@ -187,7 +166,13 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
if (end < last.at) throw new Error(`an arrival at ${last.at} ms is past the ${options.duration} ms observed`);

const { rate, config } = options;
const { spans: samples, typical } = frameSamples(trace, rate);
const recorded = new Map<Micro, Arrival>();
for (const arrival of trace) {
if (!Number.isFinite(arrival.duration) || arrival.duration <= 0) {
throw new Error(`invalid recorded frame duration: ${arrival.duration}`);
}
recorded.set(micros(arrival.timestamp), arrival);
}

// Where each group's stream finishes.
const ends = new Map<number, number>();
Expand All @@ -202,15 +187,14 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
maxDelay: sync.out.maxDelay,
});

// The decoder's "auto" target. A recorded frame has no payload to read an Opus duration from,
// so a codec without a constant one takes the trace's typical spacing.
// Match the decoder: a codec constant until a delivered packet states its own duration.
const signals = new Effect();
const frame = frameDuration(config) ?? Milli((typical * 1000) / rate);
const frame = new Signal(frameDuration(config));
const playout = signals.computed((effect) =>
target({
measured: effect.get(consumer.spread),
advertised: config.jitter !== undefined ? Milli(config.jitter) : undefined,
frame,
frame: effect.get(frame),
delay: config.delay !== undefined ? Milli(config.delay) : undefined,
}),
);
Expand Down Expand Up @@ -239,9 +223,10 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
const media = Milli.fromMicro(timestamp);
const late = sync.received(media, "audio", arrival);
arrivals.push({ kind: "frame", timestamp: media, arrival, late });
const length = samples.get(timestamp);
if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`);
ring.insert(timestamp, [new Float32Array(length).fill(0.5)]);
const entry = recorded.get(timestamp);
if (!entry) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`);
frame.set(Milli(entry.duration));
ring.insert(timestamp, [new Float32Array(Math.round((entry.duration * rate) / 1000)).fill(0.5)]);
}
})().catch((error: unknown) => {
failure = { error };
Expand Down
61 changes: 35 additions & 26 deletions quest/m1/audio-jitter-target/watch.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,36 +20,45 @@ estimator's 2 s ceiling, as native does; a subscription cut to the target hid
every frame later than it. `demo/web` sets no delay and `js/watch` stores none,
so a fresh tile runs auto.

The harness's `js/watch/src/audio/replay.ts` plays the recorded traces in
`test/audio-quality/traces/` (the #3477 traces are gone) through
`Container.Consumer` and both rings on a simulated clock, and resolves `"auto"`
from the same estimator through a real `Sync`, resizing the ring as it moves.
Its exact budgets were re-measured when `main` merged in: underruns fell to at
most 2 a minute on every trace, but a ring that runs dry now re-stalls and
refills to the target, so `stalled_share` and `silence_share` rose (up to 3% on
`relay-bbb` and `relay-mic`), and `relay-mic` converges from a 420 ms p95 over
about 22 s with skip-aheads on the way down.

- Decide whether those re-stall refills are what the goal wants, using the
replay rows as the measurement. Tighten the replay budgets with any fix.
Tighten the browser rows only from nightly runner measurements, re-measured
as `test/audio-quality/budgets.json`'s note describes.
- The replay sizes every frame as the trace's median spacing, which a trace
missing most of its frames defeats; record the codec's frame duration in the
trace and replay that instead.
- Manual run against the public relay on Chrome and Safari, the two rows the
The harness's `js/watch/src/audio/replay.ts` replays the version 3 traces in
`test/audio-quality/traces/` through `Container.Consumer`, a real `Sync`, and
both rings on a simulated clock. Each arrival carries an encoded duration
from container metadata, the Opus TOC, or the codec's fixed sample count.
Replay never infers a frame's duration from timestamp spacing. The AAC-LC
fixtures retain their original arrivals with exact 1024-sample durations;
`relay-mic` was re-recorded with Chromium's fake capture device because the
old recording retained no payloads from which to recover Opus durations.

Re-measured on 2026-10-22: all eight replay rows have zero underruns. The
fresh `relay-mic` has zero skips, stalls, or silence, a 60 ms p95 target,
and 3.5 s convergence on both rings, over a 2 ms minimum RTT. It is a new
fake-device recording, so that result proves neither an improvement on
the old path nor the manual goal. The AAC recordings still refill: worst
stalled share is 2.5% (`relay-bbb`, isolated), with `local-aac` skipping
about twice a minute while converging over 15.5 s.

What remains is one manual run by a person on Apple hardware; nothing
deterministic is left to automate. Decided 2026-10-22:

- Run against the public relay on Chrome and Safari, the two rows the
issue measured, with a real microphone and 40 ms or more of added RTT. Watch
the stats panel's audio underrun counter and the latency tab's auto readout,
on both the isolated and the postMessage ring paths. Re-record the
`relay-mic` trace in the same run (`just test audio-quality-record`); the
checked-in one used Chromium's fake capture device. Measure the publisher's
audio encoder input-to-output lag in the same run using the reporter's
instrumented harness; #3518 fixed the known cause, so the 7.35 s lag and the
88 to 275 ms/s drift the issue reported stand unconfirmed. If drift survives,
the suspects are `writeFrame` opening a group per audio frame under
WebTransport stream credit and the main-thread task queue delivering encoder
output. Audio now defaults `groupDuration` to 20 ms. A/B that default
against a group of 100 ms or so in the same run.
checked-in one used Chromium's fake capture device and proves nothing here.
- Keep the production refill policy until that run's measurements exist. A
ring that runs dry re-stalls and refills to the target; change that only if
the run shows a cause and a justified alternative beats it. The
deterministic replay ceilings match the recorded durations. Tighten browser
rows only from nightly runner measurements, re-measured as
`test/audio-quality/budgets.json`'s note describes.
- Measure the publisher's audio encoder input-to-output lag in the same run
using the reporter's instrumented harness; #3518 fixed the known cause, so
the 7.35 s lag and the 88 to 275 ms/s drift the issue reported stand
unconfirmed. If drift survives, the suspects are `writeFrame` opening a
group per audio frame under WebTransport stream credit and the main-thread
task queue delivering encoder output. Audio now defaults `groupDuration` to
20 ms. A/B that default against a group of 100 ms or so in the same run.

Decided 2026-10-15: keep this proof in m1 with the existing playback and
A/V clock work. The estimator defaults on; deferring watch-worker does not
Expand Down
24 changes: 16 additions & 8 deletions test/audio-quality/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ rendition's advertised jitter and delay. These distinguish a target increase cau
transport from one declared by the publisher. The final page environment keeps the full audio
configuration.

`--capture` also writes `<tag>.arrivals.ndjson`: `[at, timestamp, group]` on the same viewer clock as
`--capture` also writes `<tag>.arrivals.ndjson`: `[at, timestamp, group, duration]` on the same viewer clock as
`<tag>.ndjson`, before the container orders or skips groups. The observer shares the player's
connection and track, priority, and subscription max delay (at least 2 s at "auto", as the decoder
asks, not the target), so it adds no upstream subscription demand and still sees a frame later than
Expand All @@ -117,7 +117,7 @@ Chromium GC timeline in `<tag>.trace.json` and sampled CPU profile in `<tag>.pro
Tracing can perturb playback; compare it with a separate
untraced control and never use traced rows to calibrate quality budgets.

To replay a capture, put its arrivals and final environment configuration into the version 2
To replay a capture, put its arrivals and final environment configuration into the version 3
`Trace` below, subtract the first arrival's `at` from every arrival and from the last sample's
`at` for the duration, and supply the observed minimum RTT as a diagnostic. Replay measures the
target from the arrivals and holds the configuration constant; compare the raw samples when it
Expand All @@ -138,19 +138,22 @@ The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, ea
| `bursty-aac` | a nightly `bursty` AAC row's capture: each flush delivered newest first, which kept the player re-stalling until the estimator measured reordered frames |

A trace is a [`Trace`](clients/js/src/schema.ts): the catalog's audio rendition, the smallest
PROBE round trip the session saw, how long it observed, and every frame's `[at, timestamp, group]`,
with `at` on the viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, the way
`Container.Consumer` receives it, using only public `@moq/net` and `@moq/hang` in the same Chromium
over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend
PROBE round trip the session saw, how long it observed, and every frame's `[at, timestamp, group, duration]`,
with `at` on the viewer's clock and `duration` in ms. Duration comes from the container's sample
metadata, the Opus packet TOC, or the codec's fixed sample count. Capture refuses a frame whose
duration cannot be determined; timestamp gaps never stand in for missing packet payloads.
`record.ts` stamps a frame as it comes off its group's stream, the way `Container.Consumer`
receives it, with the same transport, container, and codec-duration readers over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend
on the delay, so the replay decides those again.

`replay.ts` plays a trace through the player's own `Container.Consumer` and rings on a simulated
clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a
real `Sync` resolves from the playout target that consumer measures, and reads each quantum through
the tap's classifier. The consumer makes the player's group ordering, max delay skips, and
discontinuity resets again at that delay, and the ring follows the delay as it moves; a group's stream is taken to finish with its last recorded frame. Every
frame is the trace's median spacing long, so a frame that never arrived stays missing audio, and
rendering runs to the end of the observation, so an outage after the last arrival is heard. It
frame plays for its recorded encoded duration, so missing packets stay missing audio even when
most frames never arrive. The target's codec term follows delivered packet durations, as the
decoder's does. Rendering runs to the end of the observation, so an outage after the last arrival is heard. It
writes the same samples a page does, so `analyze.ts` reduces both alike, and the shaper, transport,
and clock checks have nothing to void. Decoding is taken as instant. A replay is deterministic, so
its budgets are exactly what it measured.
Expand All @@ -160,6 +163,11 @@ just test audio-quality-record # the three recorded lanes
just test audio-quality-record --lanes relay-bbb
```

Version 3 adds per-frame durations. The AAC-LC recordings made before that field existed retain
their arrival clocks and use the codec's fixed 1024 samples per frame. The Opus `relay-mic` recording
was replaced with a new fake-device capture because its old payloads, and therefore its packet
durations, were not retained. It still does not prove real-microphone behavior or added RTT.

Recording is by hand: the public relay and a browser publisher are not something a nightly run
should depend on. Re-measure the replay budgets after re-recording. The recorder asks the relay for
ten seconds of backlog so a late group is kept, which makes a trace's tune-in heavier than a
Expand Down
Loading
Loading