Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion js/watch/src/audio/replay.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test";
import { Duration } from "@moq/clock";
import type * as Catalog from "@moq/hang/catalog";
import { type Arrival, type Options, QUANTUM, type Quantum, replay } from "./replay";
import { Track } from "@moq/net";
import { type Arrival, answering, type Options, QUANTUM, type Quantum, replay } from "./replay";

const RATE = 48000;
const CONFIG = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig;
Expand Down Expand Up @@ -431,3 +432,23 @@ describe("buffered replay against the shared clock", () => {
expect(warn).not.toHaveBeenCalled();
});
});

describe("answering", () => {
it("names the publisher's largest group first, then each newer group, and delegates the rest", async () => {
const track = new Track.Producer("audio");
const subscriber = answering(track.subscribe(), 5n);
try {
// A group at or below the answer is no news: the track's own largest is older than it.
track.createGroup(3n);
expect(await subscriber.largest()).toEqual({ group: 5n, frame: 0 });
const next = subscriber.largest();
track.createGroup(5n);
track.createGroup(9n);
expect((await next)?.group).toBe(9n);
expect(subscriber.name).toBe("audio");
} finally {
subscriber.close();
track.close();
}
});
});
47 changes: 42 additions & 5 deletions js/watch/src/audio/replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ export interface Options {
lookahead?: number;
/** The configured stretch, in ms, as the element takes it (default: the element's). */
stretch?: number;
/**
* The publisher's largest group as the subscription began, which the consumer learns as a wire
* subscriber learns it from the publisher's answer. Omitted, the replay's track names its own: the
* first group to land.
*/
largest?: number;
/** The rendition as the catalog described it: its advertised jitter, delay, and codec. */
config: Catalog.AudioConfig;
/**
Expand Down Expand Up @@ -177,6 +183,33 @@ const micros = (ms: number) => Math.round(ms * 1000);
*/
const settle = () => new Promise<void>((resolve) => setImmediate(resolve));

/**
* `track` answering `largest` the way a wire subscriber does once the publisher has answered: first
* the publisher's largest group as the subscription began, then each newer group as it lands.
* Everything else is the local subscriber's.
*/
export function answering(track: Track.Subscriber, largest: bigint): Track.Subscriber {
let answered = false;
return new Proxy(track, {
get(target, prop) {
if (prop !== "largest") {
const value = Reflect.get(target, prop, target);
return typeof value === "function" ? value.bind(target) : value;
}
return async () => {
if (!answered) {
answered = true;
return { group: largest, frame: 0 };
}
for (;;) {
const next = await target.largest();
if (!next || next.group > largest) return next;
}
};
},
});
}

/**
* Play `trace` in simulated real time, yielding every render quantum from the first arrival to the
* end of the observation.
Expand Down Expand Up @@ -218,11 +251,15 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
const maxDelay = new Derived([sync.out.delay, sync.out.maxDelay], (delay, max) =>
audioMaxDelay({ delay, maxDelay: max }),
);
const consumer = new Container.Consumer(track.subscribe({ delay: { max: WIRE_MAX_DELAY } }), {
format: new Container.Legacy.Format("audio"),
delay: new Derived([sync.out.delay, maxDelay], (min, max) => ({ min, max })),
wait: sync.out.maxDelay,
});
const subscriber = track.subscribe({ delay: { max: WIRE_MAX_DELAY } });
const consumer = new Container.Consumer(
options.largest === undefined ? subscriber : answering(subscriber, BigInt(options.largest)),
{
format: new Container.Legacy.Format("audio"),
delay: new Derived([sync.out.delay, maxDelay], (min, max) => ({ min, max })),
wait: sync.out.maxDelay,
},
);

// Match the decoder: a codec constant until a delivered packet states its own duration.
const signals = new Effect();
Expand Down
4 changes: 2 additions & 2 deletions quest/m0/interop-green/replacement-audio.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Open: where those 600 ms go. The decoder resets the ring on a changed instance
whether the ring waits for audio at the clock behind a backlog it must drop (then a join at the
live edge is the fix), or the starved page just decodes late (then the bound needs a load term
derived from the host, not a constant). Trace the ring's join and un-stall in the worklet on a
failing run; the audio-playout epic's [largest replay](/quest/m2/audio-playout/replay-largest.md)
may overlap.
failing run; the largest-group replay at tune-in
([moq-dev/moq#8844](https://github.com/moq-dev/moq/pull/8844)) may overlap.

Public API: none. Wire: none.
69 changes: 0 additions & 69 deletions quest/m2/audio-playout/replay-largest.md

This file was deleted.

20 changes: 17 additions & 3 deletions test/audio-quality/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ Chromium GC timeline in `<tag>.trace.json` and sampled CPU profile in `<tag>.pro
Tracing can perturb playback; compare it with a separate
untraced control and never use traced rows to calibrate quality budgets.

To replay a capture, put its arrivals and final environment configuration into the version 3
To replay a capture, put its arrivals, final environment configuration, and `largest` (the
observer's join, `environment.largest` in `<tag>.page.json`) into the version 4
`Trace` below, subtract the first arrival's `at` from every arrival and from the last sample's
`at` for the duration, and supply the observed minimum RTT as a diagnostic. Replay measures the
target from the arrivals and holds the configuration constant; compare the raw samples when it
Expand Down Expand Up @@ -215,7 +216,7 @@ them, each graded as a `replay` row whose profile is the file's name:
| `relay-mic-stall` | `relay-mic` with nothing arriving from 15 s to 16.5 s, then the held frames trickling in oldest first at twice real time until they meet live arrivals at 18 s: the stall a congested path causes, longer than the target, with its backlog arriving late |

A trace is a [`Trace`](clients/js/src/schema.ts): the catalog's audio rendition, the smallest
PROBE round trip the session saw, how long it observed, and every frame's `[at, timestamp, group, duration]`,
PROBE round trip the session saw, the publisher's largest group at tune-in, how long it observed, and every frame's `[at, timestamp, group, duration]`,
with `at` on the viewer's clock and `duration` in ms. Duration comes from the container's sample
metadata, the Opus packet TOC, or the codec's fixed sample count. Capture refuses a frame whose
duration cannot be determined; timestamp gaps never stand in for missing packet payloads.
Expand All @@ -224,7 +225,8 @@ receives it, with the same transport, container, and codec-duration readers over
on the delay, so the replay decides those again.

`replay.ts` plays a trace through the player's own `Container.Consumer` and ring on a simulated
clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a
clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), whose consumer is told
the trace's `largest` as a wire subscriber is told by the publisher, at the "auto" delay a
real `Sync` resolves from the playout target that consumer measures, and reads each quantum through
the tap's classifier. The consumer makes the player's group ordering, max delay skips, and
discontinuity resets again at that delay, and the ring follows the delay as it moves; a group's stream is taken to finish with its last recorded frame. Every
Expand Down Expand Up @@ -252,6 +254,18 @@ just test audio-quality-record --lanes relay-mic --mic-broadcast <name>
microphone, instead of publishing Chromium's fake capture device. Name the browser and device in
the trace's `source` before checking it in.

Version 4 adds `largest`, the publisher's largest group as the subscription began, which a wire
subscriber learns from the publisher's answer and the replay's consumer needs to tell a group it
skips unread as the join's backlog. The recorder and the `--capture` observer read it the way
`js/hang`'s consumer does: once the track's info or first group shows the subscription established
and its start resolved, so a cached group racing the answer, or live groups landing after it, do not
set it. No earlier recording kept the answer, and re-recording needs a person at the microphone and
would move every other column of the baseline, so each checked-in trace derives it: the newest group
whose first frame's media timestamp plus the trace's lowest lag (the smallest `at - timestamp`) falls
at or before the first arrival, the live edge as the first frame arrived. Its `source` says so. For
`relay-mic-rtt40` that is group 8994, where its first arrival is the stale group 7499 and the relay's
backlog reaches about 9047.

Version 3 adds per-frame durations. The AAC-LC recordings made before that field existed retain
their arrival clocks and use the codec's fixed 1024 samples per frame. The Opus `relay-mic` recording
is a real microphone captured with `--mic-broadcast`, with its packet durations retained. The
Expand Down
18 changes: 9 additions & 9 deletions test/audio-quality/budgets.json

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion test/audio-quality/clients/js/record.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,15 +123,18 @@ try {
arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain())));
// The RTT is read last: the PROBE minimum only falls as the session runs.
const rtt = await page.evaluate(() => globalThis.recorder.info()?.rtt ?? null);
const largest = await page.evaluate(() => globalThis.recorder.info()?.largest);
if (largest === undefined) throw new Error("the subscription never reported the publisher's largest group");

const trimmed = arrivals.filter(([at]) => at - start <= durationMs);
const round = (x: number, places: number) => Math.round(x * 10 ** places) / 10 ** places;
const trace: Trace = {
version: 3,
version: 4,
source,
description,
transport: info.transport,
rtt: rtt === null ? null : round(rtt, 1),
largest,
config: info.config as Trace["config"],
duration: durationMs,
arrivals: trimmed.map(([at, timestamp, group, duration]) => [
Expand Down
3 changes: 2 additions & 1 deletion test/audio-quality/clients/js/replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ for (const [name, list, known] of selectors) {

const rows: { row: Row; trace: Trace }[] = [];
for (const { profile, trace } of traces) {
if (trace.version !== 3) throw new Error(`${profile}: trace version ${trace.version}, expected 3`);
if (trace.version !== 4) throw new Error(`${profile}: trace version ${trace.version}, expected 4`);
const codec = traceCodec(trace);
if (profiles && !profiles.includes(profile)) continue;
if (!codecs.includes(codec)) continue;
Expand Down Expand Up @@ -141,6 +141,7 @@ for (const { row, trace } of rows) {
rate,
delay: "auto",
config: trace.config as unknown as Catalog.AudioConfig,
largest: trace.largest,
duration: trace.duration,
});
let last: Reading | undefined;
Expand Down
45 changes: 45 additions & 0 deletions test/audio-quality/clients/js/src/capture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,51 @@ test("the observer asks for the decoder's delay plus its stretch and warmup, not
}
});

test("the observer reads the publisher's largest group as its subscription began, not a later one", async () => {
const origin = new Origin.Producer();
const publishedRoute = origin.route(Path.from("tone.hang"), { hidden: true });
const published = publishedRoute.broadcast;
const track = published.createTrack("audio", { timescale: Time.Timescale.MILLI });
publishedRoute.announce();
const frame = (group: Group.Producer, micros: number) =>
group.writeFrame({
timestamp: Time.Timestamp.fromMicros(micros),
payload: Legacy.encodeFrame(new Uint8Array([1]), Duration.micros(micros)),
});
const source = new Broadcast({ origin, name: Path.from("tone.hang"), follow: false, catalogFormat: "manual" });
const capture = new Capture({
broadcast: new Signal(source),
track: new Signal("audio"),
config: new Signal({
codec: "opus",
sampleRate: 48000,
numberOfChannels: 1,
container: { kind: "legacy" },
} as Catalog.AudioConfig),
delay: new Signal(Duration.millis(250)),
maxDelay: new Signal(Duration.millis(250)),
});
try {
while (!track.subscription.peek()) await track.subscription.changed();
const first = track.createGroup(7n);
frame(first, 0);
while (capture.largest() === undefined) await yieldNow();
// Live groups landing after the answer raise the track's largest, not the join's.
const later = track.createGroup(12n);
frame(later, 20_000);
const seen: Arrival[] = [];
while (seen.length < 2) {
seen.push(...capture.drain());
await yieldNow();
}
expect(capture.largest()).toBe(7);
} finally {
await capture.close();
source.close();
origin.close();
}
});

/** Capture a legacy track, then end its first group with `end` once that group's reader is in flight. */
async function captureGroupEnd(end: (group: Group.Producer) => void): Promise<string | undefined> {
const origin = new Origin.Producer();
Expand Down
30 changes: 29 additions & 1 deletion test/audio-quality/clients/js/src/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Duration, Instant, now } from "@moq/clock";
import * as Catalog from "@moq/hang/catalog";
import * as Cmaf from "@moq/hang/cmaf";
import * as Container from "@moq/hang/container";
import { type Group, Error as NetError } from "@moq/net";
import { type Group, Error as NetError, type Track } from "@moq/net";
import { Effect, type Getter } from "@moq/signals";
import type { Broadcast } from "@moq/watch";
import { frameDuration, packetDuration } from "../../../../../js/watch/src/audio/config.ts";
Expand Down Expand Up @@ -32,6 +32,7 @@ export class Capture {
readonly #seen = new Set<string>();
readonly #readers = new Set<Promise<void>>();
#failure?: string;
#largest?: number;

constructor(input: Input) {
this.#signals.run((effect) => {
Expand All @@ -53,6 +54,15 @@ export class Capture {
},
});
const groups = new Set<Group.Consumer>();
const tuned = Promise.withResolvers<void>();
effect.spawn(async () => {
try {
// The first subscription's: a resubscribe after a rerun joins a track already flowing.
this.#largest ??= await joined(track, tuned.promise);
} catch (error) {
if (!effect.abort.aborted) this.#failure ??= String(error);
}
});
effect.cleanup(() => {
track.close();
for (const group of groups) group.close();
Expand All @@ -73,6 +83,7 @@ export class Capture {
for (;;) {
const group = await effect.race(track.recvGroup());
if (!group) throw new Error("the captured track ended");
tuned.resolve();
groups.add(group);
// Closing a group ends its reader cleanly after the buffered frames, so any
// other error here is a real failure, even one surfacing during close.
Expand Down Expand Up @@ -111,6 +122,11 @@ export class Capture {
return this.#arrivals.splice(0);
}

/** The publisher's largest group as the subscription began, once known. */
largest(): number | undefined {
return this.#largest;
}

/** Return the failure that stopped capture, if any. */
error(): string | undefined {
return this.#failure;
Expand All @@ -123,6 +139,18 @@ export class Capture {
}
}

/**
* The publisher's largest group as `track`'s subscription began, read the way the container consumer
* reads it: once the track's info or `tuned`, its first group, shows the subscription established and
* its start resolved, so a cached group racing the answer, or live groups landing after it, don't set it.
*/
export async function joined(track: Track.Subscriber, tuned: Promise<unknown>): Promise<number | undefined> {
await Promise.race([track.info().catch(() => undefined), tuned]);
await track.start();
const largest = await track.largest();
return largest === undefined ? undefined : Number(largest.group);
}

/** Decode the audio container selected by the catalog. */
export function format(config: Catalog.AudioConfig): Container.Format {
const container = config.container;
Expand Down
1 change: 1 addition & 0 deletions test/audio-quality/clients/js/src/probe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ export function probe(watch: MoqWatch, capture = false): Probe {
rate: config?.sampleRate,
jitter: config?.jitter,
config,
largest: recording?.largest(),
contextRate: audio.out.context.peek()?.sampleRate,
};
},
Expand Down
Loading