Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions js/publish/MIGRATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first.

## Unreleased

### The capture graph renders to no output device ([#7881](https://github.com/moq-dev/moq/pull/7881))

`Audio.Capture`'s and `Audio.Encoder`'s `root` now sit in an `AudioContext` built with `sinkId: { type: "none" }`, which shortens the bursts capture hands the encoder.
Tapping `root` for analysis works as before, but where the browser supports `sinkId`, anything connected to that context's `destination` is no longer heard.
To monitor the microphone, play the track through a context of your own.

### `maxAge: null` is unlimited, and the catalog takes the zero default ([#7729](https://github.com/moq-dev/moq/pull/7729))

`Broadcast`'s `maxAge` input accepts `null` for no publisher limit on its media tracks; omitted is still 30 seconds (`Container.MAX_AGE` from `@moq/hang`). With `null`, on-demand audio keeps 30 seconds of raw PCM, since raw history needs a finite bound.
Expand Down
30 changes: 23 additions & 7 deletions js/publish/src/audio/capture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@ function installFakeWebAudio() {
// Never resolves during the test, so the spawned worklet load stays pending until teardown.
const addModule = () => new Promise<void>(() => {});
let audioWorkletNodes = 0;
const requestedRates: (number | undefined)[] = [];
const requested: (AudioContextOptions & { sinkId?: unknown })[] = [];

// Already running, as after a gesture, so only the pending module load holds the worklet back.
class FakeAudioContext extends EventTarget {
state: AudioContextState = "running";
audioWorklet = { addModule };
constructor(options?: AudioContextOptions) {
super();
requestedRates.push(options?.sampleRate);
requested.push(options ?? {});
}
resume(): Promise<void> {
return Promise.resolve();
Expand Down Expand Up @@ -77,8 +77,8 @@ function installFakeWebAudio() {
get audioWorkletNodes() {
return audioWorkletNodes;
},
get requestedRates() {
return requestedRates;
get requested() {
return requested;
},
[Symbol.dispose]() {
for (const [name, original] of originals) {
Expand Down Expand Up @@ -136,7 +136,7 @@ async function requestedRate(sampleRate: number | undefined, override?: number)
capture.close();
await settle();

return webaudio.requestedRates.at(-1);
return webaudio.requested.at(-1)?.sampleRate;
}

test("snaps a rate Opus can't decode up to full band", async () => {
Expand All @@ -161,6 +161,22 @@ test("honors an explicit rate override", async () => {
expect(await requestedRate(48_000, 16_000)).toBe(16_000);
});

// A device's render callback can be 43 ms long, which bursts the frames handed to the encoder. The
// capture graph is never audible, so it renders to no device, which Chromium paces every 10 ms.
test("renders the capture graph to no output device", async () => {
using webaudio = installFakeWebAudio();

const capture = new Capture({
enabled: true,
source: new Signal(fakeSource()) as never,
});
await settle();
capture.close();
await settle();

expect(webaudio.requested.at(-1)?.sinkId).toEqual({ type: "none" });
});

// Regression: tweaking an encoder knob must not tear down the microphone. The capture used to take
// the codec as an input, so it rebuilt the AudioContext on a codec change, dropping the worklet and
// closing the track being published. Now the capture is codec-independent, which it has to be:
Expand All @@ -175,12 +191,12 @@ test("never rebuilds the capture graph for encoder settings", async () => {
sampleRate: rate,
});
await settle();
expect(webaudio.requestedRates.length).toBe(1);
expect(webaudio.requested.length).toBe(1);

// Only something that changes the capture itself may rebuild it.
rate.set(16_000);
await settle();
expect(webaudio.requestedRates.length).toBe(2);
expect(webaudio.requested.length).toBe(2);

capture.close();
await settle();
Expand Down
16 changes: 13 additions & 3 deletions js/publish/src/audio/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ export type CaptureOutput = {
format: Signal<Format | undefined>;

// The head of the Web Audio graph, when there is one, so callers can tap the raw capture.
// Undefined for decoded samples, which never touch Web Audio.
// Its context renders to no device where the browser supports `sinkId`, so nothing connected to
// its destination is heard. Undefined for decoded samples, which never touch Web Audio.
root: Signal<AudioNode | undefined>;
};

Expand Down Expand Up @@ -185,10 +186,19 @@ export class Capture {
// the prop or the track's applied getUserMedia constraint, and force the worklet to mix to it.
const requestedChannels = effect.get(this.channelCount) ?? requestedChannelCount(source.track);

const context = new AudioContext({
// The worklet delivers samples one render callback at a time, and a device's callback can be long:
// Chromium on PipeWire renders 2048 samples (43 ms) per callback even at "interactive", so 60 ms
// frames left 36-94 ms apart. Nothing here is audible, so render to no device, which Chromium 155+
// paces every 10 ms. Older Chromium paces it at the default device's callback size, as before, and
// a browser without `sinkId` ignores it and keeps the default device.
// Don't lower latencyHint instead: at 128 samples, a busy host misses callbacks and the capture
// clock falls behind.
const options: AudioContextOptions & { sinkId: { type: "none" } } = {
latencyHint: "interactive",
sampleRate,
});
sinkId: { type: "none" },
Comment thread
kixelated marked this conversation as resolved.
Resolved
};
const context = new AudioContext(options);
effect.cleanup(() => context.close());

// Nothing guarantees a gesture has happened yet: a pre-granted microphone reaches here on page
Expand Down
4 changes: 3 additions & 1 deletion js/publish/src/audio/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ export type EncoderOutput = {
// while there's no capture.
catalog: Signal<Catalog.AudioConfig | undefined>;
// The head of the capture graph, so callers can tap the raw capture. Volume is applied to the
// PCM rather than in the graph, so this is pre-gain. Undefined for a source that isn't a track.
// PCM rather than in the graph, so this is pre-gain. Its context renders to no device where the
// browser supports `sinkId`, so nothing connected to its destination is heard. Undefined for a
// source that isn't a track.
root: Signal<AudioNode | undefined>;
// True while encoding capture: enabled with a captured format and a track, which also takes a
// subscriber unless the broadcast's `onDemand` is false. Retained audio encoded for a subscriber
Expand Down
21 changes: 13 additions & 8 deletions quest/m1/push-to-talk-latency.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,14 @@ Biggest contributors and their knobs:
AudioContext's `latencyHint` (`"interactive"` in
`js/watch/src/audio/decoder.ts`), otherwise the platform. A real device,
such as `kixtop`'s macOS output, should be measured before acting on it.
- **Capture and render, about 52 ms together.** Both follow the AudioContext
callback cadence on this host. The capture graph also delivers 60 ms frames
in bursts, 25-109 ms apart. Knob: the capture AudioContext's `latencyHint`
(`js/publish/src/audio/capture.ts`), otherwise the platform.
- **Capture and render, about 52 ms together.** Both followed the
AudioContext callback cadence on this host: PipeWire gives Chromium 2048
samples (43 ms) per callback even at `"interactive"`. Capture no longer
does, since its context renders to no device
([moq-dev/moq#7881](https://github.com/moq-dev/moq/pull/7881)): on this host
its median dropped from 29 ms to 1-9 ms, and the median per-press arrival
spread from 45 ms to 12 ms. Render still does. Knob: the watch AudioContext's
`latencyHint`, otherwise the platform.
- **Opus frame, 60 ms.** This test chooses it; the publisher defaults to
20 ms. Under a fixed delay it is absorbed by the delay. In `"auto"` it adds
one frame to the target. Knob: the encoder's `frameDuration`.
Expand All @@ -66,7 +70,10 @@ fixed delay bypasses the jitter estimator. In `"auto"`, the 46 ms median
per-press arrival-lateness spread, from capture bursts rather than the
network, would make the measured term nonzero; that is the estimator working
as specified in [Audio jitter
target](/quest/m1/audio-jitter-target/README.md).
target](/quest/m1/audio-jitter-target/README.md). With capture fixed, the
spread left on Chromium 155+ is its 10 ms capture chunks, plus the release
flush's lookahead chunk, which goes out up to a frame ahead of its media.
Older Chromium still paces capture at the device's callback size.

Decided 2026-11-04 (maintainer, planning in
[moq-dev/moq#7816](https://github.com/moq-dev/moq/pull/7816)):
Expand All @@ -79,13 +86,11 @@ Decided 2026-11-04 (maintainer, planning in
then.
- ✅ Browser only.
- The catalog toggle forces at least a round trip of jitter buffer, so the
budget is measured once push-to-talk runs on DTX (Required below), and once
capture delivers frames steadily.
budget is measured once push-to-talk runs on DTX (Required below).

## Required

- [Push-to-talk suppresses silence instead of toggling the catalog](/quest/m1/push-to-talk/dtx.md) - removes the catalog round trip the budget would otherwise carry
- [Browser capture delivers audio frames on a steady cadence](/quest/m1/push-to-talk/capture-smoothing.md) - so `"auto"` measures the network, not capture bursts

## Related

Expand Down
1 change: 0 additions & 1 deletion quest/m1/push-to-talk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ catches this. A complete set of runs finishes this epic.

- [Pick how the browser publisher suppresses silence](/quest/m1/push-to-talk/dtx-route.md) - measures the DTX routes
- [Push-to-talk suppresses silence instead of toggling the catalog](/quest/m1/push-to-talk/dtx.md) - replaces hold with DTX
- [Browser capture delivers audio frames on a steady cadence](/quest/m1/push-to-talk/capture-smoothing.md) - so `"auto"` measures the network

## Closes

Expand Down
27 changes: 0 additions & 27 deletions quest/m1/push-to-talk/capture-smoothing.md

This file was deleted.

5 changes: 5 additions & 0 deletions quest/m1/push-to-talk/dtx.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ and the maintainer would rather fix DTX and avoid the catalog entirely.
- The publisher's mute becomes "suppress", not `enabled: false`. Keep
`enabled` for a real stop (leaving the call), where a catalog change is
correct.
- Mind the release flush's lookahead chunk. The disable path's `flush()`
sends it at once, up to a frame ahead of its media
([moq-dev/moq#7881](https://github.com/moq-dev/moq/pull/7881) measured it adding
10-22 ms to the median per-press spread). `"auto"` then measures an utterance that starts
within its 2 s window against it, so suppression should not reintroduce it.

Update the epic's acceptance and `doc/` pages that describe hold.

Expand Down
7 changes: 5 additions & 2 deletions test/audio-quality/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,11 @@ it: `capture` (worklet delivery and framing, after the media the frame still nee
press), `encode` (codec and container write), `network` (write to arrival, through the relay),
`decode` (arrival to decoded PCM), `jitter_buffer` (the head's PCM to the first PCM that fills the
playout delay), `render` (that PCM to the render onset: the worklet hand-off and the context's
render callback), and `device`, as the stages below name them. `latency` and `path` give each one's
min, median and max.
render callback), and `device`, as the stages below name them. `spread` is how far an utterance's frames
stray from a steady arrival: the latest arrival relative to its media timestamp minus the earliest,
which on this ideal path is the capture graph's cadence and what `"auto"`'s jitter estimate measures.
The release's flushed lookahead chunk is not one of the utterance's frames and is left out. `latency`
and `path` give each one's min, median and max.

The command prints `latency`, `path`, and the completeness counts, and keeps its run directory
even when it passes (set `HARNESS_KEEP=0` to drop it). Nightly runs it and keeps the raw evidence
Expand Down
12 changes: 12 additions & 0 deletions test/audio-quality/clients/js/ptt-analyze.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,18 @@ test("a complete run passes, reports its latency, and splits each press exactly"
expect(parts).toBe(press.audible);
});

test("a frame held back by a capture burst widens its press's spread", async () => {
const result = trace();
const press = result.presses[4].at;
const held = result.arrivals.find((arrival) => arrival[0] > press + 3 * FRAME);
if (!held) throw new Error("no frame to hold");
held[0] += 30;
const { exitCode, summary } = await analyze(result);
expect(exitCode).toBe(0);
expect(summary.presses[2].spread).toBeCloseTo(30);
expect(summary.latency.spread).toEqual({ min: 0, median: 0, max: expect.closeTo(30) });
});

test("the delay is filled by the first frame to reach it, not the newest before the onset", async () => {
// The sixth frame decodes just before the onset, adding to a ring already leaving its stall.
const result = trace();
Expand Down
18 changes: 13 additions & 5 deletions test/audio-quality/clients/js/ptt-analyze.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ const presses = result.presses
return [arrival[0] - press.at - (frame.timestamp - head) / 1000];
});
const delivery = waited.length ? Math.max(...waited) : null;
// What "auto"'s jitter estimate reads off this utterance: how far its frames' arrivals stray
// from their media timestamps, slowest minus fastest. An ideal path leaves only capture cadence.
const offsets = received.map(([at, media]) => at - media);
const spread = offsets.length ? Math.max(...offsets) - Math.min(...offsets) : null;
// The head frame's path from the press to projected output, one boundary after another, so
// the parts sum to `audible`. Capture and packetization split at the head frame's end on the
// media clock, placed against the newest captured chunk delivered before the press.
Expand Down Expand Up @@ -95,12 +99,13 @@ const presses = result.presses
arrival: earliest ? earliest[0] - press.at : null,
audible: audible ? audible.at - press.at : null,
delivery,
spread,
playout: audible && delivery !== null ? audible.rendered - press.at - delivery : null,
output: audible ? audible.at - audible.rendered : null,
path,
};
});
const spread = (values: (number | null)[]) => {
const summarize = (values: (number | null)[]) => {
const sorted = values.flatMap((value) => (value === null ? [] : [value])).toSorted((a, b) => a - b);
if (!sorted.length) return null;
const middle = sorted.length / 2;
Expand All @@ -111,11 +116,14 @@ const parts = ["capture", "packetization", "encode", "network", "decode", "jitte
const summary = {
// The goal: at most 300 ms of playout added on top of one-way delivery.
latency: {
delivery: spread(presses.map((press) => press.delivery)),
playout: spread(presses.map((press) => press.playout)),
output: spread(presses.map((press) => press.output)),
delivery: summarize(presses.map((press) => press.delivery)),
playout: summarize(presses.map((press) => press.playout)),
output: summarize(presses.map((press) => press.output)),
spread: summarize(presses.map((press) => press.spread)),
},
path: Object.fromEntries(parts.map((part) => [part, spread(presses.map((press) => press.path?.[part] ?? null))])),
path: Object.fromEntries(
parts.map((part) => [part, summarize(presses.map((press) => press.path?.[part] ?? null))]),
),
delay: result.state?.delay,
presses,
captures: result.captures,
Expand Down
Loading