diff --git a/js/publish/MIGRATE.md b/js/publish/MIGRATE.md index 781cafe6847..bbe065a4282 100644 --- a/js/publish/MIGRATE.md +++ b/js/publish/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### The capture graph renders to no output device ([#7881](https://github.com/moq-dev/moq/pull/7881)) + +`Audio.Capture`'s and `Audio.Encoder`'s `root` now sit in an `AudioContext` built with `sinkId: { type: "none" }`, which shortens the bursts capture hands the encoder. +Tapping `root` for analysis works as before, but where the browser supports `sinkId`, anything connected to that context's `destination` is no longer heard. +To monitor the microphone, play the track through a context of your own. + ### `maxAge: null` is unlimited, and the catalog takes the zero default ([#7729](https://github.com/moq-dev/moq/pull/7729)) `Broadcast`'s `maxAge` input accepts `null` for no publisher limit on its media tracks; omitted is still 30 seconds (`Container.MAX_AGE` from `@moq/hang`). With `null`, on-demand audio keeps 30 seconds of raw PCM, since raw history needs a finite bound. diff --git a/js/publish/src/audio/capture.test.ts b/js/publish/src/audio/capture.test.ts index 9f67d4d9231..978ec20974e 100644 --- a/js/publish/src/audio/capture.test.ts +++ b/js/publish/src/audio/capture.test.ts @@ -23,7 +23,7 @@ function installFakeWebAudio() { // Never resolves during the test, so the spawned worklet load stays pending until teardown. const addModule = () => new Promise(() => {}); let audioWorkletNodes = 0; - const requestedRates: (number | undefined)[] = []; + const requested: (AudioContextOptions & { sinkId?: unknown })[] = []; // Already running, as after a gesture, so only the pending module load holds the worklet back. class FakeAudioContext extends EventTarget { @@ -31,7 +31,7 @@ function installFakeWebAudio() { audioWorklet = { addModule }; constructor(options?: AudioContextOptions) { super(); - requestedRates.push(options?.sampleRate); + requested.push(options ?? {}); } resume(): Promise { return Promise.resolve(); @@ -77,8 +77,8 @@ function installFakeWebAudio() { get audioWorkletNodes() { return audioWorkletNodes; }, - get requestedRates() { - return requestedRates; + get requested() { + return requested; }, [Symbol.dispose]() { for (const [name, original] of originals) { @@ -136,7 +136,7 @@ async function requestedRate(sampleRate: number | undefined, override?: number) capture.close(); await settle(); - return webaudio.requestedRates.at(-1); + return webaudio.requested.at(-1)?.sampleRate; } test("snaps a rate Opus can't decode up to full band", async () => { @@ -161,6 +161,22 @@ test("honors an explicit rate override", async () => { expect(await requestedRate(48_000, 16_000)).toBe(16_000); }); +// A device's render callback can be 43 ms long, which bursts the frames handed to the encoder. The +// capture graph is never audible, so it renders to no device, which Chromium paces every 10 ms. +test("renders the capture graph to no output device", async () => { + using webaudio = installFakeWebAudio(); + + const capture = new Capture({ + enabled: true, + source: new Signal(fakeSource()) as never, + }); + await settle(); + capture.close(); + await settle(); + + expect(webaudio.requested.at(-1)?.sinkId).toEqual({ type: "none" }); +}); + // Regression: tweaking an encoder knob must not tear down the microphone. The capture used to take // the codec as an input, so it rebuilt the AudioContext on a codec change, dropping the worklet and // closing the track being published. Now the capture is codec-independent, which it has to be: @@ -175,12 +191,12 @@ test("never rebuilds the capture graph for encoder settings", async () => { sampleRate: rate, }); await settle(); - expect(webaudio.requestedRates.length).toBe(1); + expect(webaudio.requested.length).toBe(1); // Only something that changes the capture itself may rebuild it. rate.set(16_000); await settle(); - expect(webaudio.requestedRates.length).toBe(2); + expect(webaudio.requested.length).toBe(2); capture.close(); await settle(); diff --git a/js/publish/src/audio/capture.ts b/js/publish/src/audio/capture.ts index 30f575b884a..0ed53323da5 100644 --- a/js/publish/src/audio/capture.ts +++ b/js/publish/src/audio/capture.ts @@ -72,7 +72,8 @@ export type CaptureOutput = { format: Signal; // The head of the Web Audio graph, when there is one, so callers can tap the raw capture. - // Undefined for decoded samples, which never touch Web Audio. + // Its context renders to no device where the browser supports `sinkId`, so nothing connected to + // its destination is heard. Undefined for decoded samples, which never touch Web Audio. root: Signal; }; @@ -185,10 +186,19 @@ export class Capture { // the prop or the track's applied getUserMedia constraint, and force the worklet to mix to it. const requestedChannels = effect.get(this.channelCount) ?? requestedChannelCount(source.track); - const context = new AudioContext({ + // The worklet delivers samples one render callback at a time, and a device's callback can be long: + // Chromium on PipeWire renders 2048 samples (43 ms) per callback even at "interactive", so 60 ms + // frames left 36-94 ms apart. Nothing here is audible, so render to no device, which Chromium 155+ + // paces every 10 ms. Older Chromium paces it at the default device's callback size, as before, and + // a browser without `sinkId` ignores it and keeps the default device. + // Don't lower latencyHint instead: at 128 samples, a busy host misses callbacks and the capture + // clock falls behind. + const options: AudioContextOptions & { sinkId: { type: "none" } } = { latencyHint: "interactive", sampleRate, - }); + sinkId: { type: "none" }, + }; + const context = new AudioContext(options); effect.cleanup(() => context.close()); // Nothing guarantees a gesture has happened yet: a pre-granted microphone reaches here on page diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index a031b9ba8a7..d2e536d282f 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -131,7 +131,9 @@ export type EncoderOutput = { // while there's no capture. catalog: Signal; // The head of the capture graph, so callers can tap the raw capture. Volume is applied to the - // PCM rather than in the graph, so this is pre-gain. Undefined for a source that isn't a track. + // PCM rather than in the graph, so this is pre-gain. Its context renders to no device where the + // browser supports `sinkId`, so nothing connected to its destination is heard. Undefined for a + // source that isn't a track. root: Signal; // True while encoding capture: enabled with a captured format and a track, which also takes a // subscriber unless the broadcast's `onDemand` is false. Retained audio encoded for a subscriber diff --git a/quest/m1/push-to-talk-latency.md b/quest/m1/push-to-talk-latency.md index 7b8a1554137..519d30f72f0 100644 --- a/quest/m1/push-to-talk-latency.md +++ b/quest/m1/push-to-talk-latency.md @@ -50,10 +50,14 @@ Biggest contributors and their knobs: AudioContext's `latencyHint` (`"interactive"` in `js/watch/src/audio/decoder.ts`), otherwise the platform. A real device, such as `kixtop`'s macOS output, should be measured before acting on it. -- **Capture and render, about 52 ms together.** Both follow the AudioContext - callback cadence on this host. The capture graph also delivers 60 ms frames - in bursts, 25-109 ms apart. Knob: the capture AudioContext's `latencyHint` - (`js/publish/src/audio/capture.ts`), otherwise the platform. +- **Capture and render, about 52 ms together.** Both followed the + AudioContext callback cadence on this host: PipeWire gives Chromium 2048 + samples (43 ms) per callback even at `"interactive"`. Capture no longer + does, since its context renders to no device + ([moq-dev/moq#7881](https://github.com/moq-dev/moq/pull/7881)): on this host + its median dropped from 29 ms to 1-9 ms, and the median per-press arrival + spread from 45 ms to 12 ms. Render still does. Knob: the watch AudioContext's + `latencyHint`, otherwise the platform. - **Opus frame, 60 ms.** This test chooses it; the publisher defaults to 20 ms. Under a fixed delay it is absorbed by the delay. In `"auto"` it adds one frame to the target. Knob: the encoder's `frameDuration`. @@ -66,7 +70,10 @@ fixed delay bypasses the jitter estimator. In `"auto"`, the 46 ms median per-press arrival-lateness spread, from capture bursts rather than the network, would make the measured term nonzero; that is the estimator working as specified in [Audio jitter -target](/quest/m1/audio-jitter-target/README.md). +target](/quest/m1/audio-jitter-target/README.md). With capture fixed, the +spread left on Chromium 155+ is its 10 ms capture chunks, plus the release +flush's lookahead chunk, which goes out up to a frame ahead of its media. +Older Chromium still paces capture at the device's callback size. Decided 2026-11-04 (maintainer, planning in [moq-dev/moq#7816](https://github.com/moq-dev/moq/pull/7816)): @@ -79,13 +86,11 @@ Decided 2026-11-04 (maintainer, planning in then. - ✅ Browser only. - The catalog toggle forces at least a round trip of jitter buffer, so the - budget is measured once push-to-talk runs on DTX (Required below), and once - capture delivers frames steadily. + budget is measured once push-to-talk runs on DTX (Required below). ## Required - [Push-to-talk suppresses silence instead of toggling the catalog](/quest/m1/push-to-talk/dtx.md) - removes the catalog round trip the budget would otherwise carry -- [Browser capture delivers audio frames on a steady cadence](/quest/m1/push-to-talk/capture-smoothing.md) - so `"auto"` measures the network, not capture bursts ## Related diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index b3a3038af32..b58a8ac218e 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -96,7 +96,6 @@ catches this. A complete set of runs finishes this epic. - [Pick how the browser publisher suppresses silence](/quest/m1/push-to-talk/dtx-route.md) - measures the DTX routes - [Push-to-talk suppresses silence instead of toggling the catalog](/quest/m1/push-to-talk/dtx.md) - replaces hold with DTX -- [Browser capture delivers audio frames on a steady cadence](/quest/m1/push-to-talk/capture-smoothing.md) - so `"auto"` measures the network ## Closes diff --git a/quest/m1/push-to-talk/capture-smoothing.md b/quest/m1/push-to-talk/capture-smoothing.md deleted file mode 100644 index 162d3925396..00000000000 --- a/quest/m1/push-to-talk/capture-smoothing.md +++ /dev/null @@ -1,27 +0,0 @@ -# [S] Browser capture delivers audio frames on a steady cadence - -## Goal - -`@moq/publish`'s capture graph hands encoded audio frames to the network at -the frame rate, not in bursts, so the listener's `"auto"` jitter target -measures the network instead of the capture graph. On an ideal path the -per-press arrival spread drops well below the ~46 ms measured on -[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760); the PR records -before and after on Chromium and on kixtop. - -## Plan - -Planned 2026-11-04 with the maintainer. Measured on #7760: the capture graph -delivered 60 ms frames 25-109 ms apart, and capture plus framing took 29 ms -median, 110 ms max. In `"auto"` that spread became about 46 ms of measured -jitter per press, padding the target. - -Decided 2026-11-04 (maintainer): reduce it, not just report it. Start by -finding where the bursts form: the AudioWorklet's posting cadence, the capture -AudioContext's callback size (already `latencyHint: "interactive"` in -`js/publish/src/audio/capture.ts`), or batching between the worklet and the -encoder. Fix at the source; don't re-pace on a timer. - -## Related - -- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the estimator that reads this spread diff --git a/quest/m1/push-to-talk/dtx.md b/quest/m1/push-to-talk/dtx.md index ca235a62ff2..86dffc020c7 100644 --- a/quest/m1/push-to-talk/dtx.md +++ b/quest/m1/push-to-talk/dtx.md @@ -34,6 +34,11 @@ and the maintainer would rather fix DTX and avoid the catalog entirely. - The publisher's mute becomes "suppress", not `enabled: false`. Keep `enabled` for a real stop (leaving the call), where a catalog change is correct. +- Mind the release flush's lookahead chunk. The disable path's `flush()` + sends it at once, up to a frame ahead of its media + ([moq-dev/moq#7881](https://github.com/moq-dev/moq/pull/7881) measured it adding + 10-22 ms to the median per-press spread). `"auto"` then measures an utterance that starts + within its 2 s window against it, so suppression should not reintroduce it. Update the epic's acceptance and `doc/` pages that describe hold. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 7fdfd1b2868..0f8265f70bf 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -162,8 +162,11 @@ it: `capture` (worklet delivery and framing, after the media the frame still nee press), `encode` (codec and container write), `network` (write to arrival, through the relay), `decode` (arrival to decoded PCM), `jitter_buffer` (the head's PCM to the first PCM that fills the playout delay), `render` (that PCM to the render onset: the worklet hand-off and the context's -render callback), and `device`, as the stages below name them. `latency` and `path` give each one's -min, median and max. +render callback), and `device`, as the stages below name them. `spread` is how far an utterance's frames +stray from a steady arrival: the latest arrival relative to its media timestamp minus the earliest, +which on this ideal path is the capture graph's cadence and what `"auto"`'s jitter estimate measures. +The release's flushed lookahead chunk is not one of the utterance's frames and is left out. `latency` +and `path` give each one's min, median and max. The command prints `latency`, `path`, and the completeness counts, and keeps its run directory even when it passes (set `HARNESS_KEEP=0` to drop it). Nightly runs it and keeps the raw evidence diff --git a/test/audio-quality/clients/js/ptt-analyze.test.ts b/test/audio-quality/clients/js/ptt-analyze.test.ts index 211f6a9e82c..0d0e0d7a100 100644 --- a/test/audio-quality/clients/js/ptt-analyze.test.ts +++ b/test/audio-quality/clients/js/ptt-analyze.test.ts @@ -85,6 +85,18 @@ test("a complete run passes, reports its latency, and splits each press exactly" expect(parts).toBe(press.audible); }); +test("a frame held back by a capture burst widens its press's spread", async () => { + const result = trace(); + const press = result.presses[4].at; + const held = result.arrivals.find((arrival) => arrival[0] > press + 3 * FRAME); + if (!held) throw new Error("no frame to hold"); + held[0] += 30; + const { exitCode, summary } = await analyze(result); + expect(exitCode).toBe(0); + expect(summary.presses[2].spread).toBeCloseTo(30); + expect(summary.latency.spread).toEqual({ min: 0, median: 0, max: expect.closeTo(30) }); +}); + test("the delay is filled by the first frame to reach it, not the newest before the onset", async () => { // The sixth frame decodes just before the onset, adding to a ring already leaving its stall. const result = trace(); diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 5aaa74d65d0..d027d000fc6 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -48,6 +48,10 @@ const presses = result.presses return [arrival[0] - press.at - (frame.timestamp - head) / 1000]; }); const delivery = waited.length ? Math.max(...waited) : null; + // What "auto"'s jitter estimate reads off this utterance: how far its frames' arrivals stray + // from their media timestamps, slowest minus fastest. An ideal path leaves only capture cadence. + const offsets = received.map(([at, media]) => at - media); + const spread = offsets.length ? Math.max(...offsets) - Math.min(...offsets) : null; // The head frame's path from the press to projected output, one boundary after another, so // the parts sum to `audible`. Capture and packetization split at the head frame's end on the // media clock, placed against the newest captured chunk delivered before the press. @@ -95,12 +99,13 @@ const presses = result.presses arrival: earliest ? earliest[0] - press.at : null, audible: audible ? audible.at - press.at : null, delivery, + spread, playout: audible && delivery !== null ? audible.rendered - press.at - delivery : null, output: audible ? audible.at - audible.rendered : null, path, }; }); -const spread = (values: (number | null)[]) => { +const summarize = (values: (number | null)[]) => { const sorted = values.flatMap((value) => (value === null ? [] : [value])).toSorted((a, b) => a - b); if (!sorted.length) return null; const middle = sorted.length / 2; @@ -111,11 +116,14 @@ const parts = ["capture", "packetization", "encode", "network", "decode", "jitte const summary = { // The goal: at most 300 ms of playout added on top of one-way delivery. latency: { - delivery: spread(presses.map((press) => press.delivery)), - playout: spread(presses.map((press) => press.playout)), - output: spread(presses.map((press) => press.output)), + delivery: summarize(presses.map((press) => press.delivery)), + playout: summarize(presses.map((press) => press.playout)), + output: summarize(presses.map((press) => press.output)), + spread: summarize(presses.map((press) => press.spread)), }, - path: Object.fromEntries(parts.map((part) => [part, spread(presses.map((press) => press.path?.[part] ?? null))])), + path: Object.fromEntries( + parts.map((part) => [part, summarize(presses.map((press) => press.path?.[part] ?? null))]), + ), delay: result.state?.delay, presses, captures: result.captures,