diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e9f16cc96d1..535e01abdba 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -496,6 +496,13 @@ jobs: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs # A passing run is kept too: budgets.json is measured from these nights. HARNESS_KEEP: 1 + - name: Push-to-talk + if: ${{ !cancelled() }} + run: nix develop --command just test push-to-talk + shell: bash -leo pipefail {0} + env: + HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs + HARNESS_KEEP: 1 # The native player's rows, graded but not enforced: an intermittent device # stall voids about one AAC row in seven, which would fail most nights until # it is fixed. Kept beside the browser run so its ceilings can be re-measured. diff --git a/quest/m1/README.md b/quest/m1/README.md index d9b1d057e8f..53cc4af456e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -160,7 +160,8 @@ per-frame CPU growth since #5926. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor -- [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - consumers hold a disabled rendition, and an unmute encodes at once with the mic still open +- [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - consumers hold a disabled rendition, and an unmute encodes at once with the mic still open; a reordered head after a long mute is still lost at the listener +- [Push-to-talk's press-to-audible latency is chosen, not inherited](/quest/m1/push-to-talk-latency.md) - each part of an ideal-path press's 469 ms, chiefly the 300 ms delay and 115 ms device output, is reduced through its knob or kept with a reason - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS - [CI housekeeping](/quest/m1/ci-housekeeping.md) - remove the actionlint queue ignore once a release after 1.7.12 reaches the dev shell diff --git a/quest/m1/push-to-talk-latency.md b/quest/m1/push-to-talk-latency.md new file mode 100644 index 00000000000..c46e3b1edad --- /dev/null +++ b/quest/m1/push-to-talk-latency.md @@ -0,0 +1,75 @@ +# [S] Push-to-talk's press-to-audible latency is chosen, not inherited + +## Goal + +Each part of a push-to-talk press's latency on an ideal path, from capture to +device output, is either reduced through the knob that controls it or kept +with a recorded reason. `just test push-to-talk` reports the result with the +same breakdown. + +## Plan + +Measured 2026-11-04 by +[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760), for +[Push-to-talk](/quest/m1/push-to-talk/README.md). The setup was `just test +push-to-talk` through a local relay, publisher and listener on separate +sessions, with no shaping or loss, 60 ms Opus at 128 kbit/s, and a fixed 300 +ms `delay`. Chromium 156 ran headless on Linux. There were five runs of 22 +presses each, 15 of them separated, so 75 separated presses in all; one press +lost its head (see Push-to-talk) and has no breakdown, leaving 74. +`summary.json` `path` splits each press at page-clock boundaries that sum +exactly to press → projected output. + +| Part | Boundaries | Median ms | Max ms | +| --- | --- | --- | --- | +| capture | worklet delivery and framing | 29 | 110 | +| packetization | media the head frame still needed after the press | 39 | 60 | +| encode | encoder input to container write | 0.8 | 14 | +| network | container write to arrival, through the relay | 3.2 | 15 | +| decode | arrival to decoded PCM | 0.7 | 13 | +| jitter buffer | head's PCM to the first PCM that filled the delay | 252 | 268 | +| render | that PCM to the render onset | 21 | 54 | +| device | render onset to projected output | 115 | 119 | +| total | press to projected output | 469 | 576 | + +The capture and packetization split is approximate, but their 66 ms median sum +is exact. Playout on top of one-way delivery was 261 ms median and 294 ms max +over all 75, inside the 300 ms the maintainer set. + +Biggest contributors and their knobs: + +- **Playout delay, about 300 ms.** The ring starts once it holds the delay in + decoded PCM, counted from the head's first sample. That is why the jitter + buffer plus packetization sum to about 291 ms. Knob: `` `delay`. + This test fixes it at 300 ms. In `"auto"`, the default, the target is + `max(measured, advertised) + frame + catalog delay` + (`js/watch/src/audio/latency.ts`). +- **Device output, about 115 ms.** This matches Chromium's declared + `baseLatency + outputLatency` on this headless Linux host. Knob: the watch + AudioContext's `latencyHint` (`"interactive"` in + `js/watch/src/audio/decoder.ts`), otherwise the platform. A real device, + such as `kixtop`'s macOS output, should be measured before acting on it. +- **Capture and render, about 52 ms together.** Both follow the AudioContext + callback cadence on this host. The capture graph also delivers 60 ms frames + in bursts, 25-109 ms apart. Knob: the capture AudioContext's `latencyHint` + (`js/publish/src/audio/capture.ts`), otherwise the platform. +- **Opus frame, 60 ms.** This test chooses it; the publisher defaults to + 20 ms. Under a fixed delay it is absorbed by the delay. In `"auto"` it adds + one frame to the target. Knob: the encoder's `frameDuration`. + +No part looked like a bug. On an ideal path, encode and decode take about 1 ms +and the relay hop about 3 ms, and the head is not trimmed as decoder warmup. +The one lost head is an ordering defect, owned by +[Push-to-talk](/quest/m1/push-to-talk/README.md), not a latency part. The +fixed delay bypasses the jitter estimator. In `"auto"`, the 46 ms median +per-press arrival-lateness spread, from capture bursts rather than the +network, would make the measured term nonzero; that is the estimator working +as specified in [Audio jitter +target](/quest/m1/audio-jitter-target/README.md). + +Open: decide which knob to turn first, and whether to measure on a real +device before deciding. + +## Related + +- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the `"auto"` target this latency feeds diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index f7bd47307cc..917c1a84e7c 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -36,23 +36,61 @@ Decided 2026-10-28 (maintainer): replaces a publisher head buffer, which was considered and dropped as the same idea special-cased. -The epic's own work: once the children land, measure push-to-talk (22 -presses, 300 ms delay) behind bufferbloat: a rate limit below the media -bitrate, behind a deep but finite queue whose overflow drops packets. That -gates acceptance. A held listener's unmute pays no catalog round trip and no -resubscribe, so unmute → audible is the path's one-way delay plus the 300 ms. -Every frame of the utterance is encoded and sent; any frames dropped come -only from queue overflow. Rerun the reporter's setup (100 ms RTT, 2% random -loss) only as a reproduction of the original numbers, not as acceptance. -Record both in the PR. - -Also check failure recovery (found 2026-10-28 by -[moq-dev/moq#7157](https://github.com/moq-dev/moq/pull/7157)): unmuting used to -reopen the microphone, which made a new source and so recovered a failed -encoder. Capture now stays open, so only a device swap, a settings change or a -new `source` recovers one. Decide whether re-enabling should retry a failed -encoder, and record why. +The epic's own work: the acceptance proof `just test push-to-talk` (22 +presses, 60 ms Opus, 300 ms delay). A held listener's unmute pays no catalog +round trip and no resubscribe, and every frame of each utterance is encoded, +sent, received, and decoded from its first frame. + +Decided with the maintainer on +[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760): + +- ✅ **Ideal conditions only** (2026-11-04): "don't use random packet loss + either, assume ideal conditions." Both elements reach a local relay + directly, on separate sessions so the media crosses it, with no shaping, + loss, or queueing. The bufferbloat and 2% loss profiles were removed. +- ✅ **Latency: at most 300 ms of playout on top of one-way delivery** + (2026-11-03): "We can't ever expect 300ms latency if it takes more than + 300ms to deliver a packet." No clocked live start or other startup + machinery. Five ideal runs on 2026-11-04 held it on all 75 separated presses + (median 261 ms, max 294 ms). The breakdown and its knobs are in + [Push-to-talk latency](/quest/m1/push-to-talk-latency.md). +- ✅ **No encoder recovery on re-enable** (2026-11-03). Asked first why a + fatal encoder error is expected at all: it is not. No push-to-talk run + reported one, and no issue reports a publisher encoder failure; the latch + came from a review finding + ([moq-dev/moq#5441](https://github.com/moq-dev/moq/pull/5441)). WebCodecs + fails an encoder only for a config the platform rejects, an internal codec + error, or reclamation, which Chromium applies only to inactive, backgrounded + hardware codecs; Opus is software. A failure stays latched until a device, + settings or `source` change. + +The publisher's tail drop is fixed: +[moq-dev/moq#7725](https://github.com/moq-dev/moq/pull/7725) flushes the codec +before a disable's discontinuity. A frame submitted under 1 ms before release +used to be lost in 3 of 7 runs; it was lost in none of 12 since. + +What remains is a lost head at the listener. In one of 110 presses over the +2026-11-04 relay runs (run 3, press 20, after a 60 s mute), the +head's group 918 and the next frame's group 919, written at +72 and +73 ms, +reached the listener reordered: 919 at +187 ms and 918 at +192 ms, about 115 +ms in transit against the usual 3 ms. The listener decoded 919 at +191.7 ms. +Group 918 arrived 0.4 ms later and never reached the decoder, so the utterance +played from its second frame, 60 ms late. The JS ordered read still delivers a +group below its cursor as a late backfill +(`js/hang/src/container/consumer.ts`), so the head was most likely dropped as +late between the read and the decoder (`js/watch/src/audio/decoder.ts`); pin +that down first. After a declared end, the next group is known to be the end's +group plus one. Holding for it within the delay budget would keep the head +without adding startup latency, but weigh that against the no-window decision +in [Container reads drop backfill](/quest/m1/read-backfill.md). + +The runner fails on a head that is received but never decoded, so the proof +catches this. A complete set of runs finishes this epic. ## Closes - [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost + +## Related + +- [Container reads drop backfill](/quest/m1/read-backfill.md) - changes how the ordered read treats a group below its cursor, where this head was lost diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index d5275925e8a..7fdfd1b2868 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -121,6 +121,54 @@ changed during the live run. The observer only records frames delivered under the player's maximum age, so a replay cannot recover groups the transport already discarded. +## Push-to-talk + +```bash +just test push-to-talk +``` + +This separate, roughly nine-minute run drives the shipped publish and watch elements in one Chromium +page. A constant tone loops through the fake microphone, encoded as 128 kbit/s Opus with 60 ms +frames. The listener holds a fixed 300 ms delay. After warmup and a mute, it measures 22 presses, +alternating two and three seconds: the first after the warmup's five-second mute, seven with no +intervening wait, seven after five seconds, and seven after sixty seconds. Both elements reach a +local relay directly over WebTransport, with no shaping, loss, or queueing, so every millisecond +measured belongs to the media pipeline rather than the path. + +`presses.json` records captured chunks at each press, WebCodecs encoder and decoder inputs and +outputs, container writes, frame arrivals before reordering, and audible output onsets. Publisher, +receiver, and worklet observations share the page's monotonic clock. A listening worklet detects +each nonzero output onset. Its render time is mapped through the context's render clock, read with +the page clock when the port message lands, so a late message moves neither boundary; its projected +output comes from `getOutputTimestamp`. This measures the browser's projected device output, not +acoustic output from a physical speaker. Zero-gap presses can remain continuously audible, and a +later onset can belong to the preceding utterance. Their audible latency is always null. Only +separated presses have an identifiable onset. + +`summary.json` matches inputs, outputs, writes, and arrivals by media timestamp, so a late arrival +is still attributed to its utterance. The command checks pipeline completeness, not the epic's +latency verdict. It refuses missing encoder inputs, a head that never reaches the listener or never +decodes, submitted frames that never encode or reach the container, written frames that never reach +the listener, reopened capture, a new encoder (a probe or a rebuilt codec) or subscription, and +separated utterances with no audible output. It also fails on a broken driver or invalid capture. + +The latency goal is at most 300 ms of playout added on top of one-way delivery. For each separated +press, `delivery` is the slowest of the utterance's frames that arrived before its onset, measured +from that frame's first sample (its media offset after the press). `playout` is what remains of the +render onset after the press and that delivery, and `output` is the projected device output after +that onset. `path` follows the head frame from the press to projected output in parts that sum to +it: `capture` (worklet delivery and framing, after the media the frame still needed), +`packetization` (that media, placed on the media clock by the newest chunk captured before the +press), `encode` (codec and container write), `network` (write to arrival, through the relay), +`decode` (arrival to decoded PCM), `jitter_buffer` (the head's PCM to the first PCM that fills the +playout delay), `render` (that PCM to the render onset: the worklet hand-off and the context's +render callback), and `device`, as the stages below name them. `latency` and `path` give each one's +min, median and max. + +The command prints `latency`, `path`, and the completeness counts, and keeps its run directory +even when it passes (set `HARNESS_KEEP=0` to drop it). Nightly runs it and keeps the raw evidence +alongside the audio quality rows. + ## Traces The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, each graded as a diff --git a/test/audio-quality/clients/js/ptt-analyze.test.ts b/test/audio-quality/clients/js/ptt-analyze.test.ts new file mode 100644 index 00000000000..211f6a9e82c --- /dev/null +++ b/test/audio-quality/clients/js/ptt-analyze.test.ts @@ -0,0 +1,128 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const FRAME = 60; + +// 22 complete presses on an ideal path: the media clock is the page clock, every 60 ms frame is +// encoded, written, received and decoded a millisecond apart, and each separated press renders +// once the 300 ms delay is decoded, with 100 ms of projected device output after it. +function trace() { + const result = { + submitted: [] as { at: number; timestamp: number; duration: number }[], + encoded: [] as { at: number; timestamp: number; bytes: number }[], + sent: [] as { at: number; timestamp: number; bytes: number }[], + decoding: [] as { at: number; timestamp: number }[], + decoded: [] as { at: number; timestamp: number; duration: number }[], + subscriptions: [] as unknown[], + audible: [] as { at: number; rendered: number; received: number }[], + presses: [] as { at: number; on: boolean; capture?: { at: number; end: number } }[], + captures: [] as unknown[], + encoders: [] as number[], + errors: [] as string[], + arrivals: [] as [number, number, number, number][], + state: { delay: 300 }, + }; + for (let i = 0; i < 22; i++) { + const press = 10_000 + i * 10_000; + const release = press + 2000; + // The press lands 20 ms into a frame, so the head frame still needs 40 ms of media. + result.presses.push({ at: press, on: true, capture: { at: press, end: press * 1000 } }); + const start = press - 20; + for (let media = start; media + FRAME <= release; media += FRAME) { + const at = media + FRAME + 25; + if (at >= release) break; + const timestamp = media * 1000; + result.submitted.push({ at, timestamp, duration: FRAME * 1000 }); + result.encoded.push({ at: at + 1, timestamp, bytes: 959 }); + result.sent.push({ at: at + 1, timestamp, bytes: 959 }); + result.arrivals.push([at + 2, media, i, FRAME]); + result.decoding.push({ at: at + 2, timestamp }); + result.decoded.push({ at: at + 3, timestamp, duration: FRAME * 1000 }); + } + if (i === 0 || i >= 8) { + // The fifth frame completes the delay; the render onset follows its PCM by 10 ms. + const rendered = start + 5 * FRAME + 25 + 3 + 10; + result.audible.push({ at: rendered + 100, rendered, received: rendered + 5 }); + } + result.presses.push({ at: release, on: false }); + } + return result; +} + +async function analyze(result: ReturnType) { + const directory = mkdtempSync(join(tmpdir(), "ptt-analyze-")); + try { + await Bun.write(join(directory, "presses.json"), JSON.stringify(result)); + const run = Bun.spawnSync(["bun", fileURLToPath(new URL("./ptt-analyze.ts", import.meta.url)), directory]); + const summary = await Bun.file(join(directory, "summary.json")).json(); + return { exitCode: run.exitCode, stdout: run.stdout.toString(), stderr: run.stderr.toString(), summary }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +test("a complete run passes, reports its latency, and splits each press exactly", async () => { + const { exitCode, stdout, stderr, summary } = await analyze(trace()); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout).latency.playout).not.toBeNull(); + + const press = summary.presses[0]; + expect(press.path).toEqual({ + capture: 25, + packetization: 40, + encode: 1, + network: 1, + decode: 1, + jitter_buffer: 4 * FRAME, + render: 10, + device: 100, + }); + const parts = Object.values(press.path as Record).reduce((sum, part) => sum + part, 0); + expect(parts).toBe(press.audible); +}); + +test("the delay is filled by the first frame to reach it, not the newest before the onset", async () => { + // The sixth frame decodes just before the onset, adding to a ring already leaving its stall. + const result = trace(); + const sixth = (result.presses[0].at - 20 + 5 * FRAME) * 1000; + const late = result.decoded.find((frame) => frame.timestamp === sixth); + if (!late) throw new Error("no sixth frame"); + late.at = result.audible[0].rendered - 1; + const { exitCode, summary } = await analyze(result); + expect(exitCode).toBe(0); + expect(summary.presses[0].path.jitter_buffer).toBe(4 * FRAME); + expect(summary.presses[0].path.render).toBe(10); +}); + +test("a head that never reaches the listener fails the run", async () => { + const result = trace(); + const head = result.submitted.find((frame) => frame.at >= result.presses[2].at); + result.arrivals = result.arrivals.filter((arrival) => arrival[1] * 1000 !== head?.timestamp); + const { exitCode, stderr } = await analyze(result); + expect(exitCode).toBe(1); + expect(stderr).toContain("utterance head was not received"); +}); + +test("a head that arrives but never reaches the decoder fails the run", async () => { + // A head reordered behind the next frame's group, then dropped before decode. + const result = trace(); + const head = result.submitted.find((frame) => frame.at >= result.presses[2].at); + result.decoding = result.decoding.filter((frame) => frame.timestamp !== head?.timestamp); + result.decoded = result.decoded.filter((frame) => frame.timestamp !== head?.timestamp); + const { exitCode, stderr } = await analyze(result); + expect(exitCode).toBe(1); + expect(stderr).toContain("utterance head was not decoded"); +}); + +test("a new encoder fails as a probe, but the live encoder's release flush does not", async () => { + // A flush leaves no trace in the page's records, so the passing run above is the release flush. + const result = trace(); + result.encoders.push(result.presses[2].at + 1); + const { exitCode, stderr } = await analyze(result); + expect(exitCode).toBe(1); + expect(stderr).toContain("an encoder was created again"); +}); diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts new file mode 100644 index 00000000000..5aaa74d65d0 --- /dev/null +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -0,0 +1,151 @@ +/** Reduce push-to-talk evidence without substituting missing observations with zero. @module */ +import { join } from "node:path"; +import type { measurement } from "./src/ptt"; + +// The driver's final write adds the page's state to the result. +type Result = ReturnType & { state?: ReturnType }; +const [directory, ...rest] = process.argv.slice(2); +if (!directory || rest.length) throw new Error("usage: ptt-analyze.ts RUN_DIR"); +const result: Result = await Bun.file(join(directory, "presses.json")).json(); +if (result.presses.length !== 44 || result.presses.some((press, i) => press.on !== (i % 2 === 0))) { + throw new Error("expected 22 complete presses"); +} +if (result.captureError || result.errors.length) throw new Error("invalid capture"); +const encoded = new Map(result.encoded.map((frame) => [frame.timestamp, frame])); +const sent = new Map(result.sent.map((frame) => [frame.timestamp, frame])); +const arrivals = new Map(result.arrivals.map((frame) => [Math.round(frame[1] * 1000), frame])); +// The first decode of each timestamp; a later one would be a re-decode, not the frame's first. +const decoded = new Map(); +for (const frame of result.decoded) if (!decoded.has(frame.timestamp)) decoded.set(frame.timestamp, frame); +const presses = result.presses + .filter((press) => press.on) + .map((press, i) => { + const release = result.presses[i * 2 + 1]; + if (!release || release.on) throw new Error(`press ${i + 1} has no release`); + const input = result.submitted.filter((frame) => frame.at >= press.at && frame.at < release.at); + const output = input.flatMap((frame) => encoded.get(frame.timestamp) ?? []); + const published = output.flatMap((frame) => sent.get(frame.timestamp) ?? []); + const received = published.flatMap((frame) => { + const received = arrivals.get(frame.timestamp); + return received ? [received] : []; + }); + const earliest = received.toSorted((a, b) => a[0] - b[0])[0]; + // An onset during a zero-gap press may belong to its still-playing predecessor. + // A constant tone has no utterance ID, so only separated presses have identifiable onsets. + const separated = i === 0 || i >= 8; + const audible = separated + ? result.audible.find( + (frame) => frame.at >= press.at && frame.at < (result.presses[(i + 1) * 2]?.at ?? Infinity), + ) + : undefined; + // One-way delivery is the slowest frame the start waited on, measured from its first sample's + // capture: it covers packetization, transport, queueing and retransmission. What remains of the + // start is the player's own added latency; projected device output is reported apart from it. + const head = input[0]?.timestamp; + const waited = input.flatMap((frame) => { + const arrival = arrivals.get(frame.timestamp); + if (!audible || head === undefined || !arrival || arrival[0] > audible.rendered) return []; + return [arrival[0] - press.at - (frame.timestamp - head) / 1000]; + }); + const delivery = waited.length ? Math.max(...waited) : null; + // The head frame's path from the press to projected output, one boundary after another, so + // the parts sum to `audible`. Capture and packetization split at the head frame's end on the + // media clock, placed against the newest captured chunk delivered before the press. + const first = input[0]; + const path = (() => { + if (!audible || !first || !press.capture) return null; + const write = sent.get(first.timestamp); + const arrival = arrivals.get(first.timestamp); + const output = decoded.get(first.timestamp); + // The first PCM that reached the delay from the head: the ring leaves its stall on it, and a + // later frame decoded before the onset only adds to a ring already playing. + const delay = result.state?.delay; + const filled = + delay === undefined + ? undefined + : input + .flatMap((frame) => decoded.get(frame.timestamp) ?? []) + .toSorted((a, b) => a.at - b.at) + .find((frame) => frame.timestamp + (frame.duration ?? 0) - first.timestamp >= delay * 1000); + if (!write || !arrival || !output || !filled) return null; + const pressed = press.capture.end / 1000 + (press.at - press.capture.at); + const packetization = (first.timestamp + (first.duration ?? 0)) / 1000 - pressed; + return { + capture: first.at - press.at - packetization, + packetization, + encode: write.at - first.at, + network: arrival[0] - write.at, + decode: output.at - arrival[0], + jitter_buffer: filled.at - output.at, + render: audible.rendered - filled.at, + device: audible.at - audible.rendered, + }; + })(); + return { + press: i + 1, + gap: result.presses[i * 2 - 1] ? press.at - result.presses[i * 2 - 1].at : null, + duration: release.at - press.at, + submitted: input.length, + encoded: output.length, + sent: published.length, + received: received.length, + firstEncoded: input[0] ? encoded.has(input[0].timestamp) : false, + firstReceived: input[0] ? arrivals.has(input[0].timestamp) : false, + firstDecoded: input[0] ? decoded.has(input[0].timestamp) : false, + arrival: earliest ? earliest[0] - press.at : null, + audible: audible ? audible.at - press.at : null, + delivery, + playout: audible && delivery !== null ? audible.rendered - press.at - delivery : null, + output: audible ? audible.at - audible.rendered : null, + path, + }; + }); +const spread = (values: (number | null)[]) => { + const sorted = values.flatMap((value) => (value === null ? [] : [value])).toSorted((a, b) => a - b); + if (!sorted.length) return null; + const middle = sorted.length / 2; + const median = sorted.length % 2 ? sorted[Math.floor(middle)] : (sorted[middle - 1] + sorted[middle]) / 2; + return { min: sorted[0], median, max: sorted[sorted.length - 1] }; +}; +const parts = ["capture", "packetization", "encode", "network", "decode", "jitter_buffer", "render", "device"] as const; +const summary = { + // The goal: at most 300 ms of playout added on top of one-way delivery. + latency: { + delivery: spread(presses.map((press) => press.delivery)), + playout: spread(presses.map((press) => press.playout)), + output: spread(presses.map((press) => press.output)), + }, + path: Object.fromEntries(parts.map((part) => [part, spread(presses.map((press) => press.path?.[part] ?? null))])), + delay: result.state?.delay, + presses, + captures: result.captures, + encoders: result.encoders, + subscriptions: result.subscriptions, + unencoded: result.submitted.filter((frame) => !encoded.has(frame.timestamp)).length, + unsent: result.encoded.filter((frame) => !sent.has(frame.timestamp)).length, + // An ideal path loses nothing, so every written frame must reach the listener. + unreceived: result.sent.filter((frame) => !arrivals.has(frame.timestamp)).length, +}; +await Bun.write(join(directory, "summary.json"), JSON.stringify(summary, null, 2)); +// The report: the latency and its parts, and the completeness the run gates on. Per-press detail +// stays in summary.json. +const { latency, path, delay, unencoded, unsent, unreceived } = summary; +console.log(JSON.stringify({ latency, path, delay, unencoded, unsent, unreceived }, null, 2)); + +const failures: string[] = []; +if (presses.some((press, i) => (i === 0 || i >= 8) && press.audible === null)) { + failures.push("a separated utterance never reached the output"); +} +if (result.captures.length) failures.push("capture reopened"); +if (result.encoders.length) failures.push("an encoder was created again"); +if (result.subscriptions.length) failures.push("subscription reopened"); +if (presses.some((press) => !press.submitted || !press.firstEncoded)) failures.push("utterance head was not encoded"); +if (summary.unencoded || summary.unsent) failures.push("submitted audio was not encoded and sent completely"); +if (presses.some((press) => !press.firstReceived)) failures.push("utterance head was not received"); +if (presses.some((press) => press.firstReceived && !press.firstDecoded)) + failures.push("utterance head was not decoded"); +if (summary.unreceived) failures.push("written audio did not all reach the listener"); +if (failures.length) { + console.error(failures.join("; ")); + process.exitCode = 1; +} diff --git a/test/audio-quality/clients/js/ptt.html b/test/audio-quality/clients/js/ptt.html new file mode 100644 index 00000000000..3c438dcd36f --- /dev/null +++ b/test/audio-quality/clients/js/ptt.html @@ -0,0 +1,5 @@ + + +Push-to-talk measurement + + diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts new file mode 100644 index 00000000000..a4b4946bee8 --- /dev/null +++ b/test/audio-quality/clients/js/ptt.ts @@ -0,0 +1,99 @@ +/** Run 22 microphone presses through the real publisher, relay, and player. @module */ +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { createServer } from "vite"; +import { worklet } from "../../../../js/common/vite-plugin-worklet"; +import { launch } from "../../../interop/clients/js/harness"; +import type { measurement } from "./src/ptt"; + +const { values } = parseArgs({ + options: { + relay: { type: "string" }, + out: { type: "string" }, + }, +}); +const { relay, out } = values; +if (!relay || !out) throw new Error("usage: ptt.ts --relay URL --out DIR"); + +// Chromium loops this constant tone as its microphone. Its default fake device has silent +// intervals of its own, which would make an audible onset ambiguous. +const wav = Buffer.alloc(44 + 48_000 * 2); +wav.write("RIFF", 0); +wav.writeUInt32LE(wav.length - 8, 4); +wav.write("WAVEfmt ", 8); +wav.writeUInt32LE(16, 16); +wav.writeUInt16LE(1, 20); +wav.writeUInt16LE(1, 22); +wav.writeUInt32LE(48_000, 24); +wav.writeUInt32LE(96_000, 28); +wav.writeUInt16LE(2, 32); +wav.writeUInt16LE(16, 34); +wav.write("data", 36); +wav.writeUInt32LE(wav.length - 44, 40); +for (let i = 0; i < 48_000; i++) + wav.writeInt16LE(Math.round(8192 * Math.sin((2 * Math.PI * 440 * i) / 48_000)), 44 + i * 2); +const tone = join(out, "tone.wav"); +await Bun.write(tone, wav); + +const server = await createServer({ + configFile: false, + root: import.meta.dir, + plugins: [worklet()], + server: { host: "127.0.0.1", port: 0 }, +}); +await server.listen(); +const browser = await launch([ + "--autoplay-policy=no-user-gesture-required", + "--use-fake-device-for-media-stream", + "--use-fake-ui-for-media-stream", + `--use-file-for-fake-audio-capture=${tone}`, +]); +try { + const page = await browser.newPage(); + const pageErrors: string[] = []; + page.on("pageerror", (error) => pageErrors.push(String(error))); + await page.goto(`${server.resolvedUrls?.local[0]}ptt.html?${new URLSearchParams({ relay })}`); + await page.waitForFunction(() => globalThis.ptt?.state().selected && globalThis.ptt.state().tapped, undefined, { + timeout: 30_000, + }); + const initial = await page.evaluate(() => globalThis.ptt.state()); + if (initial.publisher !== "webtransport" || initial.viewer !== "webtransport") throw new Error("not WebTransport"); + if (!initial.separate) throw new Error("the watcher shares the publisher's origin, bypassing the relay"); + await page.waitForTimeout(3000); + await page.evaluate(() => globalThis.ptt.press(false)); + await page.waitForTimeout(5000); + const held = await page.evaluate(() => globalThis.ptt.state()); + if (held.active || held.config?.enabled !== false || held.selected !== initial.selected) { + throw new Error("the muted listener did not hold its rendition"); + } + // Tune-in can consume retained audio. Validate the tap on a fresh press only after the muted + // listener holds its subscription. + await page.evaluate(() => globalThis.ptt.press(true)); + await page.waitForTimeout(3000); + await page.evaluate(() => globalThis.ptt.press(false)); + await page.waitForTimeout(5000); + const warmup = await page.evaluate(() => globalThis.ptt.result()); + await Bun.write(join(out, "warmup.json"), JSON.stringify({ browser: browser.version(), initial, held, ...warmup })); + if (!warmup.audible.length) throw new Error("the tone never reached the output tap"); + await page.evaluate(() => globalThis.ptt.reset()); + for (let i = 0; i < 22; i++) { + await page.waitForTimeout(i < 8 ? 0 : i < 15 ? 5000 : 60_000); + await page.evaluate(() => globalThis.ptt.press(true)); + await page.waitForTimeout(i % 2 === 0 ? 2000 : 3000); + await page.evaluate(() => globalThis.ptt.press(false)); + console.log(`press ${i + 1}/22`); + await Bun.write(join(out, "presses.json"), JSON.stringify(await page.evaluate(() => globalThis.ptt.result()))); + } + await page.waitForTimeout(5000); + const result: ReturnType = await page.evaluate(() => globalThis.ptt.result()); + const state = await page.evaluate(() => globalThis.ptt.state()); + await Bun.write( + join(out, "presses.json"), + JSON.stringify({ browser: browser.version(), ...result, state, pageErrors }), + ); + if (pageErrors.length || result.errors.length || result.captureError || state.error) + throw new Error("measurement failed; see presses.json"); +} finally { + await browser.close(); + await server.close(); +} diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts new file mode 100644 index 00000000000..90c645aae7c --- /dev/null +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -0,0 +1,232 @@ +/** Measure the shipped push-to-talk path on one monotonic page clock. @module */ +import "@moq/publish/element"; +import "@moq/watch/element"; +import { Milli } from "@moq/clock"; +import * as Container from "@moq/hang/container"; +import { Track } from "@moq/net"; +import type MoqPublish from "@moq/publish/element"; +import { Derived } from "@moq/signals"; +import type MoqWatch from "@moq/watch/element"; +import { Capture } from "./capture"; +import type { Arrival } from "./schema"; + +/** One timestamped encoder input or output, on the page and media clocks. */ +type Frame = { at: number; timestamp: number; bytes?: number; duration?: number | null }; + +const records = { + submitted: [] as Frame[], + encoded: [] as Frame[], + sent: [] as Frame[], + decoding: [] as Frame[], + decoded: [] as Frame[], + subscriptions: [] as { at: number; name: string }[], + audible: [] as { at: number; rendered: number; received: number }[], + // The newest captured chunk delivered before a press, mapping the press onto the media clock. + presses: [] as { at: number; on: boolean; capture?: { at: number; end: number } }[], + captures: [] as { at: number; audio: boolean }[], + encoders: [] as number[], + errors: [] as string[], +}; + +// Observe WebCodecs without changing its input, output, or scheduling. A probe or a rebuilt codec +// constructs a new encoder; the live one flushes at each release, so a flush alone is no probe. +const NativeEncoder = globalThis.AudioEncoder; +globalThis.AudioEncoder = class extends NativeEncoder { + constructor(init: AudioEncoderInit) { + records.encoders.push(performance.now()); + super({ + ...init, + output: (chunk, metadata) => { + records.encoded.push({ at: performance.now(), timestamp: chunk.timestamp, bytes: chunk.byteLength }); + init.output(chunk, metadata); + }, + }); + } + override encode(data: AudioData): void { + records.submitted.push({ at: performance.now(), timestamp: data.timestamp, duration: data.duration }); + super.encode(data); + } +}; +const NativeDecoder = globalThis.AudioDecoder; +globalThis.AudioDecoder = class extends NativeDecoder { + constructor(init: AudioDecoderInit) { + super({ + ...init, + output: (data) => { + records.decoded.push({ at: performance.now(), timestamp: data.timestamp, duration: data.duration }); + init.output(data); + }, + }); + } + override decode(chunk: EncodedAudioChunk): void { + records.decoding.push({ at: performance.now(), timestamp: chunk.timestamp }); + super.decode(chunk); + } +}; +// Note each captured chunk as the publisher's graph delivers it, without consuming it. +let captured: { at: number; end: number } | undefined; +const NativeNode = globalThis.AudioWorkletNode; +globalThis.AudioWorkletNode = class extends NativeNode { + constructor(context: BaseAudioContext, name: string, options?: AudioWorkletNodeOptions) { + super(context, name, options); + if (name !== "capture") return; + this.port.addEventListener( + "message", + ({ data }: MessageEvent<{ timestamp: number; channels: Float32Array[] }>) => { + const frames = data.channels[0]?.length ?? 0; + captured = { at: performance.now(), end: data.timestamp + (frames / context.sampleRate) * 1_000_000 }; + }, + ); + } +}; +const encode = Container.Legacy.Producer.prototype.encode; +Container.Legacy.Producer.prototype.encode = function (frame, timestamp, keyframe) { + const result = encode.call(this, frame, timestamp, keyframe); + records.sent.push({ at: performance.now(), timestamp, bytes: frame.byteLength }); + return result; +}; +const subscribe = Track.Consumer.prototype.subscribe; +Track.Consumer.prototype.subscribe = function (options) { + records.subscriptions.push({ at: performance.now(), name: this.name }); + return subscribe.call(this, options); +}; +const getUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices); +navigator.mediaDevices.getUserMedia = (constraints) => { + records.captures.push({ at: performance.now(), audio: !!constraints?.audio }); + return getUserMedia(constraints); +}; + +const params = new URLSearchParams(location.search); +const required = (name: string) => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; +const publish = document.createElement("moq-publish") as MoqPublish; +publish.url = required("relay"); +publish.name = "ptt.hang"; +publish.invisible = true; +publish.audio.codec.set({ mime: "opus", frameDuration: Milli(60), bitrate: 128_000 }); +publish.source = "camera"; +document.body.append(publish); + +const watch = document.createElement("moq-watch") as MoqWatch; +// Connections pool by URL, and a pooled origin resolves a same-page publication locally. A +// listener-only query, which the relay ignores, gives the watcher its own session, so the media +// crosses the relay. +const listener = new URL(required("relay")); +listener.searchParams.set("role", "listener"); +watch.url = listener.toString(); +watch.name = publish.name; +watch.setAttribute("delay", "300ms"); +watch.setAttribute("visible", "always"); +watch.volume = 1; +watch.muted = false; +watch.paused = false; +document.body.append(watch); + +const audio = watch.player.audio; +const capture = new Capture({ + broadcast: audio.source.in.broadcast, + track: audio.source.out.track, + config: audio.source.out.config, + delay: watch.player.sync.in.delay, + // Decoder subscriptions include codec warmup beyond Sync's budget. The observer must + // request the same age so its own subscription cannot discard an otherwise visible frame. + maxDelay: new Derived([watch.player.sync.out.maxDelay, audio.source.out.config], (max, config) => + Milli(max + (config?.warmup ?? 0)), + ), +}); +const arrivals: Arrival[] = []; +let attached: AudioNode | undefined; +let tapped: AudioNode | undefined; +async function attach() { + const root = audio.out.root.peek(); + if (!root || root === attached) return; + attached = root; + const context = root.context as AudioContext; + // A constant microphone tone makes a nonzero output quantum the utterance's audible onset. + // Map its render timestamp through getOutputTimestamp, rather than timing the port message. + const source = `class Tap extends AudioWorkletProcessor { + loud = false; + process(inputs) { + const loud = !!inputs[0]?.[0]?.some(s => Math.abs(s) > 0.001); + if (loud && !this.loud) this.port.postMessage(currentTime); + this.loud = loud; + return true; + } + } + registerProcessor('ptt-tap', Tap);`; + const url = URL.createObjectURL(new Blob([source], { type: "text/javascript" })); + try { + await context.audioWorklet.addModule(url); + } catch (error) { + if (audio.out.root.peek() !== root) return; + throw error; + } finally { + URL.revokeObjectURL(url); + } + if (audio.out.root.peek() !== root) return; + const tap = new AudioWorkletNode(context, "ptt-tap"); + const mute = new GainNode(context, { gain: 0 }); + root.connect(tap); + tap.connect(mute); + mute.connect(context.destination); + tap.port.onmessage = ({ data }: MessageEvent) => { + const stamp = context.getOutputTimestamp(); + if (stamp.performanceTime === undefined || stamp.contextTime === undefined) { + records.errors.push("missing output clock"); + return; + } + // Map the onset's render time through the render clock read at this instant, so a late port + // message moves neither the render onset nor its projected output. + const received = performance.now(); + records.audible.push({ + at: stamp.performanceTime + (data - stamp.contextTime) * 1000, + rendered: received - (context.currentTime - data) * 1000, + received, + }); + }; + tapped = root; +} +audio.out.root.subscribe(() => void attach().catch((error) => records.errors.push(String(error)))); + +/** The finite measurement surface driven by ptt.ts. */ +export const measurement = { + press(on: boolean) { + const at = performance.now(); + records.presses.push(on && captured ? { at, on, capture: { ...captured } } : { at, on }); + publish.muted = !on; + }, + state() { + return { + publisher: publish.connection.transport.peek() as string | undefined, + viewer: watch.connection.transport.peek() as string | undefined, + // Distinct origins: the watcher can only reach the publication through the relay. + separate: watch.connection.origin.peek() !== publish.connection.origin.peek(), + config: publish.audio.out.catalog.peek(), + selected: audio.source.out.track.peek(), + tapped: tapped !== undefined && tapped === audio.out.root.peek(), + active: publish.audio.out.active.peek(), + context: audio.out.context.peek()?.state, + stalled: audio.out.stalled.peek(), + timestamp: audio.out.timestamp.peek(), + delay: watch.player.sync.out.delay.peek(), + error: String(publish.audio.out.error.peek() ?? audio.out.error.peek() ?? ""), + }; + }, + reset() { + for (const list of Object.values(records)) list.length = 0; + capture.drain(); + arrivals.length = 0; + }, + result() { + arrivals.push(...capture.drain()); + return { ...records, arrivals, captureError: capture.error() }; + }, +}; + +declare global { + var ptt: typeof measurement; +} +globalThis.ptt = measurement; diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json index d4b7f9d358b..6815a29fe3c 100644 --- a/test/audio-quality/clients/js/tsconfig.json +++ b/test/audio-quality/clients/js/tsconfig.json @@ -6,6 +6,8 @@ "include": [ "analyze.ts", "driver.ts", + "ptt.ts", + "ptt-analyze.ts", "native.ts", "grade.ts", "record.ts", diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts index fa1f4f32f4e..7e09d7ca278 100644 --- a/test/audio-quality/clients/js/vite.config.ts +++ b/test/audio-quality/clients/js/vite.config.ts @@ -17,6 +17,6 @@ export default defineConfig({ build: { target: "esnext", outDir: "dist", - rollupOptions: { input: ["index.html", "recorder.html", "mic.html"] }, + rollupOptions: { input: ["index.html", "recorder.html", "mic.html", "ptt.html"] }, }, }); diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh new file mode 100755 index 00000000000..f799045e4ba --- /dev/null +++ b/test/audio-quality/ptt.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Measure push-to-talk on an ideal local path: no shaping, loss, or queueing. +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then + exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@" +fi + +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +cd "$ROOT" +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source test/lib/harness.sh +harness_begin push-to-talk "just test push-to-talk$(harness_argv "$@")" +# The run directory is this measurement's evidence, so keep it even when it passes. +export HARNESS_KEEP="${HARNESS_KEEP:-1}" + +[[ $# -eq 0 ]] || { + echo 'usage: just test push-to-talk' >&2 + exit 2 +} + +bun install --frozen-lockfile +cargo build --locked -p moq-relay +TARGET_BASE="${CARGO_TARGET_DIR:-$ROOT/target}" +harness_port relay +relay_port=$HARNESS_PORT +sed "s/4443/$relay_port/g" test/audio-quality/relay.toml >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$TARGET_BASE/debug/moq-relay" "$HARNESS_RUN/relay.toml" +harness_ready "http://127.0.0.1:$relay_port/certificate.sha256" 30 "$HARNESS_PID" +# A different listener can answer HTTP before our process reports a bind collision. +if ! lsof -nP -a -p "$HARNESS_PID" -iTCP:"$relay_port" -sTCP:LISTEN >/dev/null 2>&1 || + ! lsof -nP -a -p "$HARNESS_PID" -iUDP:"$relay_port" >/dev/null 2>&1; then + cat "$HARNESS_RUN/relay.log" >&2 + echo 'relay does not own both listeners' >&2 + exit 1 +fi +harness_spawn driver "$HARNESS_RUN/driver.log" bun test/audio-quality/clients/js/ptt.ts \ + --relay "http://127.0.0.1:$relay_port" --out "$HARNESS_RUN" +driver_status=0 +harness_wait "$HARNESS_PID" || driver_status=$? +if [[ $driver_status -ne 0 ]]; then + cat "$HARNESS_RUN/driver.log" >&2 + exit "$driver_status" +fi +bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" | tee "$HARNESS_RUN/summary.log" diff --git a/test/justfile b/test/justfile index 7b702810714..467497e2ef5 100644 --- a/test/justfile +++ b/test/justfile @@ -162,6 +162,10 @@ native-play: audio-quality *args: ./audio-quality/run.sh "$@" +# Measure 22 push-to-talk presses at 300 ms on an ideal local path. +push-to-talk: + ./audio-quality/ptt.sh + # Records the arrival traces under audio-quality/traces/ that the replay rows grade: the # public relay's bbb.hang, a local relay's AAC tone, and a browser microphone through the # public relay. Needs the network. Narrow it with `--lanes relay-bbb --duration 35`; record a