From 710ae81e73d01ee4e8f1f7954f6a579e432dfc6e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:11:39 -0800 Subject: [PATCH 01/19] chore(quest): claim quest/m1/push-to-talk/README.md Quest-Claim: 8fd225ff192a8b55 From cb02a1d3b7e044be859e9efa1e21c55d1ece7cb3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:27:43 -0800 Subject: [PATCH 02/19] test(publish): measure push-to-talk across an impaired path Co-Authored-By: GPT-6.1 Sol --- .github/workflows/nightly.yml | 7 + test/audio-quality/README.md | 32 ++++ test/audio-quality/clients/js/ptt-analyze.ts | 77 ++++++++ test/audio-quality/clients/js/ptt.html | 5 + test/audio-quality/clients/js/ptt.ts | 97 ++++++++++ test/audio-quality/clients/js/src/ptt.ts | 176 +++++++++++++++++++ test/audio-quality/clients/js/tsconfig.json | 2 + test/audio-quality/clients/js/vite.config.ts | 2 +- test/audio-quality/ptt.sh | 45 +++++ test/justfile | 5 + 10 files changed, 447 insertions(+), 1 deletion(-) create mode 100644 test/audio-quality/clients/js/ptt-analyze.ts create mode 100644 test/audio-quality/clients/js/ptt.html create mode 100644 test/audio-quality/clients/js/ptt.ts create mode 100644 test/audio-quality/clients/js/src/ptt.ts create mode 100755 test/audio-quality/ptt.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e9f16cc96d1..e65320d1559 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -496,6 +496,13 @@ jobs: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs # A passing run is kept too: budgets.json is measured from these nights. HARNESS_KEEP: 1 + - name: Push-to-talk behind bufferbloat + if: ${{ !cancelled() }} + run: nix develop --command just test push-to-talk + shell: bash -leo pipefail {0} + env: + HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs + HARNESS_KEEP: 1 # The native player's rows, graded but not enforced: an intermittent device # stall voids about one AAC row in seven, which would fail most nights until # it is fixed. Kept beside the browser run so its ceilings can be re-measured. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 612503ffb66..ef036a2d03d 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -121,6 +121,38 @@ changed during the live run. The observer only records frames delivered under the player's maximum age, so a replay cannot recover groups the transport already discarded. +## Push-to-talk + +```bash +just test push-to-talk # acceptance: 64 kbit/s, finite 1 s queue, 50 ms each way +just test push-to-talk --reproduction # diagnostic: 100 ms RTT, 2% random loss +``` + +This separate, roughly nine-minute run drives the shipped publish and watch elements in one +Chromium page. A constant tone loops through the fake microphone, encoded as 128 kbit/s Opus with +60 ms frames. The listener holds a fixed 300 ms delay. After warmup and a mute, it measures 22 +presses, alternating two and three seconds: eight with no intervening wait, seven after five +seconds, and seven after sixty seconds. The publisher reaches the relay directly; the listener +uses WebTransport through the UDP shaper. No TCP service exists on the shaper port. + +`presses.json` records WebCodecs inputs and outputs, container writes, frame arrivals before +reordering, and audible output onsets. Publisher, receiver, and worklet observations share the +page's monotonic clock. A listening worklet detects each nonzero output onset and maps its render +time through `getOutputTimestamp`; the port-message arrival time is kept separately. This measures +the browser's projected device output, not acoustic output from a physical speaker. Zero-gap +presses can remain continuously audible; a missing onset is reported as null, never zero latency. + +`summary.json` matches inputs, outputs, writes, and arrivals by media timestamp, so a late arrival +is still attributed to its utterance. The command refuses missing encoder inputs, missing heads, +submitted frames that never encode or reach the container, reopened capture, repeated probes or +subscriptions, and separated utterances with no audible output. The shaper must report throttling +and queue overflow without random loss. Lost receiver frames are reported separately; aggregate +queue drops alone cannot attribute every missing frame to a particular packet. The reproduction +reports these checks without using random loss to gate acceptance. Both modes still fail on a +broken driver or invalid capture. + +Nightly runs the bufferbloated arm and keeps the raw evidence alongside the audio quality rows. + ## Traces The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, each graded as a diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts new file mode 100644 index 00000000000..3930f195dee --- /dev/null +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -0,0 +1,77 @@ +/** Reduce push-to-talk evidence without substituting missing observations with zero. @module */ +import { join } from "node:path"; +import type { measurement } from "./src/ptt"; + +type Result = ReturnType; +const [directory, diagnostic] = process.argv.slice(2); +if (!directory) throw new Error("usage: ptt-analyze.ts RUN_DIR"); +const result: Result = await Bun.file(join(directory, "presses.json")).json(); +if (result.presses.length !== 44 || result.presses.some((press, i) => press.on !== (i % 2 === 0))) { + throw new Error("expected 22 complete presses"); +} +if (result.captureError || result.errors.length) throw new Error("invalid capture"); +const encoded = new Map(result.encoded.map((frame) => [frame.timestamp, frame])); +const sent = new Map(result.sent.map((frame) => [frame.timestamp, frame])); +const arrivals = new Map(result.arrivals.map((frame) => [Math.round(frame[1] * 1000), frame])); +const presses = result.presses + .filter((press) => press.on) + .map((press, i) => { + const release = result.presses[i * 2 + 1]; + if (!release || release.on) throw new Error(`press ${i + 1} has no release`); + const input = result.submitted.filter((frame) => frame.at >= press.at && frame.at < release.at); + const output = input.flatMap((frame) => encoded.get(frame.timestamp) ?? []); + const published = output.flatMap((frame) => sent.get(frame.timestamp) ?? []); + const received = published.flatMap((frame) => { + const received = arrivals.get(frame.timestamp); + return received ? [received] : []; + }); + const first = received.toSorted((a, b) => a[0] - b[0])[0]; + const audible = result.audible.find( + (frame) => frame.at >= press.at && frame.at < (result.presses[(i + 1) * 2]?.at ?? Infinity), + ); + return { + press: i + 1, + gap: result.presses[i * 2 - 1] ? press.at - result.presses[i * 2 - 1].at : null, + duration: release.at - press.at, + submitted: input.length, + encoded: output.length, + sent: published.length, + received: received.length, + firstEncoded: input[0] ? encoded.has(input[0].timestamp) : false, + firstReceived: input[0] ? arrivals.has(input[0].timestamp) : false, + arrival: first ? first[0] - press.at : null, + audible: audible ? audible.at - press.at : null, + }; + }); +const activeDuration = presses.reduce((total, press) => total + press.duration, 0); +const offered = result.sent.reduce((total, frame) => total + (frame.bytes ?? 0) * 8, 0) / (activeDuration / 1000); +const summary = { + offered, + presses, + captures: result.captures, + probes: result.probes, + subscriptions: result.subscriptions, + unencoded: result.submitted.length - result.encoded.length, + unsent: result.encoded.filter((frame) => !sent.has(frame.timestamp)).length, +}; +await Bun.write(join(directory, "summary.json"), JSON.stringify(summary, null, 2)); +console.log(JSON.stringify(summary, null, 2)); + +const failures: string[] = []; +if (presses.some((press, i) => (i === 0 || i >= 8) && press.audible === null)) { + failures.push("a separated utterance never reached the output"); +} +if (result.captures.length) failures.push("capture reopened"); +if (result.probes.length) failures.push("encoder probed again"); +if (result.subscriptions.length) failures.push("subscription reopened"); +if (presses.some((press) => !press.submitted || !press.firstEncoded)) failures.push("utterance head was not encoded"); +if (summary.unencoded || summary.unsent) failures.push("submitted audio was not encoded and sent completely"); +const shaper = await Bun.file(join(directory, "shaper.json")).json(); +if (!diagnostic && offered <= 64_000) failures.push("the bottleneck was not below the media bitrate"); +if (!diagnostic && (!shaper.down.throttled || !shaper.down.overflowed || shaper.down.lost)) { + failures.push("bufferbloat did not act with queue overflow and no random loss"); +} +if (failures.length) { + console.error(failures.join("; ")); + if (!diagnostic) process.exitCode = 1; +} diff --git a/test/audio-quality/clients/js/ptt.html b/test/audio-quality/clients/js/ptt.html new file mode 100644 index 00000000000..3c438dcd36f --- /dev/null +++ b/test/audio-quality/clients/js/ptt.html @@ -0,0 +1,5 @@ + + +Push-to-talk measurement + + diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts new file mode 100644 index 00000000000..3a4f94ed383 --- /dev/null +++ b/test/audio-quality/clients/js/ptt.ts @@ -0,0 +1,97 @@ +/** Run 22 microphone presses through the real publisher, relay, and player. @module */ +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { createServer } from "vite"; +import { worklet } from "../../../../js/common/vite-plugin-worklet"; +import { launch } from "../../../interop/clients/js/harness"; +import type { measurement } from "./src/ptt"; + +const { values } = parseArgs({ + options: { + publisher: { type: "string" }, + viewer: { type: "string" }, + out: { type: "string" }, + }, +}); +const { publisher, viewer, out } = values; +if (!publisher || !viewer || !out) throw new Error("usage: ptt.ts --publisher URL --viewer URL --out DIR"); +const hash = await fetch(`${publisher}/certificate.sha256`).then(async (r) => { + if (!r.ok) throw new Error(`certificate: ${r.status}`); + return r.text(); +}); + +// Chromium loops this constant tone as its microphone. Its default fake device has silent +// intervals of its own, which would make an audible onset ambiguous. +const wav = Buffer.alloc(44 + 48_000 * 2); +wav.write("RIFF", 0); +wav.writeUInt32LE(wav.length - 8, 4); +wav.write("WAVEfmt ", 8); +wav.writeUInt32LE(16, 16); +wav.writeUInt16LE(1, 20); +wav.writeUInt16LE(1, 22); +wav.writeUInt32LE(48_000, 24); +wav.writeUInt32LE(96_000, 28); +wav.writeUInt16LE(2, 32); +wav.writeUInt16LE(16, 34); +wav.write("data", 36); +wav.writeUInt32LE(wav.length - 44, 40); +for (let i = 0; i < 48_000; i++) + wav.writeInt16LE(Math.round(8192 * Math.sin((2 * Math.PI * 440 * i) / 48_000)), 44 + i * 2); +const tone = join(out, "tone.wav"); +await Bun.write(tone, wav); + +const server = await createServer({ + configFile: false, + root: import.meta.dir, + plugins: [worklet()], + server: { host: "127.0.0.1", port: 0 }, +}); +await server.listen(); +const browser = await launch([ + "--autoplay-policy=no-user-gesture-required", + "--use-fake-device-for-media-stream", + "--use-fake-ui-for-media-stream", + `--use-file-for-fake-audio-capture=${tone}`, +]); +try { + const page = await browser.newPage(); + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(String(error))); + await page.route(`${viewer}/certificate.sha256`, (route) => + route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), + ); + await page.goto(`${server.resolvedUrls?.local[0]}ptt.html?${new URLSearchParams({ publisher, viewer })}`); + await page.waitForFunction(() => globalThis.ptt?.state().selected && globalThis.ptt.state().tapped, undefined, { + timeout: 30_000, + }); + const initial = await page.evaluate(() => globalThis.ptt.state()); + if (initial.publisher !== "webtransport" || initial.viewer !== "webtransport") throw new Error("not WebTransport"); + await page.waitForTimeout(3000); + await page.evaluate(() => globalThis.ptt.press(false)); + await page.waitForTimeout(5000); + const held = await page.evaluate(() => globalThis.ptt.state()); + if (held.active || held.config?.enabled !== false || held.selected !== initial.selected) { + throw new Error("the muted listener did not hold its rendition"); + } + const warmup = await page.evaluate(() => globalThis.ptt.result()); + await Bun.write(join(out, "warmup.json"), JSON.stringify({ initial, held, ...warmup })); + if (!warmup.audible.length) throw new Error("the tone never reached the output tap"); + await page.evaluate(() => globalThis.ptt.reset()); + for (let i = 0; i < 22; i++) { + await page.waitForTimeout(i < 8 ? 0 : i < 15 ? 5000 : 60_000); + await page.evaluate(() => globalThis.ptt.press(true)); + await page.waitForTimeout(i % 2 === 0 ? 2000 : 3000); + await page.evaluate(() => globalThis.ptt.press(false)); + console.log(`press ${i + 1}/22`); + await Bun.write(join(out, "presses.json"), JSON.stringify(await page.evaluate(() => globalThis.ptt.result()))); + } + await page.waitForTimeout(5000); + const result: ReturnType = await page.evaluate(() => globalThis.ptt.result()); + const state = await page.evaluate(() => globalThis.ptt.state()); + await Bun.write(join(out, "presses.json"), JSON.stringify({ ...result, state, errors })); + if (errors.length || result.errors.length || result.captureError || state.error) + throw new Error("measurement failed; see presses.json"); +} finally { + await browser.close(); + await server.close(); +} diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts new file mode 100644 index 00000000000..ba77015dde0 --- /dev/null +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -0,0 +1,176 @@ +/** Measure the shipped push-to-talk path on one monotonic page clock. @module */ +import "@moq/publish/element"; +import "@moq/watch/element"; +import { Milli } from "@moq/clock"; +import * as Container from "@moq/hang/container"; +import { Track } from "@moq/net"; +import type MoqPublish from "@moq/publish/element"; +import type MoqWatch from "@moq/watch/element"; +import { Capture } from "./capture"; +import type { Arrival } from "./schema"; + +/** One timestamped encoder input or output, on the page and media clocks. */ +type Frame = { at: number; timestamp: number; bytes?: number; duration?: number | null }; + +const records = { + submitted: [] as Frame[], + encoded: [] as Frame[], + sent: [] as Frame[], + subscriptions: [] as { at: number; name: string }[], + audible: [] as { at: number; received: number }[], + presses: [] as { at: number; on: boolean }[], + captures: [] as { at: number; audio: boolean }[], + probes: [] as number[], + errors: [] as string[], +}; + +// Observe WebCodecs without changing its input, output, or scheduling. The probe's flush is +// separate from the live encoder, so a probe repeated on unmute is visible too. +const NativeEncoder = globalThis.AudioEncoder; +globalThis.AudioEncoder = class extends NativeEncoder { + constructor(init: AudioEncoderInit) { + super({ + ...init, + output: (chunk, metadata) => { + records.encoded.push({ at: performance.now(), timestamp: chunk.timestamp, bytes: chunk.byteLength }); + init.output(chunk, metadata); + }, + }); + } + override encode(data: AudioData): void { + records.submitted.push({ at: performance.now(), timestamp: data.timestamp, duration: data.duration }); + super.encode(data); + } + override flush(): Promise { + records.probes.push(performance.now()); + return super.flush(); + } +}; +const encode = Container.Legacy.Producer.prototype.encode; +Container.Legacy.Producer.prototype.encode = function (frame, timestamp, keyframe) { + const result = encode.call(this, frame, timestamp, keyframe); + records.sent.push({ at: performance.now(), timestamp, bytes: frame.byteLength }); + return result; +}; +const subscribe = Track.Consumer.prototype.subscribe; +Track.Consumer.prototype.subscribe = function (options) { + records.subscriptions.push({ at: performance.now(), name: this.name }); + return subscribe.call(this, options); +}; +const getUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices); +navigator.mediaDevices.getUserMedia = (constraints) => { + records.captures.push({ at: performance.now(), audio: !!constraints?.audio }); + return getUserMedia(constraints); +}; + +const params = new URLSearchParams(location.search); +const required = (name: string) => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; +const publish = document.createElement("moq-publish") as MoqPublish; +publish.url = required("publisher"); +publish.name = "ptt.hang"; +publish.invisible = true; +publish.audio.codec.set({ mime: "opus", frameDuration: Milli(60), bitrate: 128_000 }); +publish.source = "camera"; +document.body.append(publish); + +const watch = document.createElement("moq-watch") as MoqWatch; +watch.url = required("viewer"); +watch.name = publish.name; +watch.setAttribute("delay", "300ms"); +watch.setAttribute("visible", "always"); +watch.volume = 1; +watch.muted = false; +watch.paused = false; +document.body.append(watch); + +const audio = watch.player.audio; +const capture = new Capture({ + broadcast: audio.source.in.broadcast, + track: audio.source.out.track, + config: audio.source.out.config, + delay: watch.player.sync.in.delay, + maxDelay: watch.player.sync.out.maxDelay, +}); +const arrivals: Arrival[] = []; +let attached: AudioNode | undefined; +let tapped: AudioNode | undefined; +async function attach() { + const root = audio.out.root.peek(); + if (!root || root === attached) return; + attached = root; + const context = root.context as AudioContext; + // A constant microphone tone makes a nonzero output quantum the utterance's audible onset. + // Map its render timestamp through getOutputTimestamp, rather than timing the port message. + const source = `class Tap extends AudioWorkletProcessor { + loud = false; + process(inputs) { + const loud = !!inputs[0]?.[0]?.some(s => Math.abs(s) > 0.001); + if (loud && !this.loud) this.port.postMessage(currentTime); + this.loud = loud; + return true; + } + } + registerProcessor('ptt-tap', Tap);`; + const url = URL.createObjectURL(new Blob([source], { type: "text/javascript" })); + try { + await context.audioWorklet.addModule(url); + } finally { + URL.revokeObjectURL(url); + } + const tap = new AudioWorkletNode(context, "ptt-tap"); + const mute = new GainNode(context, { gain: 0 }); + root.connect(tap); + tap.connect(mute); + mute.connect(context.destination); + tap.port.onmessage = ({ data }: MessageEvent) => { + const stamp = context.getOutputTimestamp(); + if (stamp.performanceTime === undefined || stamp.contextTime === undefined) { + records.errors.push("missing output clock"); + return; + } + records.audible.push({ + at: stamp.performanceTime + (data - stamp.contextTime) * 1000, + received: performance.now(), + }); + }; + tapped = root; +} +audio.out.root.subscribe(() => void attach().catch((error) => records.errors.push(String(error)))); + +/** The finite measurement surface driven by ptt.ts. */ +export const measurement = { + press(on: boolean) { + const at = performance.now(); + records.presses.push({ at, on }); + publish.muted = !on; + }, + state() { + return { + publisher: publish.connection.transport.peek() as string | undefined, + viewer: watch.connection.transport.peek() as string | undefined, + config: publish.audio.out.catalog.peek(), + selected: audio.source.out.track.peek(), + tapped: tapped !== undefined && tapped === audio.out.root.peek(), + active: publish.audio.out.active.peek(), + error: String(publish.audio.out.error.peek() ?? audio.out.error.peek() ?? ""), + }; + }, + reset() { + for (const list of Object.values(records)) list.length = 0; + capture.drain(); + arrivals.length = 0; + }, + result() { + arrivals.push(...capture.drain()); + return { ...records, arrivals, captureError: capture.error() }; + }, +}; + +declare global { + var ptt: typeof measurement; +} +globalThis.ptt = measurement; diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json index d4b7f9d358b..6815a29fe3c 100644 --- a/test/audio-quality/clients/js/tsconfig.json +++ b/test/audio-quality/clients/js/tsconfig.json @@ -6,6 +6,8 @@ "include": [ "analyze.ts", "driver.ts", + "ptt.ts", + "ptt-analyze.ts", "native.ts", "grade.ts", "record.ts", diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts index fa1f4f32f4e..7e09d7ca278 100644 --- a/test/audio-quality/clients/js/vite.config.ts +++ b/test/audio-quality/clients/js/vite.config.ts @@ -17,6 +17,6 @@ export default defineConfig({ build: { target: "esnext", outDir: "dist", - rollupOptions: { input: ["index.html", "recorder.html", "mic.html"] }, + rollupOptions: { input: ["index.html", "recorder.html", "mic.html", "ptt.html"] }, }, }); diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh new file mode 100755 index 00000000000..e763d64835c --- /dev/null +++ b/test/audio-quality/ptt.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# The bufferbloated push-to-talk acceptance run; --reproduction is diagnostic only. +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then + exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@" +fi + +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +cd "$ROOT" +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source test/lib/harness.sh +harness_begin push-to-talk "just test push-to-talk$(harness_argv "$@")" + +case "${1:-}" in + "") shape=(--delay 50ms --rate 64000 --queue 1s) ;; + --reproduction) shape=(--delay 50ms --loss 0.02) ;; + *) echo 'usage: just test push-to-talk [--reproduction]' >&2; exit 2 ;; +esac +[[ $# -le 1 ]] || { echo 'too many arguments' >&2; exit 2; } + +bun install --frozen-lockfile +cargo build --locked -p moq-relay -p moq-shaper +TARGET_BASE="${CARGO_TARGET_DIR:-$ROOT/target}" +harness_port relay +relay_port=$HARNESS_PORT +harness_port shaper +shaper_port=$HARNESS_PORT +sed "s/4443/$relay_port/g" test/audio-quality/relay.toml >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$TARGET_BASE/debug/moq-relay" "$HARNESS_RUN/relay.toml" +harness_ready "http://127.0.0.1:$relay_port/certificate.sha256" 30 "$HARNESS_PID" +harness_spawn shaper "$HARNESS_RUN/shaper.log" "$TARGET_BASE/debug/moq-shaper" \ + --listen "127.0.0.1:$shaper_port" --target "127.0.0.1:$relay_port" --seed 7 \ + --report "$HARNESS_RUN/shaper.json" "${shape[@]}" +shaper_pid=$HARNESS_PID +harness_spawn driver "$HARNESS_RUN/driver.log" bun test/audio-quality/clients/js/ptt.ts \ + --publisher "http://127.0.0.1:$relay_port" --viewer "http://127.0.0.1:$shaper_port" --out "$HARNESS_RUN" +status=0 +harness_wait "$HARNESS_PID" || status=$? +kill -TERM -- -"$shaper_pid" 2>/dev/null || true +harness_wait "$shaper_pid" || status=$? +analysis=() +[[ "${1:-}" != --reproduction ]] || analysis=(--diagnostic) +bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" "${analysis[@]}" >"$HARNESS_RUN/summary.log" || status=$? +exit "$status" diff --git a/test/justfile b/test/justfile index a03400dbef0..4f3a31e8b74 100644 --- a/test/justfile +++ b/test/justfile @@ -161,6 +161,11 @@ native-play: audio-quality *args: ./audio-quality/run.sh "$@" +# Measure 22 push-to-talk presses at 300 ms behind a finite bufferbloated bottleneck. +[positional-arguments] +push-to-talk *args: + ./audio-quality/ptt.sh "$@" + # Records the arrival traces under audio-quality/traces/ that the replay rows grade: the # public relay's bbb.hang, a local relay's AAC tone, and a browser microphone through the # public relay. Needs the network. Narrow it with `--lanes relay-bbb --duration 35`. From d58de97a3db2ea3f95eb73e7675f2840515dcd4f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:30:42 -0800 Subject: [PATCH 03/19] test(publish): record the measured browser version Co-Authored-By: GPT-6.1 Sol --- test/audio-quality/clients/js/ptt.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts index 3a4f94ed383..6e8a9bbd858 100644 --- a/test/audio-quality/clients/js/ptt.ts +++ b/test/audio-quality/clients/js/ptt.ts @@ -74,7 +74,7 @@ try { throw new Error("the muted listener did not hold its rendition"); } const warmup = await page.evaluate(() => globalThis.ptt.result()); - await Bun.write(join(out, "warmup.json"), JSON.stringify({ initial, held, ...warmup })); + await Bun.write(join(out, "warmup.json"), JSON.stringify({ browser: browser.version(), initial, held, ...warmup })); if (!warmup.audible.length) throw new Error("the tone never reached the output tap"); await page.evaluate(() => globalThis.ptt.reset()); for (let i = 0; i < 22; i++) { @@ -88,7 +88,7 @@ try { await page.waitForTimeout(5000); const result: ReturnType = await page.evaluate(() => globalThis.ptt.result()); const state = await page.evaluate(() => globalThis.ptt.state()); - await Bun.write(join(out, "presses.json"), JSON.stringify({ ...result, state, errors })); + await Bun.write(join(out, "presses.json"), JSON.stringify({ browser: browser.version(), ...result, state, errors })); if (errors.length || result.errors.length || result.captureError || state.error) throw new Error("measurement failed; see presses.json"); } finally { From 12163a28f7f64d8dd205c4aabd4abacf83aa103e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:32:34 -0800 Subject: [PATCH 04/19] style(test): format push-to-talk evidence output Co-Authored-By: GPT-6.1 Sol --- test/audio-quality/clients/js/ptt.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts index 6e8a9bbd858..393ab6dab06 100644 --- a/test/audio-quality/clients/js/ptt.ts +++ b/test/audio-quality/clients/js/ptt.ts @@ -88,7 +88,10 @@ try { await page.waitForTimeout(5000); const result: ReturnType = await page.evaluate(() => globalThis.ptt.result()); const state = await page.evaluate(() => globalThis.ptt.state()); - await Bun.write(join(out, "presses.json"), JSON.stringify({ browser: browser.version(), ...result, state, errors })); + await Bun.write( + join(out, "presses.json"), + JSON.stringify({ browser: browser.version(), ...result, state, errors }), + ); if (errors.length || result.errors.length || result.captureError || state.error) throw new Error("measurement failed; see presses.json"); } finally { From 144910e8b0181b12fb6f6089af5de668327933a3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:37:54 -0800 Subject: [PATCH 05/19] test(publish): format runner and record the open recovery decision Co-Authored-By: GPT-6.1 Sol --- quest/m1/push-to-talk/README.md | 12 ++++++++++++ test/audio-quality/ptt.sh | 14 ++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index f7bd47307cc..eb61a8106a9 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -56,3 +56,15 @@ encoder, and record why. ## Closes - [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost + +## Waiting + +- Maintainer: choose fatal encoder recovery with capture held open. Current audio + and video encoders keep a failure latched through disable/re-enable; the audio + test `audio failure stays local through demand changes and capture format + re-arms a fresh track` verifies it. Options: (1) re-arm once on each explicit + disabled-to-enabled edge (recommended, so the next press can recover while + preserving capture and publishing a fresh track name), or (2) retain the latch + and require a source or settings change. This is a product recovery policy; + tests cannot decide the intended user action. An automatic retry loop is not + proposed. diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index e763d64835c..38bd4e217d0 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -13,11 +13,17 @@ source test/lib/harness.sh harness_begin push-to-talk "just test push-to-talk$(harness_argv "$@")" case "${1:-}" in - "") shape=(--delay 50ms --rate 64000 --queue 1s) ;; - --reproduction) shape=(--delay 50ms --loss 0.02) ;; - *) echo 'usage: just test push-to-talk [--reproduction]' >&2; exit 2 ;; +"") shape=(--delay 50ms --rate 64000 --queue 1s) ;; +--reproduction) shape=(--delay 50ms --loss 0.02) ;; +*) + echo 'usage: just test push-to-talk [--reproduction]' >&2 + exit 2 + ;; esac -[[ $# -le 1 ]] || { echo 'too many arguments' >&2; exit 2; } +[[ $# -le 1 ]] || { + echo 'too many arguments' >&2 + exit 2 +} bun install --frozen-lockfile cargo build --locked -p moq-relay -p moq-shaper From f0b78ac5babb1f45d1c50bfe6abfa9e29463fa1e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:59:23 -0800 Subject: [PATCH 06/19] test(publish): preserve valid measurement evidence and record blockers Co-Authored-By: GPT-6.1 Sol --- quest/m1/push-to-talk/README.md | 24 ++++++++++++++-- test/audio-quality/README.md | 4 +-- test/audio-quality/clients/js/ptt-analyze.ts | 5 ++-- test/audio-quality/clients/js/ptt.ts | 6 ++++ test/audio-quality/clients/js/src/ptt.ts | 10 ++++++- test/audio-quality/ptt.sh | 30 ++++++++++++++------ 6 files changed, 63 insertions(+), 16 deletions(-) diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index eb61a8106a9..593867f6929 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -36,8 +36,8 @@ Decided 2026-10-28 (maintainer): replaces a publisher head buffer, which was considered and dropped as the same idea special-cased. -The epic's own work: once the children land, measure push-to-talk (22 -presses, 300 ms delay) behind bufferbloat: a rate limit below the media +The epic's own work: finish the acceptance proof with `just test push-to-talk` +(22 presses, 60 ms Opus, 300 ms delay) behind bufferbloat: a rate limit below the media bitrate, behind a deep but finite queue whose overflow drops packets. That gates acceptance. A held listener's unmute pays no catalog round trip and no resubscribe, so unmute → audible is the path's one-way delay plus the 300 ms. @@ -46,6 +46,14 @@ only from queue overflow. Rerun the reporter's setup (100 ms RTT, 2% random loss) only as a reproduction of the original numbers, not as acceptance. Record both in the PR. +The first full bufferbloat run also found one 60 ms frame submitted to +WebCodecs 0.3 ms before release that never produced output: disabling closes +the encoder synchronously and can discard queued codec work. Drain submitted +output to its original track before the discontinuity, keeping asynchronous +output owned by the original publication across source/broadcast changes. +Rerun the proof after fixing it. The measurement runner deliberately fails +on this loss. + Also check failure recovery (found 2026-10-28 by [moq-dev/moq#7157](https://github.com/moq-dev/moq/pull/7157)): unmuting used to reopen the microphone, which made a new source and so recovered a failed @@ -68,3 +76,15 @@ encoder, and record why. and require a source or settings change. This is a product recovery policy; tests cannot decide the intended user action. An automatic retry loop is not proposed. + +- Maintainer: choose live utterance startup semantics. The 2026-11-03 measurement + preserved subscriptions and delivered separated utterance heads in about + 100-140 ms, but projected device output under bufferbloat was much later + (median 950 ms across 17 observed onsets). `AudioRingBuffer.write` waits for + the target's media span before leaving a stall; filling 300 ms of media over + a below-bitrate path takes longer than 300 ms of wall time. Options: (1) honor + the existing goal with a clocked live start that retains the first frame and + uses bounded silence while waiting (recommended; buffered playback can keep + its data-driven prefill), or (2) retain media-driven live prefill and revise + the goal's latency expectation. This changes playback policy, so it needs a + maintainer decision rather than a looser measurement assertion. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index ef036a2d03d..c677f2b45af 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -124,7 +124,7 @@ transport already discarded. ## Push-to-talk ```bash -just test push-to-talk # acceptance: 64 kbit/s, finite 1 s queue, 50 ms each way +just test push-to-talk # bufferbloat: 64 kbit/s, finite 1 s queue, 50 ms each way just test push-to-talk --reproduction # diagnostic: 100 ms RTT, 2% random loss ``` @@ -143,7 +143,7 @@ the browser's projected device output, not acoustic output from a physical speak presses can remain continuously audible; a missing onset is reported as null, never zero latency. `summary.json` matches inputs, outputs, writes, and arrivals by media timestamp, so a late arrival -is still attributed to its utterance. The command refuses missing encoder inputs, missing heads, +is still attributed to its utterance. The command checks pipeline completeness, not the epic's latency verdict. It refuses missing encoder inputs, missing heads, submitted frames that never encode or reach the container, reopened capture, repeated probes or subscriptions, and separated utterances with no audible output. The shaper must report throttling and queue overflow without random loss. Lost receiver frames are reported separately; aggregate diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 3930f195dee..d24d0fc1216 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -4,7 +4,8 @@ import type { measurement } from "./src/ptt"; type Result = ReturnType; const [directory, diagnostic] = process.argv.slice(2); -if (!directory) throw new Error("usage: ptt-analyze.ts RUN_DIR"); +if (!directory || (diagnostic !== undefined && diagnostic !== "--diagnostic")) + throw new Error("usage: ptt-analyze.ts RUN_DIR [--diagnostic]"); const result: Result = await Bun.file(join(directory, "presses.json")).json(); if (result.presses.length !== 44 || result.presses.some((press, i) => press.on !== (i % 2 === 0))) { throw new Error("expected 22 complete presses"); @@ -51,7 +52,7 @@ const summary = { captures: result.captures, probes: result.probes, subscriptions: result.subscriptions, - unencoded: result.submitted.length - result.encoded.length, + unencoded: result.submitted.filter((frame) => !encoded.has(frame.timestamp)).length, unsent: result.encoded.filter((frame) => !sent.has(frame.timestamp)).length, }; await Bun.write(join(directory, "summary.json"), JSON.stringify(summary, null, 2)); diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts index 393ab6dab06..c40c285e9cb 100644 --- a/test/audio-quality/clients/js/ptt.ts +++ b/test/audio-quality/clients/js/ptt.ts @@ -73,6 +73,12 @@ try { if (held.active || held.config?.enabled !== false || held.selected !== initial.selected) { throw new Error("the muted listener did not hold its rendition"); } + // Tune-in can consume retained audio and fill the impaired queue. Validate the tap on + // a fresh press only after the muted listener holds its subscription and the queue drains. + await page.evaluate(() => globalThis.ptt.press(true)); + await page.waitForTimeout(3000); + await page.evaluate(() => globalThis.ptt.press(false)); + await page.waitForTimeout(5000); const warmup = await page.evaluate(() => globalThis.ptt.result()); await Bun.write(join(out, "warmup.json"), JSON.stringify({ browser: browser.version(), initial, held, ...warmup })); if (!warmup.audible.length) throw new Error("the tone never reached the output tap"); diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index ba77015dde0..9e38f498eee 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -5,6 +5,7 @@ import { Milli } from "@moq/clock"; import * as Container from "@moq/hang/container"; import { Track } from "@moq/net"; import type MoqPublish from "@moq/publish/element"; +import { Derived } from "@moq/signals"; import type MoqWatch from "@moq/watch/element"; import { Capture } from "./capture"; import type { Arrival } from "./schema"; @@ -93,7 +94,11 @@ const capture = new Capture({ track: audio.source.out.track, config: audio.source.out.config, delay: watch.player.sync.in.delay, - maxDelay: watch.player.sync.out.maxDelay, + // Decoder subscriptions include codec warmup beyond Sync's budget. The observer must + // request the same age so its own subscription cannot discard an otherwise visible frame. + maxDelay: new Derived([watch.player.sync.out.maxDelay, audio.source.out.config], (max, config) => + Milli(max + (config?.warmup ?? 0)), + ), }); const arrivals: Arrival[] = []; let attached: AudioNode | undefined; @@ -156,6 +161,9 @@ export const measurement = { selected: audio.source.out.track.peek(), tapped: tapped !== undefined && tapped === audio.out.root.peek(), active: publish.audio.out.active.peek(), + context: audio.out.context.peek()?.state, + stalled: audio.out.stalled.peek(), + timestamp: audio.out.timestamp.peek(), error: String(publish.audio.out.error.peek() ?? audio.out.error.peek() ?? ""), }; }, diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index 38bd4e217d0..cf896ffa98c 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The bufferbloated push-to-talk acceptance run; --reproduction is diagnostic only. +# Measure push-to-talk behind bufferbloat; --reproduction is diagnostic only. set -euo pipefail if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then @@ -13,12 +13,12 @@ source test/lib/harness.sh harness_begin push-to-talk "just test push-to-talk$(harness_argv "$@")" case "${1:-}" in -"") shape=(--delay 50ms --rate 64000 --queue 1s) ;; ---reproduction) shape=(--delay 50ms --loss 0.02) ;; -*) - echo 'usage: just test push-to-talk [--reproduction]' >&2 - exit 2 - ;; + "") shape=(--delay 50ms --rate 64000 --queue 1s) ;; + --reproduction) shape=(--delay 50ms --loss 0.02) ;; + *) + echo 'usage: just test push-to-talk [--reproduction]' >&2 + exit 2 + ;; esac [[ $# -le 1 ]] || { echo 'too many arguments' >&2 @@ -35,6 +35,13 @@ shaper_port=$HARNESS_PORT sed "s/4443/$relay_port/g" test/audio-quality/relay.toml >"$HARNESS_RUN/relay.toml" harness_spawn relay "$HARNESS_RUN/relay.log" "$TARGET_BASE/debug/moq-relay" "$HARNESS_RUN/relay.toml" harness_ready "http://127.0.0.1:$relay_port/certificate.sha256" 30 "$HARNESS_PID" +# A different listener can answer HTTP before our process reports a bind collision. +if ! lsof -nP -a -p "$HARNESS_PID" -iTCP:"$relay_port" -sTCP:LISTEN >/dev/null 2>&1 || + ! lsof -nP -a -p "$HARNESS_PID" -iUDP:"$relay_port" >/dev/null 2>&1; then + cat "$HARNESS_RUN/relay.log" >&2 + echo 'relay does not own both listeners' >&2 + exit 1 +fi harness_spawn shaper "$HARNESS_RUN/shaper.log" "$TARGET_BASE/debug/moq-shaper" \ --listen "127.0.0.1:$shaper_port" --target "127.0.0.1:$relay_port" --seed 7 \ --report "$HARNESS_RUN/shaper.json" "${shape[@]}" @@ -42,10 +49,15 @@ shaper_pid=$HARNESS_PID harness_spawn driver "$HARNESS_RUN/driver.log" bun test/audio-quality/clients/js/ptt.ts \ --publisher "http://127.0.0.1:$relay_port" --viewer "http://127.0.0.1:$shaper_port" --out "$HARNESS_RUN" status=0 -harness_wait "$HARNESS_PID" || status=$? +if ! harness_wait "$HARNESS_PID"; then + status=1 + cat "$HARNESS_RUN/driver.log" >&2 +fi kill -TERM -- -"$shaper_pid" 2>/dev/null || true harness_wait "$shaper_pid" || status=$? analysis=() [[ "${1:-}" != --reproduction ]] || analysis=(--diagnostic) -bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" "${analysis[@]}" >"$HARNESS_RUN/summary.log" || status=$? +if [[ $status -eq 0 ]]; then + bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" "${analysis[@]}" >"$HARNESS_RUN/summary.log" || status=$? +fi exit "$status" From abd94b206e9a6b2e39e0ff442a3b25e30c30e6c6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:16:48 -0800 Subject: [PATCH 07/19] test(publish): fix local review findings in the measurement harness Co-Authored-By: GPT-6.1 Sol --- quest/m1/push-to-talk/README.md | 7 +++--- test/audio-quality/README.md | 14 ++++++----- test/audio-quality/clients/js/ptt-analyze.ts | 11 ++++++--- test/audio-quality/clients/js/ptt.ts | 8 +++---- test/audio-quality/clients/js/src/ptt.ts | 4 ++++ test/audio-quality/ptt.sh | 25 ++++++++++++++++---- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index 593867f6929..163b394b6a3 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -78,10 +78,9 @@ encoder, and record why. proposed. - Maintainer: choose live utterance startup semantics. The 2026-11-03 measurement - preserved subscriptions and delivered separated utterance heads in about - 100-140 ms, but projected device output under bufferbloat was much later - (median 950 ms across 17 observed onsets). `AudioRingBuffer.write` waits for - the target's media span before leaving a stall; filling 300 ms of media over + preserved subscriptions and encoded every utterance head, but projected + device output under bufferbloat missed the latency goal on separated presses. + `AudioRingBuffer.write` waits for the target's media span before leaving a stall; filling 300 ms of media over a below-bitrate path takes longer than 300 ms of wall time. Options: (1) honor the existing goal with a clocked live start that retains the first frame and uses bounded silence while waiting (recommended; buffered playback can keep diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index c677f2b45af..1052296c440 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -131,25 +131,27 @@ just test push-to-talk --reproduction # diagnostic: 100 ms RTT, 2% random loss This separate, roughly nine-minute run drives the shipped publish and watch elements in one Chromium page. A constant tone loops through the fake microphone, encoded as 128 kbit/s Opus with 60 ms frames. The listener holds a fixed 300 ms delay. After warmup and a mute, it measures 22 -presses, alternating two and three seconds: eight with no intervening wait, seven after five -seconds, and seven after sixty seconds. The publisher reaches the relay directly; the listener -uses WebTransport through the UDP shaper. No TCP service exists on the shaper port. +presses, alternating two and three seconds: the first after the warmup's five-second mute, +seven with no intervening wait, seven after five seconds, and seven after sixty seconds. The +publisher reaches the relay directly; the listener uses WebTransport through the UDP shaper. No TCP service exists on the shaper port. `presses.json` records WebCodecs inputs and outputs, container writes, frame arrivals before reordering, and audible output onsets. Publisher, receiver, and worklet observations share the page's monotonic clock. A listening worklet detects each nonzero output onset and maps its render time through `getOutputTimestamp`; the port-message arrival time is kept separately. This measures the browser's projected device output, not acoustic output from a physical speaker. Zero-gap -presses can remain continuously audible; a missing onset is reported as null, never zero latency. +presses can remain continuously audible, and a later onset can belong to the preceding utterance. +Their audible latency is always null. Only separated presses have an identifiable onset. `summary.json` matches inputs, outputs, writes, and arrivals by media timestamp, so a late arrival -is still attributed to its utterance. The command checks pipeline completeness, not the epic's latency verdict. It refuses missing encoder inputs, missing heads, +is still attributed to its utterance. The command checks pipeline completeness, not the epic's +latency verdict. It refuses missing encoder inputs, missing heads, submitted frames that never encode or reach the container, reopened capture, repeated probes or subscriptions, and separated utterances with no audible output. The shaper must report throttling and queue overflow without random loss. Lost receiver frames are reported separately; aggregate queue drops alone cannot attribute every missing frame to a particular packet. The reproduction reports these checks without using random loss to gate acceptance. Both modes still fail on a -broken driver or invalid capture. +broken driver, invalid capture, or shaper startup/verification failure. Nightly runs the bufferbloated arm and keeps the raw evidence alongside the audio quality rows. diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index d24d0fc1216..464c5fb8495 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -27,9 +27,14 @@ const presses = result.presses return received ? [received] : []; }); const first = received.toSorted((a, b) => a[0] - b[0])[0]; - const audible = result.audible.find( - (frame) => frame.at >= press.at && frame.at < (result.presses[(i + 1) * 2]?.at ?? Infinity), - ); + // An onset during a zero-gap press may belong to its still-playing predecessor. + // A constant tone has no utterance ID, so only separated presses have identifiable onsets. + const separated = i === 0 || i >= 8; + const audible = separated + ? result.audible.find( + (frame) => frame.at >= press.at && frame.at < (result.presses[(i + 1) * 2]?.at ?? Infinity), + ) + : undefined; return { press: i + 1, gap: result.presses[i * 2 - 1] ? press.at - result.presses[i * 2 - 1].at : null, diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts index c40c285e9cb..9f698c39dfa 100644 --- a/test/audio-quality/clients/js/ptt.ts +++ b/test/audio-quality/clients/js/ptt.ts @@ -55,8 +55,8 @@ const browser = await launch([ ]); try { const page = await browser.newPage(); - const errors: string[] = []; - page.on("pageerror", (error) => errors.push(String(error))); + const pageErrors: string[] = []; + page.on("pageerror", (error) => pageErrors.push(String(error))); await page.route(`${viewer}/certificate.sha256`, (route) => route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), ); @@ -96,9 +96,9 @@ try { const state = await page.evaluate(() => globalThis.ptt.state()); await Bun.write( join(out, "presses.json"), - JSON.stringify({ browser: browser.version(), ...result, state, errors }), + JSON.stringify({ browser: browser.version(), ...result, state, pageErrors }), ); - if (errors.length || result.errors.length || result.captureError || state.error) + if (pageErrors.length || result.errors.length || result.captureError || state.error) throw new Error("measurement failed; see presses.json"); } finally { await browser.close(); diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index 9e38f498eee..d6e619b8f48 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -123,9 +123,13 @@ async function attach() { const url = URL.createObjectURL(new Blob([source], { type: "text/javascript" })); try { await context.audioWorklet.addModule(url); + } catch (error) { + if (audio.out.root.peek() !== root) return; + throw error; } finally { URL.revokeObjectURL(url); } + if (audio.out.root.peek() !== root) return; const tap = new AudioWorkletNode(context, "ptt-tap"); const mute = new GainNode(context, { gain: 0 }); root.connect(tap); diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index cf896ffa98c..035175f09b4 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -46,18 +46,33 @@ harness_spawn shaper "$HARNESS_RUN/shaper.log" "$TARGET_BASE/debug/moq-shaper" \ --listen "127.0.0.1:$shaper_port" --target "127.0.0.1:$relay_port" --seed 7 \ --report "$HARNESS_RUN/shaper.json" "${shape[@]}" shaper_pid=$HARNESS_PID +until grep -q '^shaper:' "$HARNESS_RUN/shaper.log" 2>/dev/null; do + if harness_exited "$shaper_pid"; then + cat "$HARNESS_RUN/shaper.log" >&2 + echo 'shaper exited before binding' >&2 + exit 1 + fi + sleep 0.05 +done harness_spawn driver "$HARNESS_RUN/driver.log" bun test/audio-quality/clients/js/ptt.ts \ --publisher "http://127.0.0.1:$relay_port" --viewer "http://127.0.0.1:$shaper_port" --out "$HARNESS_RUN" -status=0 -if ! harness_wait "$HARNESS_PID"; then - status=1 +driver_status=0 +harness_wait "$HARNESS_PID" || driver_status=$? +if [[ $driver_status -ne 0 ]]; then cat "$HARNESS_RUN/driver.log" >&2 fi kill -TERM -- -"$shaper_pid" 2>/dev/null || true -harness_wait "$shaper_pid" || status=$? +shaper_status=0 +harness_wait "$shaper_pid" || shaper_status=$? +printf '%s\n' "$shaper_status" >"$HARNESS_RUN/shaper.status" +if [[ $shaper_status -ne 0 ]]; then + cat "$HARNESS_RUN/shaper.log" >&2 +fi +status=$driver_status +[[ $shaper_status -eq 0 ]] || status=$shaper_status analysis=() [[ "${1:-}" != --reproduction ]] || analysis=(--diagnostic) -if [[ $status -eq 0 ]]; then +if [[ $driver_status -eq 0 ]]; then bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" "${analysis[@]}" >"$HARNESS_RUN/summary.log" || status=$? fi exit "$status" From 320b2b19ce12a243a0cb7468672ccdb6f3d4396b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 17:21:09 -0800 Subject: [PATCH 08/19] test(publish): grade push-to-talk playout on top of one-way delivery The maintainer restated the latency goal as at most 300 ms of playout added on top of one-way delivery, so path queueing is not charged to the player. The analyzer now reports per press the slowest delivery the onset waited on, the playout that remains, and projected device output. The quest records both 2026-11-03 decisions: no clocked live start, and no encoder re-arm on re-enable, since no fatal encoder error was observed. The tail drain now waits on the audio-across-stop quest. Co-Authored-By: Claude Opus 5.5 --- quest/m1/push-to-talk/README.md | 77 ++++++++++---------- test/audio-quality/README.md | 8 ++ test/audio-quality/clients/js/ptt-analyze.ts | 26 +++++++ 3 files changed, 73 insertions(+), 38 deletions(-) diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index 163b394b6a3..5887dc415dd 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -40,50 +40,51 @@ The epic's own work: finish the acceptance proof with `just test push-to-talk` (22 presses, 60 ms Opus, 300 ms delay) behind bufferbloat: a rate limit below the media bitrate, behind a deep but finite queue whose overflow drops packets. That gates acceptance. A held listener's unmute pays no catalog round trip and no -resubscribe, so unmute → audible is the path's one-way delay plus the 300 ms. -Every frame of the utterance is encoded and sent; any frames dropped come -only from queue overflow. Rerun the reporter's setup (100 ms RTT, 2% random -loss) only as a reproduction of the original numbers, not as acceptance. +resubscribe. Every frame of the utterance is encoded and sent; any frames +dropped come only from the path. Rerun the reporter's setup (100 ms RTT, 2% +random loss) only as a reproduction of the original numbers, not as acceptance. Record both in the PR. +Decided 2026-11-03 (maintainer, on +[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760)): + +- ✅ **Latency: at most 300 ms of playout on top of one-way delivery.** "We + can't ever expect 300ms latency if it takes more than 300ms to deliver a + packet." Queueing on the path (bufferbloat) is delivery, not playout, and no + clocked live start or other startup machinery is added. The runner reports + per press `delivery` (the slowest frame the onset waited on, from its first + sample's capture, so packetization, queueing and retransmission count) and + `playout` (the rest); projected device output is reported apart. Re-graded on + 2026-11-03 traces, playout was median 303 ms (152-545) on the reproduction + and 339 ms (307-895) behind bufferbloat. The player's target is its delay plus + measured jitter plus one frame (`js/watch/src/audio/latency.ts`), so the frame + is packetization, already counted in delivery. The bufferbloat tail is media + the 64 kbit/s link never delivered: about half of each utterance never arrived, + so the ring waited on later frames. +- ✅ **No encoder recovery on re-enable.** Asked first why a fatal encoder error + is expected at all: it is not. No push-to-talk run reported one (the runner + fails on `out.error`), and no issue reports a publisher encoder failure; the + latch came from a review finding + ([moq-dev/moq#5441](https://github.com/moq-dev/moq/pull/5441)). WebCodecs fails + an encoder only for a config the platform rejects (`NotSupportedError`, which + the same settings would hit again), an internal codec error (`EncodingError`), + or reclamation (`QuotaExceededError`), which Chromium applies only to + inactive, backgrounded hardware codecs; Opus is a software codec. A failure + stays latched until a device, settings or `source` change. + The first full bufferbloat run also found one 60 ms frame submitted to WebCodecs 0.3 ms before release that never produced output: disabling closes -the encoder synchronously and can discard queued codec work. Drain submitted -output to its original track before the discontinuity, keeping asynchronous -output owned by the original publication across source/broadcast changes. -Rerun the proof after fixing it. The measurement runner deliberately fails -on this loss. +the encoder synchronously and can discard queued codec work. +[Buffered audio across stop](/quest/m1/audio-across-stop.md) keeps the codec +across a disable and flushes it before the discontinuity, which drains that +tail to its original track. Once it lands, rerun the proof; the runner +deliberately fails on this loss. If the rerun is complete and within the +latency goal, this epic is done. + +## Required -Also check failure recovery (found 2026-10-28 by -[moq-dev/moq#7157](https://github.com/moq-dev/moq/pull/7157)): unmuting used to -reopen the microphone, which made a new source and so recovered a failed -encoder. Capture now stays open, so only a device swap, a settings change or a -new `source` recovers one. Decide whether re-enabling should retry a failed -encoder, and record why. +- [Buffered audio across stop](/quest/m1/audio-across-stop.md) - flushes the codec before a disable's discontinuity, so the last frame before a release is published ## Closes - [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost - -## Waiting - -- Maintainer: choose fatal encoder recovery with capture held open. Current audio - and video encoders keep a failure latched through disable/re-enable; the audio - test `audio failure stays local through demand changes and capture format - re-arms a fresh track` verifies it. Options: (1) re-arm once on each explicit - disabled-to-enabled edge (recommended, so the next press can recover while - preserving capture and publishing a fresh track name), or (2) retain the latch - and require a source or settings change. This is a product recovery policy; - tests cannot decide the intended user action. An automatic retry loop is not - proposed. - -- Maintainer: choose live utterance startup semantics. The 2026-11-03 measurement - preserved subscriptions and encoded every utterance head, but projected - device output under bufferbloat missed the latency goal on separated presses. - `AudioRingBuffer.write` waits for the target's media span before leaving a stall; filling 300 ms of media over - a below-bitrate path takes longer than 300 ms of wall time. Options: (1) honor - the existing goal with a clocked live start that retains the first frame and - uses bounded silence while waiting (recommended; buffered playback can keep - its data-driven prefill), or (2) retain media-driven live prefill and revise - the goal's latency expectation. This changes playback policy, so it needs a - maintainer decision rather than a looser measurement assertion. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 5d1f7b5508e..4693f3a7e1a 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -153,6 +153,14 @@ queue drops alone cannot attribute every missing frame to a particular packet. T reports these checks without using random loss to gate acceptance. Both modes still fail on a broken driver, invalid capture, or shaper startup/verification failure. +The latency goal is at most 300 ms of playout added on top of one-way delivery, so queueing on the +path is not charged to the player. For each separated press, `delivery` is the slowest of the +utterance's frames that arrived before its onset, measured from that frame's first sample (its media +offset after the press): it covers packetization, transport, queueing and retransmission. `playout` +is what remains of the onset's port-message arrival after the press and that delivery. `output` is +the projected device output latency, reported apart from the player. `latency` gives each one's min, +median and max. + Nightly runs the bufferbloated arm and keeps the raw evidence alongside the audio quality rows. ## Traces diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 464c5fb8495..8548ce106ba 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -35,6 +35,16 @@ const presses = result.presses (frame) => frame.at >= press.at && frame.at < (result.presses[(i + 1) * 2]?.at ?? Infinity), ) : undefined; + // One-way delivery is the slowest frame the start waited on, measured from its first sample's + // capture: it covers packetization, transport, queueing and retransmission. What remains of the + // start is the player's own added latency; projected device output is reported apart from it. + const head = input[0]?.timestamp; + const waited = input.flatMap((frame) => { + const arrival = arrivals.get(frame.timestamp); + if (!audible || head === undefined || !arrival || arrival[0] > audible.received) return []; + return [arrival[0] - press.at - (frame.timestamp - head) / 1000]; + }); + const delivery = waited.length ? Math.max(...waited) : null; return { press: i + 1, gap: result.presses[i * 2 - 1] ? press.at - result.presses[i * 2 - 1].at : null, @@ -47,12 +57,28 @@ const presses = result.presses firstReceived: input[0] ? arrivals.has(input[0].timestamp) : false, arrival: first ? first[0] - press.at : null, audible: audible ? audible.at - press.at : null, + delivery, + playout: audible && delivery !== null ? audible.received - press.at - delivery : null, + output: audible ? audible.at - audible.received : null, }; }); const activeDuration = presses.reduce((total, press) => total + press.duration, 0); const offered = result.sent.reduce((total, frame) => total + (frame.bytes ?? 0) * 8, 0) / (activeDuration / 1000); +const spread = (values: (number | null)[]) => { + const sorted = values.flatMap((value) => (value === null ? [] : [value])).toSorted((a, b) => a - b); + if (!sorted.length) return null; + const middle = sorted.length / 2; + const median = sorted.length % 2 ? sorted[Math.floor(middle)] : (sorted[middle - 1] + sorted[middle]) / 2; + return { min: sorted[0], median, max: sorted[sorted.length - 1] }; +}; const summary = { offered, + // The goal: at most 300 ms of playout added on top of one-way delivery. + latency: { + delivery: spread(presses.map((press) => press.delivery)), + playout: spread(presses.map((press) => press.playout)), + output: spread(presses.map((press) => press.output)), + }, presses, captures: result.captures, probes: result.probes, From 8a3f4a514305d1dfc072037a9d791f3f76cdb2aa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 17:47:22 -0800 Subject: [PATCH 09/19] quest(push-to-talk): grade the merged head's runs against the restated goal Co-Authored-By: Claude Opus 5.5 --- quest/m1/push-to-talk/README.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index 5887dc415dd..3be466d174e 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -54,13 +54,16 @@ Decided 2026-11-03 (maintainer, on clocked live start or other startup machinery is added. The runner reports per press `delivery` (the slowest frame the onset waited on, from its first sample's capture, so packetization, queueing and retransmission count) and - `playout` (the rest); projected device output is reported apart. Re-graded on - 2026-11-03 traces, playout was median 303 ms (152-545) on the reproduction - and 339 ms (307-895) behind bufferbloat. The player's target is its delay plus - measured jitter plus one frame (`js/watch/src/audio/latency.ts`), so the frame - is packetization, already counted in delivery. The bufferbloat tail is media - the 64 kbit/s link never delivered: about half of each utterance never arrived, - so the ring waited on later frames. + `playout` (the rest); projected device output (~73 ms) is reported apart. + The player's target is its delay plus measured jitter plus one frame + (`js/watch/src/audio/latency.ts`); the frame is packetization, already in + delivery. Graded on 2026-11-04 UTC runs of the merged head, over the 15 + separated presses: the reproduction's playout was median 296 ms (279-424), + with 9 presses at or under 300 ms, 4 more under 320 ms, and two at 384 and + 424 ms. Behind bufferbloat it was median 331 ms (301-595), with four presses + above 500 ms; delivery took 333-514 ms, and only 493 of 908 frames arrived, + since the 64 kbit/s link carries about half the stream. The tails are not + yet attributed; the player's jitter allowance is the likely term. - ✅ **No encoder recovery on re-enable.** Asked first why a fatal encoder error is expected at all: it is not. No push-to-talk run reported one (the runner fails on `out.error`), and no issue reports a publisher encoder failure; the @@ -78,8 +81,8 @@ the encoder synchronously and can discard queued codec work. [Buffered audio across stop](/quest/m1/audio-across-stop.md) keeps the codec across a disable and flushes it before the discontinuity, which drains that tail to its original track. Once it lands, rerun the proof; the runner -deliberately fails on this loss. If the rerun is complete and within the -latency goal, this epic is done. +deliberately fails on this loss. Grade the rerun against the latency goal +above; a complete rerun within it finishes this epic. ## Required From 0d9f90cb42beed9cc56fc4f2b905c26f10975180 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 17:57:01 -0800 Subject: [PATCH 10/19] test(publish): split push-to-talk playout at the render onset The port message that reports an onset reaches the main thread late by whatever the page was doing, which inflated playout and shrank device output by the same amount. Project the render onset from the output timestamp less the context's declared output path instead. Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/README.md | 7 ++++--- test/audio-quality/clients/js/ptt-analyze.ts | 6 +++--- test/audio-quality/clients/js/src/ptt.ts | 8 ++++++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 4693f3a7e1a..b309061e1a5 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -157,9 +157,10 @@ The latency goal is at most 300 ms of playout added on top of one-way delivery, path is not charged to the player. For each separated press, `delivery` is the slowest of the utterance's frames that arrived before its onset, measured from that frame's first sample (its media offset after the press): it covers packetization, transport, queueing and retransmission. `playout` -is what remains of the onset's port-message arrival after the press and that delivery. `output` is -the projected device output latency, reported apart from the player. `latency` gives each one's min, -median and max. +is what remains of the render onset after the press and that delivery. The render onset is the +projected output less the context's declared `baseLatency` and `outputLatency`, so a late port +message moves neither; that declared path is `output`, reported apart from the player. `latency` +gives each one's min, median and max. Nightly runs the bufferbloated arm and keeps the raw evidence alongside the audio quality rows. diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 8548ce106ba..92f4c0241ca 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -41,7 +41,7 @@ const presses = result.presses const head = input[0]?.timestamp; const waited = input.flatMap((frame) => { const arrival = arrivals.get(frame.timestamp); - if (!audible || head === undefined || !arrival || arrival[0] > audible.received) return []; + if (!audible || head === undefined || !arrival || arrival[0] > audible.rendered) return []; return [arrival[0] - press.at - (frame.timestamp - head) / 1000]; }); const delivery = waited.length ? Math.max(...waited) : null; @@ -58,8 +58,8 @@ const presses = result.presses arrival: first ? first[0] - press.at : null, audible: audible ? audible.at - press.at : null, delivery, - playout: audible && delivery !== null ? audible.received - press.at - delivery : null, - output: audible ? audible.at - audible.received : null, + playout: audible && delivery !== null ? audible.rendered - press.at - delivery : null, + output: audible ? audible.at - audible.rendered : null, }; }); const activeDuration = presses.reduce((total, press) => total + press.duration, 0); diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index d6e619b8f48..1edaea082fb 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -18,7 +18,7 @@ const records = { encoded: [] as Frame[], sent: [] as Frame[], subscriptions: [] as { at: number; name: string }[], - audible: [] as { at: number; received: number }[], + audible: [] as { at: number; rendered: number; received: number }[], presses: [] as { at: number; on: boolean }[], captures: [] as { at: number; audio: boolean }[], probes: [] as number[], @@ -141,8 +141,12 @@ async function attach() { records.errors.push("missing output clock"); return; } + // The render onset on the page clock is its projected output less the context's declared + // output path, so a late port message moves neither boundary. + const at = stamp.performanceTime + (data - stamp.contextTime) * 1000; records.audible.push({ - at: stamp.performanceTime + (data - stamp.contextTime) * 1000, + at, + rendered: at - (context.baseLatency + context.outputLatency) * 1000, received: performance.now(), }); }; From 814f2f875bdeb1d9f58090ef4cfbc954cd49d347 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 18:11:53 -0800 Subject: [PATCH 11/19] test(publish): read the push-to-talk render onset from the render clock The context's declared output latency overstated the observed path, so subtracting it moved the onset earlier than the render. Map the onset through currentTime read beside the page clock instead. Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/README.md | 8 ++++---- test/audio-quality/clients/js/src/ptt.ts | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index b309061e1a5..d616c03402d 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -157,10 +157,10 @@ The latency goal is at most 300 ms of playout added on top of one-way delivery, path is not charged to the player. For each separated press, `delivery` is the slowest of the utterance's frames that arrived before its onset, measured from that frame's first sample (its media offset after the press): it covers packetization, transport, queueing and retransmission. `playout` -is what remains of the render onset after the press and that delivery. The render onset is the -projected output less the context's declared `baseLatency` and `outputLatency`, so a late port -message moves neither; that declared path is `output`, reported apart from the player. `latency` -gives each one's min, median and max. +is what remains of the render onset after the press and that delivery. The render onset maps the +worklet's render time through the context's render clock, read with the page clock when the port +message lands, so a late message moves neither boundary. `output` is the projected device output +after that onset, reported apart from the player. `latency` gives each one's min, median and max. Nightly runs the bufferbloated arm and keeps the raw evidence alongside the audio quality rows. diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index 1edaea082fb..17fdba70726 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -141,13 +141,13 @@ async function attach() { records.errors.push("missing output clock"); return; } - // The render onset on the page clock is its projected output less the context's declared - // output path, so a late port message moves neither boundary. - const at = stamp.performanceTime + (data - stamp.contextTime) * 1000; + // Map the onset's render time through the render clock read at this instant, so a late port + // message moves neither the render onset nor its projected output. + const received = performance.now(); records.audible.push({ - at, - rendered: at - (context.baseLatency + context.outputLatency) * 1000, - received: performance.now(), + at: stamp.performanceTime + (data - stamp.contextTime) * 1000, + rendered: received - (context.currentTime - data) * 1000, + received, }); }; tapped = root; From 18250bbaf0491e83dd3e9d54c29be4f8952389b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 18:22:27 -0800 Subject: [PATCH 12/19] quest(push-to-talk): grade on the render onset and count the tail race Co-Authored-By: Claude Opus 5.5 --- quest/m1/push-to-talk/README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md index 3be466d174e..404a81d53eb 100644 --- a/quest/m1/push-to-talk/README.md +++ b/quest/m1/push-to-talk/README.md @@ -50,20 +50,19 @@ Decided 2026-11-03 (maintainer, on - ✅ **Latency: at most 300 ms of playout on top of one-way delivery.** "We can't ever expect 300ms latency if it takes more than 300ms to deliver a - packet." Queueing on the path (bufferbloat) is delivery, not playout, and no - clocked live start or other startup machinery is added. The runner reports - per press `delivery` (the slowest frame the onset waited on, from its first - sample's capture, so packetization, queueing and retransmission count) and - `playout` (the rest); projected device output (~73 ms) is reported apart. - The player's target is its delay plus measured jitter plus one frame + packet." Queueing on the path (bufferbloat) is delivery, not playout, so the + bufferbloat profile is not graded on latency, and no clocked live start or + other startup machinery is added. The runner reports per separated press + `delivery` (the slowest frame the onset waited on, from its first sample's + capture, so packetization and retransmission count), `playout` (the rest, up + to the render onset on the render clock), and projected device `output` + apart. The player's target is its delay plus measured jitter plus one frame (`js/watch/src/audio/latency.ts`); the frame is packetization, already in - delivery. Graded on 2026-11-04 UTC runs of the merged head, over the 15 - separated presses: the reproduction's playout was median 296 ms (279-424), - with 9 presses at or under 300 ms, 4 more under 320 ms, and two at 384 and - 424 ms. Behind bufferbloat it was median 331 ms (301-595), with four presses - above 500 ms; delivery took 333-514 ms, and only 493 of 908 frames arrived, - since the 64 kbit/s link carries about half the stream. The tails are not - yet attributed; the player's jitter allowance is the likely term. + delivery. On the reproduction at 2026-11-04 UTC, playout was 166/262/310 ms + (min/median/max), with 14 of 15 presses at or under 300 ms and one at 310 + ms. Behind bufferbloat, for information, playout was 333/494/708 ms beside + delivery of 310-546 ms, likely the jitter term following that spread (not + attributed). - ✅ **No encoder recovery on re-enable.** Asked first why a fatal encoder error is expected at all: it is not. No push-to-talk run reported one (the runner fails on `out.error`), and no issue reports a publisher encoder failure; the @@ -77,12 +76,13 @@ Decided 2026-11-03 (maintainer, on The first full bufferbloat run also found one 60 ms frame submitted to WebCodecs 0.3 ms before release that never produced output: disabling closes -the encoder synchronously and can discard queued codec work. +the encoder synchronously and can discard queued codec work. It recurred in 3 +of 7 runs on 2026-11-04, each a frame submitted under 1 ms before release. [Buffered audio across stop](/quest/m1/audio-across-stop.md) keeps the codec across a disable and flushes it before the discontinuity, which drains that tail to its original track. Once it lands, rerun the proof; the runner -deliberately fails on this loss. Grade the rerun against the latency goal -above; a complete rerun within it finishes this epic. +deliberately fails on this loss. A complete rerun whose reproduction holds the +latency goal above finishes this epic. ## Required From 663eb94e6fe9fe6483ed72928889506d44205ee6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 18:44:43 -0800 Subject: [PATCH 13/19] test(publish): measure push-to-talk on an ideal path and split its latency The maintainer asked to grade under ideal conditions only, so both elements now reach a local relay directly; the bufferbloat and random loss profiles and the shaper plumbing are gone. The page also taps decoder input and output and the publisher's capture chunks, and the analyzer splits each separated press into capture, packetization, encode, network, decode, jitter buffer, and device spans that sum to the press-to-output latency. A flush at release is the live encoder's drain since the disable now flushes before cutting, so a repeated probe is detected as a newly constructed encoder instead. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 2 +- test/audio-quality/README.md | 70 ++++++++++---------- test/audio-quality/clients/js/ptt-analyze.ts | 55 ++++++++++----- test/audio-quality/clients/js/ptt.ts | 20 ++---- test/audio-quality/clients/js/src/ptt.ts | 53 ++++++++++++--- test/audio-quality/ptt.sh | 49 ++------------ test/justfile | 7 +- 7 files changed, 134 insertions(+), 122 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e65320d1559..535e01abdba 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -496,7 +496,7 @@ jobs: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs # A passing run is kept too: budgets.json is measured from these nights. HARNESS_KEEP: 1 - - name: Push-to-talk behind bufferbloat + - name: Push-to-talk if: ${{ !cancelled() }} run: nix develop --command just test push-to-talk shell: bash -leo pipefail {0} diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index d616c03402d..3de23193648 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -124,45 +124,47 @@ transport already discarded. ## Push-to-talk ```bash -just test push-to-talk # bufferbloat: 64 kbit/s, finite 1 s queue, 50 ms each way -just test push-to-talk --reproduction # diagnostic: 100 ms RTT, 2% random loss +just test push-to-talk ``` -This separate, roughly nine-minute run drives the shipped publish and watch elements in one -Chromium page. A constant tone loops through the fake microphone, encoded as 128 kbit/s Opus with -60 ms frames. The listener holds a fixed 300 ms delay. After warmup and a mute, it measures 22 -presses, alternating two and three seconds: the first after the warmup's five-second mute, -seven with no intervening wait, seven after five seconds, and seven after sixty seconds. The -publisher reaches the relay directly; the listener uses WebTransport through the UDP shaper. No TCP service exists on the shaper port. - -`presses.json` records WebCodecs inputs and outputs, container writes, frame arrivals before -reordering, and audible output onsets. Publisher, receiver, and worklet observations share the -page's monotonic clock. A listening worklet detects each nonzero output onset and maps its render -time through `getOutputTimestamp`; the port-message arrival time is kept separately. This measures -the browser's projected device output, not acoustic output from a physical speaker. Zero-gap -presses can remain continuously audible, and a later onset can belong to the preceding utterance. -Their audible latency is always null. Only separated presses have an identifiable onset. +This separate, roughly nine-minute run drives the shipped publish and watch elements in one Chromium +page. A constant tone loops through the fake microphone, encoded as 128 kbit/s Opus with 60 ms +frames. The listener holds a fixed 300 ms delay. After warmup and a mute, it measures 22 presses, +alternating two and three seconds: the first after the warmup's five-second mute, seven with no +intervening wait, seven after five seconds, and seven after sixty seconds. Both elements reach a +local relay directly over WebTransport, with no shaping, loss, or queueing, so every millisecond +measured belongs to the media pipeline rather than the path. + +`presses.json` records captured chunks at each press, WebCodecs encoder and decoder inputs and +outputs, container writes, frame arrivals before reordering, and audible output onsets. Publisher, +receiver, and worklet observations share the page's monotonic clock. A listening worklet detects +each nonzero output onset. Its render time is mapped through the context's render clock, read with +the page clock when the port message lands, so a late message moves neither boundary; its projected +output comes from `getOutputTimestamp`. This measures the browser's projected device output, not +acoustic output from a physical speaker. Zero-gap presses can remain continuously audible, and a +later onset can belong to the preceding utterance. Their audible latency is always null. Only +separated presses have an identifiable onset. `summary.json` matches inputs, outputs, writes, and arrivals by media timestamp, so a late arrival is still attributed to its utterance. The command checks pipeline completeness, not the epic's -latency verdict. It refuses missing encoder inputs, missing heads, -submitted frames that never encode or reach the container, reopened capture, repeated probes or -subscriptions, and separated utterances with no audible output. The shaper must report throttling -and queue overflow without random loss. Lost receiver frames are reported separately; aggregate -queue drops alone cannot attribute every missing frame to a particular packet. The reproduction -reports these checks without using random loss to gate acceptance. Both modes still fail on a -broken driver, invalid capture, or shaper startup/verification failure. - -The latency goal is at most 300 ms of playout added on top of one-way delivery, so queueing on the -path is not charged to the player. For each separated press, `delivery` is the slowest of the -utterance's frames that arrived before its onset, measured from that frame's first sample (its media -offset after the press): it covers packetization, transport, queueing and retransmission. `playout` -is what remains of the render onset after the press and that delivery. The render onset maps the -worklet's render time through the context's render clock, read with the page clock when the port -message lands, so a late message moves neither boundary. `output` is the projected device output -after that onset, reported apart from the player. `latency` gives each one's min, median and max. - -Nightly runs the bufferbloated arm and keeps the raw evidence alongside the audio quality rows. +latency verdict. It refuses missing encoder inputs, missing heads, submitted frames that never +encode or reach the container, reopened capture, a new encoder (a probe or a rebuilt codec) or +subscription, and separated utterances with no audible output. It also fails on a broken driver or +invalid capture. + +The latency goal is at most 300 ms of playout added on top of one-way delivery. For each separated +press, `delivery` is the slowest of the utterance's frames that arrived before its onset, measured +from that frame's first sample (its media offset after the press). `playout` is what remains of the +render onset after the press and that delivery, and `output` is the projected device output after +that onset. `path` follows the head frame from the press to projected output in parts that sum to +it: `capture` (worklet delivery and framing, after the media the frame still needed), +`packetization` (that media, placed on the media clock by the newest chunk captured before the +press), `encode` (codec and container write), `network` (write to arrival, through the relay), +`decode` (arrival to decoded PCM), `jitter_buffer` (decoded PCM to the render onset: the playout +delay filling, with the ring's render quantum), and `device`, as the stages below name them. +`latency` and `path` give each one's min, median and max. + +Nightly runs it and keeps the raw evidence alongside the audio quality rows. ## Traces diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 92f4c0241ca..4bcce53ed82 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -2,10 +2,10 @@ import { join } from "node:path"; import type { measurement } from "./src/ptt"; -type Result = ReturnType; -const [directory, diagnostic] = process.argv.slice(2); -if (!directory || (diagnostic !== undefined && diagnostic !== "--diagnostic")) - throw new Error("usage: ptt-analyze.ts RUN_DIR [--diagnostic]"); +// The driver's final write adds the page's state to the result. +type Result = ReturnType & { state?: ReturnType }; +const [directory, ...rest] = process.argv.slice(2); +if (!directory || rest.length) throw new Error("usage: ptt-analyze.ts RUN_DIR"); const result: Result = await Bun.file(join(directory, "presses.json")).json(); if (result.presses.length !== 44 || result.presses.some((press, i) => press.on !== (i % 2 === 0))) { throw new Error("expected 22 complete presses"); @@ -14,6 +14,9 @@ if (result.captureError || result.errors.length) throw new Error("invalid captur const encoded = new Map(result.encoded.map((frame) => [frame.timestamp, frame])); const sent = new Map(result.sent.map((frame) => [frame.timestamp, frame])); const arrivals = new Map(result.arrivals.map((frame) => [Math.round(frame[1] * 1000), frame])); +// The first decode of each timestamp; a later one would be a re-decode, not the frame's first. +const decoded = new Map(); +for (const frame of result.decoded) if (!decoded.has(frame.timestamp)) decoded.set(frame.timestamp, frame); const presses = result.presses .filter((press) => press.on) .map((press, i) => { @@ -26,7 +29,7 @@ const presses = result.presses const received = arrivals.get(frame.timestamp); return received ? [received] : []; }); - const first = received.toSorted((a, b) => a[0] - b[0])[0]; + const earliest = received.toSorted((a, b) => a[0] - b[0])[0]; // An onset during a zero-gap press may belong to its still-playing predecessor. // A constant tone has no utterance ID, so only separated presses have identifiable onsets. const separated = i === 0 || i >= 8; @@ -45,6 +48,28 @@ const presses = result.presses return [arrival[0] - press.at - (frame.timestamp - head) / 1000]; }); const delivery = waited.length ? Math.max(...waited) : null; + // The head frame's path from the press to projected output, one boundary after another, so + // the parts sum to `audible`. Capture and packetization split at the head frame's end on the + // media clock, placed against the newest captured chunk delivered before the press. + const first = input[0]; + const path = (() => { + if (!audible || !first || !press.capture) return null; + const write = sent.get(first.timestamp); + const arrival = arrivals.get(first.timestamp); + const output = decoded.get(first.timestamp); + if (!write || !arrival || !output) return null; + const pressed = press.capture.end / 1000 + (press.at - press.capture.at); + const packetization = (first.timestamp + (first.duration ?? 0)) / 1000 - pressed; + return { + capture: first.at - press.at - packetization, + packetization, + encode: write.at - first.at, + network: arrival[0] - write.at, + decode: output.at - arrival[0], + jitter_buffer: audible.rendered - output.at, + device: audible.at - audible.rendered, + }; + })(); return { press: i + 1, gap: result.presses[i * 2 - 1] ? press.at - result.presses[i * 2 - 1].at : null, @@ -55,15 +80,14 @@ const presses = result.presses received: received.length, firstEncoded: input[0] ? encoded.has(input[0].timestamp) : false, firstReceived: input[0] ? arrivals.has(input[0].timestamp) : false, - arrival: first ? first[0] - press.at : null, + arrival: earliest ? earliest[0] - press.at : null, audible: audible ? audible.at - press.at : null, delivery, playout: audible && delivery !== null ? audible.rendered - press.at - delivery : null, output: audible ? audible.at - audible.rendered : null, + path, }; }); -const activeDuration = presses.reduce((total, press) => total + press.duration, 0); -const offered = result.sent.reduce((total, frame) => total + (frame.bytes ?? 0) * 8, 0) / (activeDuration / 1000); const spread = (values: (number | null)[]) => { const sorted = values.flatMap((value) => (value === null ? [] : [value])).toSorted((a, b) => a - b); if (!sorted.length) return null; @@ -71,17 +95,19 @@ const spread = (values: (number | null)[]) => { const median = sorted.length % 2 ? sorted[Math.floor(middle)] : (sorted[middle - 1] + sorted[middle]) / 2; return { min: sorted[0], median, max: sorted[sorted.length - 1] }; }; +const parts = ["capture", "packetization", "encode", "network", "decode", "jitter_buffer", "device"] as const; const summary = { - offered, // The goal: at most 300 ms of playout added on top of one-way delivery. latency: { delivery: spread(presses.map((press) => press.delivery)), playout: spread(presses.map((press) => press.playout)), output: spread(presses.map((press) => press.output)), }, + path: Object.fromEntries(parts.map((part) => [part, spread(presses.map((press) => press.path?.[part] ?? null))])), + delay: result.state?.delay, presses, captures: result.captures, - probes: result.probes, + encoders: result.encoders, subscriptions: result.subscriptions, unencoded: result.submitted.filter((frame) => !encoded.has(frame.timestamp)).length, unsent: result.encoded.filter((frame) => !sent.has(frame.timestamp)).length, @@ -94,16 +120,11 @@ if (presses.some((press, i) => (i === 0 || i >= 8) && press.audible === null)) { failures.push("a separated utterance never reached the output"); } if (result.captures.length) failures.push("capture reopened"); -if (result.probes.length) failures.push("encoder probed again"); +if (result.encoders.length) failures.push("an encoder was created again"); if (result.subscriptions.length) failures.push("subscription reopened"); if (presses.some((press) => !press.submitted || !press.firstEncoded)) failures.push("utterance head was not encoded"); if (summary.unencoded || summary.unsent) failures.push("submitted audio was not encoded and sent completely"); -const shaper = await Bun.file(join(directory, "shaper.json")).json(); -if (!diagnostic && offered <= 64_000) failures.push("the bottleneck was not below the media bitrate"); -if (!diagnostic && (!shaper.down.throttled || !shaper.down.overflowed || shaper.down.lost)) { - failures.push("bufferbloat did not act with queue overflow and no random loss"); -} if (failures.length) { console.error(failures.join("; ")); - if (!diagnostic) process.exitCode = 1; + process.exitCode = 1; } diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts index 9f698c39dfa..fef9b4fcd61 100644 --- a/test/audio-quality/clients/js/ptt.ts +++ b/test/audio-quality/clients/js/ptt.ts @@ -8,17 +8,12 @@ import type { measurement } from "./src/ptt"; const { values } = parseArgs({ options: { - publisher: { type: "string" }, - viewer: { type: "string" }, + relay: { type: "string" }, out: { type: "string" }, }, }); -const { publisher, viewer, out } = values; -if (!publisher || !viewer || !out) throw new Error("usage: ptt.ts --publisher URL --viewer URL --out DIR"); -const hash = await fetch(`${publisher}/certificate.sha256`).then(async (r) => { - if (!r.ok) throw new Error(`certificate: ${r.status}`); - return r.text(); -}); +const { relay, out } = values; +if (!relay || !out) throw new Error("usage: ptt.ts --relay URL --out DIR"); // Chromium loops this constant tone as its microphone. Its default fake device has silent // intervals of its own, which would make an audible onset ambiguous. @@ -57,10 +52,7 @@ try { const page = await browser.newPage(); const pageErrors: string[] = []; page.on("pageerror", (error) => pageErrors.push(String(error))); - await page.route(`${viewer}/certificate.sha256`, (route) => - route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), - ); - await page.goto(`${server.resolvedUrls?.local[0]}ptt.html?${new URLSearchParams({ publisher, viewer })}`); + await page.goto(`${server.resolvedUrls?.local[0]}ptt.html?${new URLSearchParams({ relay })}`); await page.waitForFunction(() => globalThis.ptt?.state().selected && globalThis.ptt.state().tapped, undefined, { timeout: 30_000, }); @@ -73,8 +65,8 @@ try { if (held.active || held.config?.enabled !== false || held.selected !== initial.selected) { throw new Error("the muted listener did not hold its rendition"); } - // Tune-in can consume retained audio and fill the impaired queue. Validate the tap on - // a fresh press only after the muted listener holds its subscription and the queue drains. + // Tune-in can consume retained audio. Validate the tap on a fresh press only after the muted + // listener holds its subscription. await page.evaluate(() => globalThis.ptt.press(true)); await page.waitForTimeout(3000); await page.evaluate(() => globalThis.ptt.press(false)); diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index 17fdba70726..c877e7314b0 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -17,19 +17,23 @@ const records = { submitted: [] as Frame[], encoded: [] as Frame[], sent: [] as Frame[], + decoding: [] as Frame[], + decoded: [] as Frame[], subscriptions: [] as { at: number; name: string }[], audible: [] as { at: number; rendered: number; received: number }[], - presses: [] as { at: number; on: boolean }[], + // The newest captured chunk delivered before a press, mapping the press onto the media clock. + presses: [] as { at: number; on: boolean; capture?: { at: number; end: number } }[], captures: [] as { at: number; audio: boolean }[], - probes: [] as number[], + encoders: [] as number[], errors: [] as string[], }; -// Observe WebCodecs without changing its input, output, or scheduling. The probe's flush is -// separate from the live encoder, so a probe repeated on unmute is visible too. +// Observe WebCodecs without changing its input, output, or scheduling. A probe or a rebuilt codec +// constructs a new encoder; the live one flushes at each release, so a flush alone is no probe. const NativeEncoder = globalThis.AudioEncoder; globalThis.AudioEncoder = class extends NativeEncoder { constructor(init: AudioEncoderInit) { + records.encoders.push(performance.now()); super({ ...init, output: (chunk, metadata) => { @@ -42,9 +46,37 @@ globalThis.AudioEncoder = class extends NativeEncoder { records.submitted.push({ at: performance.now(), timestamp: data.timestamp, duration: data.duration }); super.encode(data); } - override flush(): Promise { - records.probes.push(performance.now()); - return super.flush(); +}; +const NativeDecoder = globalThis.AudioDecoder; +globalThis.AudioDecoder = class extends NativeDecoder { + constructor(init: AudioDecoderInit) { + super({ + ...init, + output: (data) => { + records.decoded.push({ at: performance.now(), timestamp: data.timestamp, duration: data.duration }); + init.output(data); + }, + }); + } + override decode(chunk: EncodedAudioChunk): void { + records.decoding.push({ at: performance.now(), timestamp: chunk.timestamp }); + super.decode(chunk); + } +}; +// Note each captured chunk as the publisher's graph delivers it, without consuming it. +let captured: { at: number; end: number } | undefined; +const NativeNode = globalThis.AudioWorkletNode; +globalThis.AudioWorkletNode = class extends NativeNode { + constructor(context: BaseAudioContext, name: string, options?: AudioWorkletNodeOptions) { + super(context, name, options); + if (name !== "capture") return; + this.port.addEventListener( + "message", + ({ data }: MessageEvent<{ timestamp: number; channels: Float32Array[] }>) => { + const frames = data.channels[0]?.length ?? 0; + captured = { at: performance.now(), end: data.timestamp + (frames / context.sampleRate) * 1_000_000 }; + }, + ); } }; const encode = Container.Legacy.Producer.prototype.encode; @@ -71,7 +103,7 @@ const required = (name: string) => { return value; }; const publish = document.createElement("moq-publish") as MoqPublish; -publish.url = required("publisher"); +publish.url = required("relay"); publish.name = "ptt.hang"; publish.invisible = true; publish.audio.codec.set({ mime: "opus", frameDuration: Milli(60), bitrate: 128_000 }); @@ -79,7 +111,7 @@ publish.source = "camera"; document.body.append(publish); const watch = document.createElement("moq-watch") as MoqWatch; -watch.url = required("viewer"); +watch.url = publish.url; watch.name = publish.name; watch.setAttribute("delay", "300ms"); watch.setAttribute("visible", "always"); @@ -158,7 +190,7 @@ audio.out.root.subscribe(() => void attach().catch((error) => records.errors.pus export const measurement = { press(on: boolean) { const at = performance.now(); - records.presses.push({ at, on }); + records.presses.push(on && captured ? { at, on, capture: { ...captured } } : { at, on }); publish.muted = !on; }, state() { @@ -172,6 +204,7 @@ export const measurement = { context: audio.out.context.peek()?.state, stalled: audio.out.stalled.peek(), timestamp: audio.out.timestamp.peek(), + delay: watch.player.sync.out.delay.peek(), error: String(publish.audio.out.error.peek() ?? audio.out.error.peek() ?? ""), }; }, diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index 035175f09b4..171f73cfe1f 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Measure push-to-talk behind bufferbloat; --reproduction is diagnostic only. +# Measure push-to-talk on an ideal local path: no shaping, loss, or queueing. set -euo pipefail if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then @@ -12,26 +12,16 @@ cd "$ROOT" source test/lib/harness.sh harness_begin push-to-talk "just test push-to-talk$(harness_argv "$@")" -case "${1:-}" in - "") shape=(--delay 50ms --rate 64000 --queue 1s) ;; - --reproduction) shape=(--delay 50ms --loss 0.02) ;; - *) - echo 'usage: just test push-to-talk [--reproduction]' >&2 - exit 2 - ;; -esac -[[ $# -le 1 ]] || { - echo 'too many arguments' >&2 +[[ $# -eq 0 ]] || { + echo 'usage: just test push-to-talk' >&2 exit 2 } bun install --frozen-lockfile -cargo build --locked -p moq-relay -p moq-shaper +cargo build --locked -p moq-relay TARGET_BASE="${CARGO_TARGET_DIR:-$ROOT/target}" harness_port relay relay_port=$HARNESS_PORT -harness_port shaper -shaper_port=$HARNESS_PORT sed "s/4443/$relay_port/g" test/audio-quality/relay.toml >"$HARNESS_RUN/relay.toml" harness_spawn relay "$HARNESS_RUN/relay.log" "$TARGET_BASE/debug/moq-relay" "$HARNESS_RUN/relay.toml" harness_ready "http://127.0.0.1:$relay_port/certificate.sha256" 30 "$HARNESS_PID" @@ -42,37 +32,12 @@ if ! lsof -nP -a -p "$HARNESS_PID" -iTCP:"$relay_port" -sTCP:LISTEN >/dev/null 2 echo 'relay does not own both listeners' >&2 exit 1 fi -harness_spawn shaper "$HARNESS_RUN/shaper.log" "$TARGET_BASE/debug/moq-shaper" \ - --listen "127.0.0.1:$shaper_port" --target "127.0.0.1:$relay_port" --seed 7 \ - --report "$HARNESS_RUN/shaper.json" "${shape[@]}" -shaper_pid=$HARNESS_PID -until grep -q '^shaper:' "$HARNESS_RUN/shaper.log" 2>/dev/null; do - if harness_exited "$shaper_pid"; then - cat "$HARNESS_RUN/shaper.log" >&2 - echo 'shaper exited before binding' >&2 - exit 1 - fi - sleep 0.05 -done harness_spawn driver "$HARNESS_RUN/driver.log" bun test/audio-quality/clients/js/ptt.ts \ - --publisher "http://127.0.0.1:$relay_port" --viewer "http://127.0.0.1:$shaper_port" --out "$HARNESS_RUN" + --relay "http://127.0.0.1:$relay_port" --out "$HARNESS_RUN" driver_status=0 harness_wait "$HARNESS_PID" || driver_status=$? if [[ $driver_status -ne 0 ]]; then cat "$HARNESS_RUN/driver.log" >&2 + exit "$driver_status" fi -kill -TERM -- -"$shaper_pid" 2>/dev/null || true -shaper_status=0 -harness_wait "$shaper_pid" || shaper_status=$? -printf '%s\n' "$shaper_status" >"$HARNESS_RUN/shaper.status" -if [[ $shaper_status -ne 0 ]]; then - cat "$HARNESS_RUN/shaper.log" >&2 -fi -status=$driver_status -[[ $shaper_status -eq 0 ]] || status=$shaper_status -analysis=() -[[ "${1:-}" != --reproduction ]] || analysis=(--diagnostic) -if [[ $driver_status -eq 0 ]]; then - bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" "${analysis[@]}" >"$HARNESS_RUN/summary.log" || status=$? -fi -exit "$status" +bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" >"$HARNESS_RUN/summary.log" diff --git a/test/justfile b/test/justfile index 698b2036901..467497e2ef5 100644 --- a/test/justfile +++ b/test/justfile @@ -162,10 +162,9 @@ native-play: audio-quality *args: ./audio-quality/run.sh "$@" -# Measure 22 push-to-talk presses at 300 ms behind a finite bufferbloated bottleneck. -[positional-arguments] -push-to-talk *args: - ./audio-quality/ptt.sh "$@" +# Measure 22 push-to-talk presses at 300 ms on an ideal local path. +push-to-talk: + ./audio-quality/ptt.sh # Records the arrival traces under audio-quality/traces/ that the replay rows grade: the # public relay's bbb.hang, a local relay's AAC tone, and a browser microphone through the From 1cf73e5b9f159ff0e5435a3d5d00316eb3c3b127 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 18:58:38 -0800 Subject: [PATCH 14/19] test(publish): split the push-to-talk render hand-off from the jitter buffer Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/README.md | 7 ++++--- test/audio-quality/clients/js/ptt-analyze.ts | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 3de23193648..6fde0f6014c 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -160,9 +160,10 @@ that onset. `path` follows the head frame from the press to projected output in it: `capture` (worklet delivery and framing, after the media the frame still needed), `packetization` (that media, placed on the media clock by the newest chunk captured before the press), `encode` (codec and container write), `network` (write to arrival, through the relay), -`decode` (arrival to decoded PCM), `jitter_buffer` (decoded PCM to the render onset: the playout -delay filling, with the ring's render quantum), and `device`, as the stages below name them. -`latency` and `path` give each one's min, median and max. +`decode` (arrival to decoded PCM), `jitter_buffer` (the head's PCM to the newest PCM decoded before +the onset, which filled the playout delay), `render` (that PCM to the render onset: the worklet +hand-off and the context's render callback), and `device`, as the stages below name them. `latency` +and `path` give each one's min, median and max. Nightly runs it and keeps the raw evidence alongside the audio quality rows. diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 4bcce53ed82..ae1ce319afa 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -57,7 +57,15 @@ const presses = result.presses const write = sent.get(first.timestamp); const arrival = arrivals.get(first.timestamp); const output = decoded.get(first.timestamp); - if (!write || !arrival || !output) return null; + // The utterance's newest PCM decoded before the render onset: the one that filled the delay. + const filled = input + .flatMap((frame) => decoded.get(frame.timestamp) ?? []) + .filter((frame) => frame.at <= audible.rendered) + .reduce<(typeof result.decoded)[number] | undefined>( + (newest, frame) => (newest && newest.at >= frame.at ? newest : frame), + undefined, + ); + if (!write || !arrival || !output || !filled) return null; const pressed = press.capture.end / 1000 + (press.at - press.capture.at); const packetization = (first.timestamp + (first.duration ?? 0)) / 1000 - pressed; return { @@ -66,7 +74,8 @@ const presses = result.presses encode: write.at - first.at, network: arrival[0] - write.at, decode: output.at - arrival[0], - jitter_buffer: audible.rendered - output.at, + jitter_buffer: filled.at - output.at, + render: audible.rendered - filled.at, device: audible.at - audible.rendered, }; })(); @@ -95,7 +104,7 @@ const spread = (values: (number | null)[]) => { const median = sorted.length % 2 ? sorted[Math.floor(middle)] : (sorted[middle - 1] + sorted[middle]) / 2; return { min: sorted[0], median, max: sorted[sorted.length - 1] }; }; -const parts = ["capture", "packetization", "encode", "network", "decode", "jitter_buffer", "device"] as const; +const parts = ["capture", "packetization", "encode", "network", "decode", "jitter_buffer", "render", "device"] as const; const summary = { // The goal: at most 300 ms of playout added on top of one-way delivery. latency: { From 774548f50c1e49b4b385a2d6607af0eedd660138 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 19:36:22 -0800 Subject: [PATCH 15/19] quest: finish push-to-talk and plan its latency follow-up Five ideal-path runs on main with the disable flush were complete, held every rendition and capture, and kept playout within 300 ms of delivery on all 75 separated presses, so the epic is done and #7101 closes. The per-press breakdown moves to a small follow-up quest that records each part's numbers and knob and leaves the choice of knob to it. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 2 +- quest/m1/push-to-talk-latency.md | 70 ++++++++++++++ quest/m1/push-to-talk/README.md | 93 ------------------- ...bandwidth-grant-in-moq-audio-instead-of.md | 2 +- 4 files changed, 72 insertions(+), 95 deletions(-) create mode 100644 quest/m1/push-to-talk-latency.md delete mode 100644 quest/m1/push-to-talk/README.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 3fe2ed0bf93..74cff5c81a1 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -160,7 +160,7 @@ per-frame CPU growth since #5926. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor -- [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - consumers hold a disabled rendition, and an unmute encodes at once with the mic still open +- [Push-to-talk's press-to-audible latency is chosen, not inherited](/quest/m1/push-to-talk-latency.md) - each part of an ideal-path press's 466 ms, chiefly the 300 ms delay and 115 ms device output, is reduced through its knob or kept with a reason - [moq-mux presents a broadcast on the publisher's timestamps](/quest/m1/presentation-timeline.md) - `moq play`'s audio-anchored playout timing as a moq-mux API the studio's broadcast sources share - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS diff --git a/quest/m1/push-to-talk-latency.md b/quest/m1/push-to-talk-latency.md new file mode 100644 index 00000000000..b39683a463d --- /dev/null +++ b/quest/m1/push-to-talk-latency.md @@ -0,0 +1,70 @@ +# [S] Push-to-talk's press-to-audible latency is chosen, not inherited + +## Goal + +Each part of a push-to-talk press's latency on an ideal path, from capture to +device output, is either reduced through the knob that controls it or kept +with a recorded reason. `just test push-to-talk` reports the result with the +same breakdown. + +## Plan + +Measured 2026-11-04 by +[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760), which finished +push-to-talk ([#7101](https://github.com/moq-dev/moq/issues/7101)). The setup +was `just test push-to-talk` on a direct local relay with no shaping or loss, +60 ms Opus at 128 kbit/s, and a fixed 300 ms `delay`. Chromium 156 ran +headless on Linux. There were five runs of 75 separated presses. `summary.json` +`path` splits each press at page-clock boundaries that sum exactly to press → +projected output. + +| Part | Boundaries | Median ms | Max ms | +| --- | --- | --- | --- | +| capture | worklet delivery and framing | 28 | 72 | +| packetization | media the head frame still needed after the press | 40 | 60 | +| encode | encoder input to container write | 0.9 | 23 | +| network | container write to arrival, through the relay | 0.2 | 4 | +| decode | arrival to decoded PCM | 0.7 | 20 | +| jitter buffer | head's PCM to the PCM that filled the delay | 256 | 297 | +| render | that PCM to the render onset | 24 | 49 | +| device | render onset to projected output | 115 | 122 | +| total | press to projected output | 466 | 505 | + +The capture and packetization split is approximate, but their 63 ms median +sum is exact. Playout on top of one-way delivery was 266 ms median and 289 ms +max, inside the 300 ms the maintainer set. + +Biggest contributors and their knobs: + +- **Playout delay, about 300 ms.** The ring starts once it holds the delay in + decoded PCM, counted from the head's first sample. That is why the jitter + buffer plus packetization sum to about 296 ms. Knob: `` `delay`. + This test fixes it at 300 ms. In `"auto"`, the default, the target is + `max(measured, advertised) + frame + catalog delay` + (`js/watch/src/audio/latency.ts`). +- **Device output, about 115 ms.** This matches Chromium's declared + `baseLatency + outputLatency` on this headless Linux host. Knob: the watch + AudioContext's `latencyHint` (`"interactive"` in + `js/watch/src/audio/decoder.ts`), otherwise the platform. A real device, + such as `kixtop`'s macOS output, should be measured before acting on it. +- **Capture and render, about 52 ms together.** Both follow the AudioContext + callback cadence on this host. The capture graph also delivers 60 ms frames + in bursts, 25-109 ms apart. Knob: the capture AudioContext's `latencyHint` + (`js/publish/src/audio/capture.ts`), otherwise the platform. +- **Opus frame, 60 ms.** This test chooses it; the publisher defaults to + 20 ms. Under a fixed delay it is absorbed by the delay. In `"auto"` it adds + one frame to the target. Knob: the encoder's `frameDuration`. + +No part looked like a bug. On an ideal path network, encode and decode are +about 1 ms each, and the head is not trimmed as decoder warmup. The fixed +delay bypasses the jitter estimator. In `"auto"`, the 44 ms median per-press +arrival-lateness spread, from capture bursts rather than the network, would +make the measured term nonzero; that is the estimator working as specified +in [Audio jitter target](/quest/m1/audio-jitter-target/README.md). + +Open: decide which knob to turn first, and whether to measure on a real +device before deciding. + +## Related + +- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the `"auto"` target this latency feeds diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md deleted file mode 100644 index 404a81d53eb..00000000000 --- a/quest/m1/push-to-talk/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# [M] Push-to-talk loses no speech - -## Goal - -A listener subscribed while a publisher is muted hears the next utterance from -its first frame. Unmuting costs no catalog round trip, no resubscribe and no -microphone reopen. A muted rendition still costs no media bytes. - -## Plan - -From [#7101](https://github.com/moq-dev/moq/issues/7101), traced 2026-10-28. With -`Audio.Encoder` `enabled: false`, watch deselects the rendition and -unsubscribes. On unmute the listener waits for the catalog flip, then its -SUBSCRIBE reaches the publisher. The demand gate discards everything encoded -before that (`js/publish/src/audio/encoder.ts:686`). At 100 ms RTT that is -three one-way trips plus the probe and a frame (~515 ms to the first fresh -frame). The emptied buffer then refills the 300 ms delay, so the result is -~820-950 ms to audible, with 250-400 ms of speech never encoded. - -Decided 2026-10-28 (maintainer): - -- **Hold, not DTX.** DTX is for silence the app can't predict. Push-to-talk - knows when it is silent, and a held subscription costs no media bytes and no - encoder CPU. DTX is off on main anyway ([#4919](https://github.com/moq-dev/moq/pull/4919)). -- **Players hold a disabled rendition**, audio and video, by default, in - every first-party player. Exports keep dropping it, since a held track - sends nothing and an interleaving muxer or HLS player may wait on it. The - hang spec weakens from "must not select" to "should not select" a disabled - rendition. -- **The publisher keeps its probe result across a disable**, and the - element's `muted` and `invisible` leave capture open. -- **Encoding no longer waits for demand**, in every first-party publisher - (finished by [moq-dev/moq#7554](https://github.com/moq-dev/moq/pull/7554)). - A listener who - subscribes after the unmute still gets the start within its `delay.max`. This - replaces a publisher head buffer, which was considered and dropped as the - same idea special-cased. - -The epic's own work: finish the acceptance proof with `just test push-to-talk` -(22 presses, 60 ms Opus, 300 ms delay) behind bufferbloat: a rate limit below the media -bitrate, behind a deep but finite queue whose overflow drops packets. That -gates acceptance. A held listener's unmute pays no catalog round trip and no -resubscribe. Every frame of the utterance is encoded and sent; any frames -dropped come only from the path. Rerun the reporter's setup (100 ms RTT, 2% -random loss) only as a reproduction of the original numbers, not as acceptance. -Record both in the PR. - -Decided 2026-11-03 (maintainer, on -[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760)): - -- ✅ **Latency: at most 300 ms of playout on top of one-way delivery.** "We - can't ever expect 300ms latency if it takes more than 300ms to deliver a - packet." Queueing on the path (bufferbloat) is delivery, not playout, so the - bufferbloat profile is not graded on latency, and no clocked live start or - other startup machinery is added. The runner reports per separated press - `delivery` (the slowest frame the onset waited on, from its first sample's - capture, so packetization and retransmission count), `playout` (the rest, up - to the render onset on the render clock), and projected device `output` - apart. The player's target is its delay plus measured jitter plus one frame - (`js/watch/src/audio/latency.ts`); the frame is packetization, already in - delivery. On the reproduction at 2026-11-04 UTC, playout was 166/262/310 ms - (min/median/max), with 14 of 15 presses at or under 300 ms and one at 310 - ms. Behind bufferbloat, for information, playout was 333/494/708 ms beside - delivery of 310-546 ms, likely the jitter term following that spread (not - attributed). -- ✅ **No encoder recovery on re-enable.** Asked first why a fatal encoder error - is expected at all: it is not. No push-to-talk run reported one (the runner - fails on `out.error`), and no issue reports a publisher encoder failure; the - latch came from a review finding - ([moq-dev/moq#5441](https://github.com/moq-dev/moq/pull/5441)). WebCodecs fails - an encoder only for a config the platform rejects (`NotSupportedError`, which - the same settings would hit again), an internal codec error (`EncodingError`), - or reclamation (`QuotaExceededError`), which Chromium applies only to - inactive, backgrounded hardware codecs; Opus is a software codec. A failure - stays latched until a device, settings or `source` change. - -The first full bufferbloat run also found one 60 ms frame submitted to -WebCodecs 0.3 ms before release that never produced output: disabling closes -the encoder synchronously and can discard queued codec work. It recurred in 3 -of 7 runs on 2026-11-04, each a frame submitted under 1 ms before release. -[Buffered audio across stop](/quest/m1/audio-across-stop.md) keeps the codec -across a disable and flushes it before the discontinuity, which drains that -tail to its original track. Once it lands, rerun the proof; the runner -deliberately fails on this loss. A complete rerun whose reproduction holds the -latency goal above finishes this epic. - -## Required - -- [Buffered audio across stop](/quest/m1/audio-across-stop.md) - flushes the codec before a disable's discontinuity, so the last frame before a release is published - -## Closes - -- [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost diff --git a/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md b/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md index 1bb1d2d1eb3..9f11e236e9c 100644 --- a/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md +++ b/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md @@ -45,7 +45,7 @@ grant; `Options::bandwidth` documents that 2026-10-04 with the enabled flag), through the disable and re-enable rule [simulcast rung disable](/quest/m1/simulcast-rung-disable.md) puts beside `Control` rather than a second one; the encoder never errors. Viewers with - no enabled alternative hold it ([push-to-talk](/quest/m1/push-to-talk/README.md)), + no enabled alternative hold it (the push-to-talk hold, [#7101](https://github.com/moq-dev/moq/issues/7101)), so its demand can stay while its grant is below the floor; recovery evaluates a hypothetical share against the current estimate, as the ladder controller does. The reservation's ceiling is From d2d74e0103a15dbafefa52bf060dab05d080751f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 19:41:41 -0800 Subject: [PATCH 16/19] test(publish): keep and print push-to-talk evidence, and test the analyzer The shared harness deletes a passing run's directory, and the analyzer's report went only into it, so a local run printed nothing. The command now keeps its run by default and prints the latency, its parts, and the completeness counts. A synthetic ideal trace checks that each press splits exactly into its parts, that the live encoder's release flush passes, and that a newly constructed encoder fails as a re-probe. Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/README.md | 4 +- .../clients/js/ptt-analyze.test.ts | 95 +++++++++++++++++++ test/audio-quality/clients/js/ptt-analyze.ts | 5 +- test/audio-quality/ptt.sh | 4 +- 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 test/audio-quality/clients/js/ptt-analyze.test.ts diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 6fde0f6014c..e9bcc019957 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -165,7 +165,9 @@ the onset, which filled the playout delay), `render` (that PCM to the render ons hand-off and the context's render callback), and `device`, as the stages below name them. `latency` and `path` give each one's min, median and max. -Nightly runs it and keeps the raw evidence alongside the audio quality rows. +The command prints `latency`, `path`, and the completeness counts, and keeps its run directory +even when it passes (set `HARNESS_KEEP=0` to drop it). Nightly runs it and keeps the raw evidence +alongside the audio quality rows. ## Traces diff --git a/test/audio-quality/clients/js/ptt-analyze.test.ts b/test/audio-quality/clients/js/ptt-analyze.test.ts new file mode 100644 index 00000000000..80813b366ba --- /dev/null +++ b/test/audio-quality/clients/js/ptt-analyze.test.ts @@ -0,0 +1,95 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const FRAME = 60; + +// 22 complete presses on an ideal path: the media clock is the page clock, every 60 ms frame is +// encoded, written, received and decoded a millisecond apart, and each separated press renders +// once the 300 ms delay is decoded, with 100 ms of projected device output after it. +function trace() { + const result = { + submitted: [] as { at: number; timestamp: number; duration: number }[], + encoded: [] as { at: number; timestamp: number; bytes: number }[], + sent: [] as { at: number; timestamp: number; bytes: number }[], + decoding: [] as { at: number; timestamp: number }[], + decoded: [] as { at: number; timestamp: number; duration: number }[], + subscriptions: [] as unknown[], + audible: [] as { at: number; rendered: number; received: number }[], + presses: [] as { at: number; on: boolean; capture?: { at: number; end: number } }[], + captures: [] as unknown[], + encoders: [] as number[], + errors: [] as string[], + arrivals: [] as [number, number, number, number][], + state: { delay: 300 }, + }; + for (let i = 0; i < 22; i++) { + const press = 10_000 + i * 10_000; + const release = press + 2000; + // The press lands 20 ms into a frame, so the head frame still needs 40 ms of media. + result.presses.push({ at: press, on: true, capture: { at: press, end: press * 1000 } }); + const start = press - 20; + for (let media = start; media + FRAME <= release; media += FRAME) { + const at = media + FRAME + 25; + if (at >= release) break; + const timestamp = media * 1000; + result.submitted.push({ at, timestamp, duration: FRAME * 1000 }); + result.encoded.push({ at: at + 1, timestamp, bytes: 959 }); + result.sent.push({ at: at + 1, timestamp, bytes: 959 }); + result.arrivals.push([at + 2, media, i, FRAME]); + result.decoding.push({ at: at + 2, timestamp }); + result.decoded.push({ at: at + 3, timestamp, duration: FRAME * 1000 }); + } + if (i === 0 || i >= 8) { + // The fifth frame completes the delay; the render onset follows its PCM by 10 ms. + const rendered = start + 5 * FRAME + 25 + 3 + 10; + result.audible.push({ at: rendered + 100, rendered, received: rendered + 5 }); + } + result.presses.push({ at: release, on: false }); + } + return result; +} + +async function analyze(result: ReturnType) { + const directory = mkdtempSync(join(tmpdir(), "ptt-analyze-")); + try { + await Bun.write(join(directory, "presses.json"), JSON.stringify(result)); + const run = Bun.spawnSync(["bun", fileURLToPath(new URL("./ptt-analyze.ts", import.meta.url)), directory]); + const summary = await Bun.file(join(directory, "summary.json")).json(); + return { exitCode: run.exitCode, stdout: run.stdout.toString(), stderr: run.stderr.toString(), summary }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +test("a complete run passes, reports its latency, and splits each press exactly", async () => { + const { exitCode, stdout, stderr, summary } = await analyze(trace()); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout).latency.playout).not.toBeNull(); + + const press = summary.presses[0]; + expect(press.path).toEqual({ + capture: 25, + packetization: 40, + encode: 1, + network: 1, + decode: 1, + jitter_buffer: 4 * FRAME, + render: 10, + device: 100, + }); + const parts = Object.values(press.path as Record).reduce((sum, part) => sum + part, 0); + expect(parts).toBe(press.audible); +}); + +test("a new encoder fails as a probe, but the live encoder's release flush does not", async () => { + // A flush leaves no trace in the page's records, so the passing run above is the release flush. + const result = trace(); + result.encoders.push(result.presses[2].at + 1); + const { exitCode, stderr } = await analyze(result); + expect(exitCode).toBe(1); + expect(stderr).toContain("an encoder was created again"); +}); diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index ae1ce319afa..2577161b750 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -122,7 +122,10 @@ const summary = { unsent: result.encoded.filter((frame) => !sent.has(frame.timestamp)).length, }; await Bun.write(join(directory, "summary.json"), JSON.stringify(summary, null, 2)); -console.log(JSON.stringify(summary, null, 2)); +// The report: the latency and its parts, and the completeness the run gates on. Per-press detail +// stays in summary.json. +const { latency, path, delay, unencoded, unsent } = summary; +console.log(JSON.stringify({ latency, path, delay, unencoded, unsent }, null, 2)); const failures: string[] = []; if (presses.some((press, i) => (i === 0 || i >= 8) && press.audible === null)) { diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index 171f73cfe1f..f799045e4ba 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -11,6 +11,8 @@ cd "$ROOT" # shellcheck source-path=SCRIPTDIR source=../lib/harness.sh source test/lib/harness.sh harness_begin push-to-talk "just test push-to-talk$(harness_argv "$@")" +# The run directory is this measurement's evidence, so keep it even when it passes. +export HARNESS_KEEP="${HARNESS_KEEP:-1}" [[ $# -eq 0 ]] || { echo 'usage: just test push-to-talk' >&2 @@ -40,4 +42,4 @@ if [[ $driver_status -ne 0 ]]; then cat "$HARNESS_RUN/driver.log" >&2 exit "$driver_status" fi -bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" >"$HARNESS_RUN/summary.log" +bun test/audio-quality/clients/js/ptt-analyze.ts "$HARNESS_RUN" | tee "$HARNESS_RUN/summary.log" From 5bdb3036c7e722a66f08fb87309ec04c2f47c4e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 19:53:09 -0800 Subject: [PATCH 17/19] test(publish): route the push-to-talk listener through the relay Connections pool by URL, and a pooled origin resolves a same-page publication locally, so with one URL the listener never crossed the relay. A listener-only query the relay ignores gives it its own session, and the driver now refuses a run whose two origins are the same. On an ideal path nothing excuses a lost frame, so the analyzer fails on a head or any written frame the listener never received. The jitter buffer now ends at the first frame to reach the delay rather than the newest decoded before the onset. Co-Authored-By: Claude Opus 5.5 --- quest/m1/push-to-talk-latency.md | 6 ++--- .../clients/js/ptt-analyze.test.ts | 22 ++++++++++++++++ test/audio-quality/clients/js/ptt-analyze.ts | 26 ++++++++++++------- test/audio-quality/clients/js/ptt.ts | 1 + test/audio-quality/clients/js/src/ptt.ts | 9 ++++++- 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/quest/m1/push-to-talk-latency.md b/quest/m1/push-to-talk-latency.md index b39683a463d..c13c4e40e26 100644 --- a/quest/m1/push-to-talk-latency.md +++ b/quest/m1/push-to-talk-latency.md @@ -14,9 +14,9 @@ Measured 2026-11-04 by push-to-talk ([#7101](https://github.com/moq-dev/moq/issues/7101)). The setup was `just test push-to-talk` on a direct local relay with no shaping or loss, 60 ms Opus at 128 kbit/s, and a fixed 300 ms `delay`. Chromium 156 ran -headless on Linux. There were five runs of 75 separated presses. `summary.json` -`path` splits each press at page-clock boundaries that sum exactly to press → -projected output. +headless on Linux. There were five runs of 22 presses each, 15 of them +separated, so 75 separated presses in all. `summary.json` `path` splits each +press at page-clock boundaries that sum exactly to press → projected output. | Part | Boundaries | Median ms | Max ms | | --- | --- | --- | --- | diff --git a/test/audio-quality/clients/js/ptt-analyze.test.ts b/test/audio-quality/clients/js/ptt-analyze.test.ts index 80813b366ba..c055d72b525 100644 --- a/test/audio-quality/clients/js/ptt-analyze.test.ts +++ b/test/audio-quality/clients/js/ptt-analyze.test.ts @@ -85,6 +85,28 @@ test("a complete run passes, reports its latency, and splits each press exactly" expect(parts).toBe(press.audible); }); +test("the delay is filled by the first frame to reach it, not the newest before the onset", async () => { + // The sixth frame decodes just before the onset, adding to a ring already leaving its stall. + const result = trace(); + const sixth = (result.presses[0].at - 20 + 5 * FRAME) * 1000; + const late = result.decoded.find((frame) => frame.timestamp === sixth); + if (!late) throw new Error("no sixth frame"); + late.at = result.audible[0].rendered - 1; + const { exitCode, summary } = await analyze(result); + expect(exitCode).toBe(0); + expect(summary.presses[0].path.jitter_buffer).toBe(4 * FRAME); + expect(summary.presses[0].path.render).toBe(10); +}); + +test("a head that never reaches the listener fails the run", async () => { + const result = trace(); + const head = result.submitted.find((frame) => frame.at >= result.presses[2].at); + result.arrivals = result.arrivals.filter((arrival) => arrival[1] * 1000 !== head?.timestamp); + const { exitCode, stderr } = await analyze(result); + expect(exitCode).toBe(1); + expect(stderr).toContain("utterance head was not received"); +}); + test("a new encoder fails as a probe, but the live encoder's release flush does not", async () => { // A flush leaves no trace in the page's records, so the passing run above is the release flush. const result = trace(); diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index 2577161b750..acb72e5ba86 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -57,14 +57,16 @@ const presses = result.presses const write = sent.get(first.timestamp); const arrival = arrivals.get(first.timestamp); const output = decoded.get(first.timestamp); - // The utterance's newest PCM decoded before the render onset: the one that filled the delay. - const filled = input - .flatMap((frame) => decoded.get(frame.timestamp) ?? []) - .filter((frame) => frame.at <= audible.rendered) - .reduce<(typeof result.decoded)[number] | undefined>( - (newest, frame) => (newest && newest.at >= frame.at ? newest : frame), - undefined, - ); + // The first PCM that reached the delay from the head: the ring leaves its stall on it, and a + // later frame decoded before the onset only adds to a ring already playing. + const delay = result.state?.delay; + const filled = + delay === undefined + ? undefined + : input + .flatMap((frame) => decoded.get(frame.timestamp) ?? []) + .toSorted((a, b) => a.at - b.at) + .find((frame) => frame.timestamp + (frame.duration ?? 0) - first.timestamp >= delay * 1000); if (!write || !arrival || !output || !filled) return null; const pressed = press.capture.end / 1000 + (press.at - press.capture.at); const packetization = (first.timestamp + (first.duration ?? 0)) / 1000 - pressed; @@ -120,12 +122,14 @@ const summary = { subscriptions: result.subscriptions, unencoded: result.submitted.filter((frame) => !encoded.has(frame.timestamp)).length, unsent: result.encoded.filter((frame) => !sent.has(frame.timestamp)).length, + // An ideal path loses nothing, so every written frame must reach the listener. + unreceived: result.sent.filter((frame) => !arrivals.has(frame.timestamp)).length, }; await Bun.write(join(directory, "summary.json"), JSON.stringify(summary, null, 2)); // The report: the latency and its parts, and the completeness the run gates on. Per-press detail // stays in summary.json. -const { latency, path, delay, unencoded, unsent } = summary; -console.log(JSON.stringify({ latency, path, delay, unencoded, unsent }, null, 2)); +const { latency, path, delay, unencoded, unsent, unreceived } = summary; +console.log(JSON.stringify({ latency, path, delay, unencoded, unsent, unreceived }, null, 2)); const failures: string[] = []; if (presses.some((press, i) => (i === 0 || i >= 8) && press.audible === null)) { @@ -136,6 +140,8 @@ if (result.encoders.length) failures.push("an encoder was created again"); if (result.subscriptions.length) failures.push("subscription reopened"); if (presses.some((press) => !press.submitted || !press.firstEncoded)) failures.push("utterance head was not encoded"); if (summary.unencoded || summary.unsent) failures.push("submitted audio was not encoded and sent completely"); +if (presses.some((press) => !press.firstReceived)) failures.push("utterance head was not received"); +if (summary.unreceived) failures.push("written audio did not all reach the listener"); if (failures.length) { console.error(failures.join("; ")); process.exitCode = 1; diff --git a/test/audio-quality/clients/js/ptt.ts b/test/audio-quality/clients/js/ptt.ts index fef9b4fcd61..a4b4946bee8 100644 --- a/test/audio-quality/clients/js/ptt.ts +++ b/test/audio-quality/clients/js/ptt.ts @@ -58,6 +58,7 @@ try { }); const initial = await page.evaluate(() => globalThis.ptt.state()); if (initial.publisher !== "webtransport" || initial.viewer !== "webtransport") throw new Error("not WebTransport"); + if (!initial.separate) throw new Error("the watcher shares the publisher's origin, bypassing the relay"); await page.waitForTimeout(3000); await page.evaluate(() => globalThis.ptt.press(false)); await page.waitForTimeout(5000); diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index c877e7314b0..90c645aae7c 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -111,7 +111,12 @@ publish.source = "camera"; document.body.append(publish); const watch = document.createElement("moq-watch") as MoqWatch; -watch.url = publish.url; +// Connections pool by URL, and a pooled origin resolves a same-page publication locally. A +// listener-only query, which the relay ignores, gives the watcher its own session, so the media +// crosses the relay. +const listener = new URL(required("relay")); +listener.searchParams.set("role", "listener"); +watch.url = listener.toString(); watch.name = publish.name; watch.setAttribute("delay", "300ms"); watch.setAttribute("visible", "always"); @@ -197,6 +202,8 @@ export const measurement = { return { publisher: publish.connection.transport.peek() as string | undefined, viewer: watch.connection.transport.peek() as string | undefined, + // Distinct origins: the watcher can only reach the publication through the relay. + separate: watch.connection.origin.peek() !== publish.connection.origin.peek(), config: publish.audio.out.catalog.peek(), selected: audio.source.out.track.peek(), tapped: tapped !== undefined && tapped === audio.out.root.peek(), From ce6ea0deded676bb55c0f1ea944e932f9ff5f5e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 20:47:00 -0800 Subject: [PATCH 18/19] quest(push-to-talk): keep the epic for a head lost at the listener Five runs through the relay were complete at the publisher and on the wire, but in one press after a 60 s mute the head's group arrived just behind the next frame's and never reached the decoder. That is the epic's Goal failing, so the epic stays with that defect as its remaining work, and the runner now fails on a head received but never decoded. The latency quest takes the relay runs' numbers. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 3 +- quest/m1/push-to-talk-latency.md | 57 ++++++----- quest/m1/push-to-talk/README.md | 96 +++++++++++++++++++ ...bandwidth-grant-in-moq-audio-instead-of.md | 2 +- .../clients/js/ptt-analyze.test.ts | 11 +++ test/audio-quality/clients/js/ptt-analyze.ts | 3 + 6 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 quest/m1/push-to-talk/README.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 74cff5c81a1..46eef9dc6d7 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -160,7 +160,8 @@ per-frame CPU growth since #5926. - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor -- [Push-to-talk's press-to-audible latency is chosen, not inherited](/quest/m1/push-to-talk-latency.md) - each part of an ideal-path press's 466 ms, chiefly the 300 ms delay and 115 ms device output, is reduced through its knob or kept with a reason +- [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - consumers hold a disabled rendition, and an unmute encodes at once with the mic still open; a reordered head after a long mute is still lost at the listener +- [Push-to-talk's press-to-audible latency is chosen, not inherited](/quest/m1/push-to-talk-latency.md) - each part of an ideal-path press's 469 ms, chiefly the 300 ms delay and 115 ms device output, is reduced through its knob or kept with a reason - [moq-mux presents a broadcast on the publisher's timestamps](/quest/m1/presentation-timeline.md) - `moq play`'s audio-anchored playout timing as a moq-mux API the studio's broadcast sources share - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS diff --git a/quest/m1/push-to-talk-latency.md b/quest/m1/push-to-talk-latency.md index c13c4e40e26..c46e3b1edad 100644 --- a/quest/m1/push-to-talk-latency.md +++ b/quest/m1/push-to-talk-latency.md @@ -10,35 +10,37 @@ same breakdown. ## Plan Measured 2026-11-04 by -[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760), which finished -push-to-talk ([#7101](https://github.com/moq-dev/moq/issues/7101)). The setup -was `just test push-to-talk` on a direct local relay with no shaping or loss, -60 ms Opus at 128 kbit/s, and a fixed 300 ms `delay`. Chromium 156 ran -headless on Linux. There were five runs of 22 presses each, 15 of them -separated, so 75 separated presses in all. `summary.json` `path` splits each -press at page-clock boundaries that sum exactly to press → projected output. +[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760), for +[Push-to-talk](/quest/m1/push-to-talk/README.md). The setup was `just test +push-to-talk` through a local relay, publisher and listener on separate +sessions, with no shaping or loss, 60 ms Opus at 128 kbit/s, and a fixed 300 +ms `delay`. Chromium 156 ran headless on Linux. There were five runs of 22 +presses each, 15 of them separated, so 75 separated presses in all; one press +lost its head (see Push-to-talk) and has no breakdown, leaving 74. +`summary.json` `path` splits each press at page-clock boundaries that sum +exactly to press → projected output. | Part | Boundaries | Median ms | Max ms | | --- | --- | --- | --- | -| capture | worklet delivery and framing | 28 | 72 | -| packetization | media the head frame still needed after the press | 40 | 60 | -| encode | encoder input to container write | 0.9 | 23 | -| network | container write to arrival, through the relay | 0.2 | 4 | -| decode | arrival to decoded PCM | 0.7 | 20 | -| jitter buffer | head's PCM to the PCM that filled the delay | 256 | 297 | -| render | that PCM to the render onset | 24 | 49 | -| device | render onset to projected output | 115 | 122 | -| total | press to projected output | 466 | 505 | +| capture | worklet delivery and framing | 29 | 110 | +| packetization | media the head frame still needed after the press | 39 | 60 | +| encode | encoder input to container write | 0.8 | 14 | +| network | container write to arrival, through the relay | 3.2 | 15 | +| decode | arrival to decoded PCM | 0.7 | 13 | +| jitter buffer | head's PCM to the first PCM that filled the delay | 252 | 268 | +| render | that PCM to the render onset | 21 | 54 | +| device | render onset to projected output | 115 | 119 | +| total | press to projected output | 469 | 576 | -The capture and packetization split is approximate, but their 63 ms median -sum is exact. Playout on top of one-way delivery was 266 ms median and 289 ms -max, inside the 300 ms the maintainer set. +The capture and packetization split is approximate, but their 66 ms median sum +is exact. Playout on top of one-way delivery was 261 ms median and 294 ms max +over all 75, inside the 300 ms the maintainer set. Biggest contributors and their knobs: - **Playout delay, about 300 ms.** The ring starts once it holds the delay in decoded PCM, counted from the head's first sample. That is why the jitter - buffer plus packetization sum to about 296 ms. Knob: `` `delay`. + buffer plus packetization sum to about 291 ms. Knob: `` `delay`. This test fixes it at 300 ms. In `"auto"`, the default, the target is `max(measured, advertised) + frame + catalog delay` (`js/watch/src/audio/latency.ts`). @@ -55,12 +57,15 @@ Biggest contributors and their knobs: 20 ms. Under a fixed delay it is absorbed by the delay. In `"auto"` it adds one frame to the target. Knob: the encoder's `frameDuration`. -No part looked like a bug. On an ideal path network, encode and decode are -about 1 ms each, and the head is not trimmed as decoder warmup. The fixed -delay bypasses the jitter estimator. In `"auto"`, the 44 ms median per-press -arrival-lateness spread, from capture bursts rather than the network, would -make the measured term nonzero; that is the estimator working as specified -in [Audio jitter target](/quest/m1/audio-jitter-target/README.md). +No part looked like a bug. On an ideal path, encode and decode take about 1 ms +and the relay hop about 3 ms, and the head is not trimmed as decoder warmup. +The one lost head is an ordering defect, owned by +[Push-to-talk](/quest/m1/push-to-talk/README.md), not a latency part. The +fixed delay bypasses the jitter estimator. In `"auto"`, the 46 ms median +per-press arrival-lateness spread, from capture bursts rather than the +network, would make the measured term nonzero; that is the estimator working +as specified in [Audio jitter +target](/quest/m1/audio-jitter-target/README.md). Open: decide which knob to turn first, and whether to measure on a real device before deciding. diff --git a/quest/m1/push-to-talk/README.md b/quest/m1/push-to-talk/README.md new file mode 100644 index 00000000000..917c1a84e7c --- /dev/null +++ b/quest/m1/push-to-talk/README.md @@ -0,0 +1,96 @@ +# [M] Push-to-talk loses no speech + +## Goal + +A listener subscribed while a publisher is muted hears the next utterance from +its first frame. Unmuting costs no catalog round trip, no resubscribe and no +microphone reopen. A muted rendition still costs no media bytes. + +## Plan + +From [#7101](https://github.com/moq-dev/moq/issues/7101), traced 2026-10-28. With +`Audio.Encoder` `enabled: false`, watch deselects the rendition and +unsubscribes. On unmute the listener waits for the catalog flip, then its +SUBSCRIBE reaches the publisher. The demand gate discards everything encoded +before that (`js/publish/src/audio/encoder.ts:686`). At 100 ms RTT that is +three one-way trips plus the probe and a frame (~515 ms to the first fresh +frame). The emptied buffer then refills the 300 ms delay, so the result is +~820-950 ms to audible, with 250-400 ms of speech never encoded. + +Decided 2026-10-28 (maintainer): + +- **Hold, not DTX.** DTX is for silence the app can't predict. Push-to-talk + knows when it is silent, and a held subscription costs no media bytes and no + encoder CPU. DTX is off on main anyway ([#4919](https://github.com/moq-dev/moq/pull/4919)). +- **Players hold a disabled rendition**, audio and video, by default, in + every first-party player. Exports keep dropping it, since a held track + sends nothing and an interleaving muxer or HLS player may wait on it. The + hang spec weakens from "must not select" to "should not select" a disabled + rendition. +- **The publisher keeps its probe result across a disable**, and the + element's `muted` and `invisible` leave capture open. +- **Encoding no longer waits for demand**, in every first-party publisher + (finished by [moq-dev/moq#7554](https://github.com/moq-dev/moq/pull/7554)). + A listener who + subscribes after the unmute still gets the start within its `delay.max`. This + replaces a publisher head buffer, which was considered and dropped as the + same idea special-cased. + +The epic's own work: the acceptance proof `just test push-to-talk` (22 +presses, 60 ms Opus, 300 ms delay). A held listener's unmute pays no catalog +round trip and no resubscribe, and every frame of each utterance is encoded, +sent, received, and decoded from its first frame. + +Decided with the maintainer on +[moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760): + +- ✅ **Ideal conditions only** (2026-11-04): "don't use random packet loss + either, assume ideal conditions." Both elements reach a local relay + directly, on separate sessions so the media crosses it, with no shaping, + loss, or queueing. The bufferbloat and 2% loss profiles were removed. +- ✅ **Latency: at most 300 ms of playout on top of one-way delivery** + (2026-11-03): "We can't ever expect 300ms latency if it takes more than + 300ms to deliver a packet." No clocked live start or other startup + machinery. Five ideal runs on 2026-11-04 held it on all 75 separated presses + (median 261 ms, max 294 ms). The breakdown and its knobs are in + [Push-to-talk latency](/quest/m1/push-to-talk-latency.md). +- ✅ **No encoder recovery on re-enable** (2026-11-03). Asked first why a + fatal encoder error is expected at all: it is not. No push-to-talk run + reported one, and no issue reports a publisher encoder failure; the latch + came from a review finding + ([moq-dev/moq#5441](https://github.com/moq-dev/moq/pull/5441)). WebCodecs + fails an encoder only for a config the platform rejects, an internal codec + error, or reclamation, which Chromium applies only to inactive, backgrounded + hardware codecs; Opus is software. A failure stays latched until a device, + settings or `source` change. + +The publisher's tail drop is fixed: +[moq-dev/moq#7725](https://github.com/moq-dev/moq/pull/7725) flushes the codec +before a disable's discontinuity. A frame submitted under 1 ms before release +used to be lost in 3 of 7 runs; it was lost in none of 12 since. + +What remains is a lost head at the listener. In one of 110 presses over the +2026-11-04 relay runs (run 3, press 20, after a 60 s mute), the +head's group 918 and the next frame's group 919, written at +72 and +73 ms, +reached the listener reordered: 919 at +187 ms and 918 at +192 ms, about 115 +ms in transit against the usual 3 ms. The listener decoded 919 at +191.7 ms. +Group 918 arrived 0.4 ms later and never reached the decoder, so the utterance +played from its second frame, 60 ms late. The JS ordered read still delivers a +group below its cursor as a late backfill +(`js/hang/src/container/consumer.ts`), so the head was most likely dropped as +late between the read and the decoder (`js/watch/src/audio/decoder.ts`); pin +that down first. After a declared end, the next group is known to be the end's +group plus one. Holding for it within the delay budget would keep the head +without adding startup latency, but weigh that against the no-window decision +in [Container reads drop backfill](/quest/m1/read-backfill.md). + +The runner fails on a head that is received but never decoded, so the proof +catches this. A complete set of runs finishes this epic. + +## Closes + +- [#7101](https://github.com/moq-dev/moq/issues/7101) - a disabled audio rendition is deselected, so the start of the next utterance is lost + +## Related + +- [Container reads drop backfill](/quest/m1/read-backfill.md) - changes how the ordered read treats a group below its cursor, where this head was lost diff --git a/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md b/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md index 9f11e236e9c..1bb1d2d1eb3 100644 --- a/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md +++ b/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md @@ -45,7 +45,7 @@ grant; `Options::bandwidth` documents that 2026-10-04 with the enabled flag), through the disable and re-enable rule [simulcast rung disable](/quest/m1/simulcast-rung-disable.md) puts beside `Control` rather than a second one; the encoder never errors. Viewers with - no enabled alternative hold it (the push-to-talk hold, [#7101](https://github.com/moq-dev/moq/issues/7101)), + no enabled alternative hold it ([push-to-talk](/quest/m1/push-to-talk/README.md)), so its demand can stay while its grant is below the floor; recovery evaluates a hypothetical share against the current estimate, as the ladder controller does. The reservation's ceiling is diff --git a/test/audio-quality/clients/js/ptt-analyze.test.ts b/test/audio-quality/clients/js/ptt-analyze.test.ts index c055d72b525..211f6a9e82c 100644 --- a/test/audio-quality/clients/js/ptt-analyze.test.ts +++ b/test/audio-quality/clients/js/ptt-analyze.test.ts @@ -107,6 +107,17 @@ test("a head that never reaches the listener fails the run", async () => { expect(stderr).toContain("utterance head was not received"); }); +test("a head that arrives but never reaches the decoder fails the run", async () => { + // A head reordered behind the next frame's group, then dropped before decode. + const result = trace(); + const head = result.submitted.find((frame) => frame.at >= result.presses[2].at); + result.decoding = result.decoding.filter((frame) => frame.timestamp !== head?.timestamp); + result.decoded = result.decoded.filter((frame) => frame.timestamp !== head?.timestamp); + const { exitCode, stderr } = await analyze(result); + expect(exitCode).toBe(1); + expect(stderr).toContain("utterance head was not decoded"); +}); + test("a new encoder fails as a probe, but the live encoder's release flush does not", async () => { // A flush leaves no trace in the page's records, so the passing run above is the release flush. const result = trace(); diff --git a/test/audio-quality/clients/js/ptt-analyze.ts b/test/audio-quality/clients/js/ptt-analyze.ts index acb72e5ba86..5aaa74d65d0 100644 --- a/test/audio-quality/clients/js/ptt-analyze.ts +++ b/test/audio-quality/clients/js/ptt-analyze.ts @@ -91,6 +91,7 @@ const presses = result.presses received: received.length, firstEncoded: input[0] ? encoded.has(input[0].timestamp) : false, firstReceived: input[0] ? arrivals.has(input[0].timestamp) : false, + firstDecoded: input[0] ? decoded.has(input[0].timestamp) : false, arrival: earliest ? earliest[0] - press.at : null, audible: audible ? audible.at - press.at : null, delivery, @@ -141,6 +142,8 @@ if (result.subscriptions.length) failures.push("subscription reopened"); if (presses.some((press) => !press.submitted || !press.firstEncoded)) failures.push("utterance head was not encoded"); if (summary.unencoded || summary.unsent) failures.push("submitted audio was not encoded and sent completely"); if (presses.some((press) => !press.firstReceived)) failures.push("utterance head was not received"); +if (presses.some((press) => press.firstReceived && !press.firstDecoded)) + failures.push("utterance head was not decoded"); if (summary.unreceived) failures.push("written audio did not all reach the listener"); if (failures.length) { console.error(failures.join("; ")); From 3155ec7f67e308689156fd4da80f4db980b2b132 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 21:03:39 -0800 Subject: [PATCH 19/19] docs(audio-quality): describe the push-to-talk jitter buffer and refusals as checked Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index e9bcc019957..7fdfd1b2868 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -147,10 +147,10 @@ separated presses have an identifiable onset. `summary.json` matches inputs, outputs, writes, and arrivals by media timestamp, so a late arrival is still attributed to its utterance. The command checks pipeline completeness, not the epic's -latency verdict. It refuses missing encoder inputs, missing heads, submitted frames that never -encode or reach the container, reopened capture, a new encoder (a probe or a rebuilt codec) or -subscription, and separated utterances with no audible output. It also fails on a broken driver or -invalid capture. +latency verdict. It refuses missing encoder inputs, a head that never reaches the listener or never +decodes, submitted frames that never encode or reach the container, written frames that never reach +the listener, reopened capture, a new encoder (a probe or a rebuilt codec) or subscription, and +separated utterances with no audible output. It also fails on a broken driver or invalid capture. The latency goal is at most 300 ms of playout added on top of one-way delivery. For each separated press, `delivery` is the slowest of the utterance's frames that arrived before its onset, measured @@ -160,10 +160,10 @@ that onset. `path` follows the head frame from the press to projected output in it: `capture` (worklet delivery and framing, after the media the frame still needed), `packetization` (that media, placed on the media clock by the newest chunk captured before the press), `encode` (codec and container write), `network` (write to arrival, through the relay), -`decode` (arrival to decoded PCM), `jitter_buffer` (the head's PCM to the newest PCM decoded before -the onset, which filled the playout delay), `render` (that PCM to the render onset: the worklet -hand-off and the context's render callback), and `device`, as the stages below name them. `latency` -and `path` give each one's min, median and max. +`decode` (arrival to decoded PCM), `jitter_buffer` (the head's PCM to the first PCM that fills the +playout delay), `render` (that PCM to the render onset: the worklet hand-off and the context's +render callback), and `device`, as the stages below name them. `latency` and `path` give each one's +min, median and max. The command prints `latency`, `path`, and the completeness counts, and keeps its run directory even when it passes (set `HARNESS_KEEP=0` to drop it). Nightly runs it and keeps the raw evidence