diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 0c6d6445bcb..677caa4e7fe 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -41,7 +41,6 @@ Public API: none. Wire: none. ## Required - [Media resource instrumentation](/quest/m1/test-flakes-2/resource-instrumentation.md) - the media test stops seeing a playing player report nothing to release -- [Tone check grace](/quest/m1/test-flakes-2/tone-grace.md) - the interop tone check charges its grace only for audible playback, so a refilling ring under CPU starvation is not a failure - [IETF22 fallback](/quest/m1/test-flakes-2/ietf22-fallback.md) - an IETF22 browser viewer follows a fallback within the media lane's bound, keeping its session and its audio - [macOS receive credit](/quest/m1/test-flakes-2/macos-receive-credit.md) - `partial_control_requests_give_way` passes on the macOS lane, after checking for a shared platform cause - [moq-mux missing sequence](/quest/m1/test-flakes-2/mux-missing-sequence.md) - `missing_sequence_skips_on_live_track` runs on a paused clock diff --git a/quest/m1/test-flakes-2/tone-grace.md b/quest/m1/test-flakes-2/tone-grace.md deleted file mode 100644 index 509bf93c4e2..00000000000 --- a/quest/m1/test-flakes-2/tone-grace.md +++ /dev/null @@ -1,23 +0,0 @@ -# [S] The interop tone check charges its grace only for audible playback - -## Goal - -The JS interop driver's tone check fails only when audio that played was -silent or wrong, never because the player was still refilling its ring. Under -CPU starvation a browser cell waits out the re-stall instead of failing fast -with silence or a false "wrong tone". - -## Plan - -Found while closing the cpp to js budget flake -([moq-dev/moq#6442](https://github.com/moq-dev/moq/pull/6442)), never seen in -CI. With 8 browsers on 2 CPUs, 7 of 500+ main browser cells failed in under -12 s: the auto playout delay rose to 1 to 5 s, the ring re-stalled to refill, -and the 3 s `TONE_GRACE_MS` in `test/interop/clients/js/driver.ts` expired on -silence. Once it read stuttered audio as a wrong tone. - -Reproduce it under the same starvation, then measure the grace from audible -playback (the player's own stalled/playing state) rather than wall time since -the check began, keeping the overall cell budget unchanged. - -Public API: none. Wire: none. diff --git a/test/interop/clients/js/driver.ts b/test/interop/clients/js/driver.ts index 3b8c4c6a485..e90b2896842 100644 --- a/test/interop/clients/js/driver.ts +++ b/test/interop/clients/js/driver.ts @@ -39,6 +39,7 @@ import { waitForWatch, } from "./harness"; import { type Sample, TONE_CHECK } from "./src/contract"; +import { ToneCheck } from "./tone"; /** Remove the page's players, which closes their sessions, and wait until every transport has. */ async function closeSessions(page: Page): Promise { @@ -96,25 +97,8 @@ if ( process.exit(2); } -/** - * Whether every channel the player outputs carries its own tone, judged as tone.py judges a native - * subscriber, from the `codec` rendition. - */ -function toneHeard(state: Sample, codec: string): boolean { - return ( - state.audioCodec?.split(".")[0] === codec && - state.audioContext === "running" && - state.audioChannels === tones.length && - state.tones?.length === tones.length && - state.tones.every(({ rms, share }) => rms >= TONE_CHECK.minRms && share >= TONE_CHECK.minShare) - ); -} - const PAUSE_STABILITY_MS = 750; -/** How long audible audio may play before it has to carry the tone: the analyser's window, several times over. */ -const TONE_GRACE_MS = 3000; - async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: number): Promise { let previous = await readPlayerState(page); let stableSince = Date.now(); @@ -184,18 +168,20 @@ async function watch(codec: string): Promise { } const interactionDeadline = Date.now() + timeoutMs; + const tone = new ToneCheck({ codec, channels: tones.length }); + let failedAudio: Sample | undefined; try { - // Once audio plays, the tone gets a bounded grace rather than the whole timeout, so a - // wrong tone fails as fast as it does in the native subscribers. - await waitForState(page, errors, { - deadline: interactionDeadline, - description: "audible audio", - predicate: (state) => state.audioContext === "running" && (state.rms ?? 0) >= TONE_CHECK.minRms, - }); playing = await waitForState(page, errors, { - deadline: Math.min(interactionDeadline, Date.now() + TONE_GRACE_MS), + deadline: interactionDeadline, description: "the audio tone", - predicate: (state) => toneHeard(state, codec), + predicate: (state) => { + const result = tone.read(state); + if (result === "failed") { + failedAudio = state; + throw new Failure("audio tone", `after 3 s of playback: ${JSON.stringify(state)}`); + } + return result === "heard"; + }, }); } catch (err) { // Only a tone wait that ran out is an audio check failure; a page error reports as itself. @@ -203,10 +189,11 @@ async function watch(codec: string): Promise { // `wrong tone` is the marker the harness's control requires: audible audio from the pinned // rendition, in the publisher's layout, that does not carry its tones. Anything else failing is // a plain audio failure. - const state = await readPlayerState(page); + const state = failedAudio ?? (await readPlayerState(page)); const wrongTone = state.audioCodec?.split(".")[0] === codec && state.audioContext === "running" && + !state.audioStalled && state.audioChannels === tones.length && state.tones?.length === tones.length && state.tones.every(({ rms }) => rms >= TONE_CHECK.minRms); diff --git a/test/interop/clients/js/tone.test.ts b/test/interop/clients/js/tone.test.ts new file mode 100644 index 00000000000..f33ecad9d88 --- /dev/null +++ b/test/interop/clients/js/tone.test.ts @@ -0,0 +1,64 @@ +import { expect, test } from "bun:test"; +import { ToneCheck } from "./tone"; + +const wrong = { + at: 0, + audioContext: "running", + audioStalled: false, + audioCodec: "opus", + audioChannels: 2, + tones: [ + { rms: 0.35, share: 0 }, + { rms: 0.35, share: 0 }, + ], +}; +const expected = { codec: "opus", channels: 2 }; +const correct = wrong.tones.map(() => ({ rms: 0.35, share: 1 })); + +test("a refill pauses rather than resets or consumes the tone grace", () => { + const check = new ToneCheck(expected); + expect(check.read(wrong)).toBe("waiting"); + expect(check.read({ ...wrong, at: 1000 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 1100, audioStalled: true })).toBe("waiting"); + expect(check.read({ ...wrong, at: 6100, audioStalled: true })).toBe("waiting"); + expect(check.read({ ...wrong, at: 6200 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 8199 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 8200 })).toBe("failed"); +}); + +test("repeated observations cannot spend playback grace while the page is starved", () => { + const check = new ToneCheck(expected); + for (let poll = 0; poll < 100; poll++) expect(check.read(wrong)).toBe("waiting"); + expect(check.read({ ...wrong, at: 2999 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 3000 })).toBe("failed"); +}); + +test("suspension and startup buffering contribute no playback time", () => { + const check = new ToneCheck(expected); + expect(check.read({ ...wrong, audioStalled: true })).toBe("waiting"); + expect(check.read({ ...wrong, at: 5000 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 6000 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 6100, audioContext: "suspended" })).toBe("waiting"); + expect(check.read({ ...wrong, at: 16100, audioContext: "suspended" })).toBe("waiting"); + expect(check.read({ ...wrong, at: 16200 })).toBe("waiting"); + expect(check.read({ ...wrong, at: 18200 })).toBe("failed"); +}); + +test("analyser residue while stalled cannot satisfy the tone check", () => { + const check = new ToneCheck(expected); + expect(check.read({ ...wrong, audioStalled: true, tones: correct })).toBe("waiting"); + expect(check.read({ ...wrong, at: 10000, tones: correct })).toBe("heard"); +}); + +for (const bad of [ + { tones: wrong.tones.map(() => ({ rms: 0, share: 1 })) }, + { audioCodec: "mp4a.40.2", tones: correct }, + { audioChannels: 1, tones: correct }, + { tones: correct.slice(0, 1) }, +]) { + test(`playing invalid audio still fails: ${JSON.stringify(bad)}`, () => { + const check = new ToneCheck(expected); + expect(check.read({ ...wrong, ...bad })).toBe("waiting"); + expect(check.read({ ...wrong, ...bad, at: 3000 })).toBe("failed"); + }); +} diff --git a/test/interop/clients/js/tone.ts b/test/interop/clients/js/tone.ts new file mode 100644 index 00000000000..dd401f06463 --- /dev/null +++ b/test/interop/clients/js/tone.ts @@ -0,0 +1,37 @@ +import { type Sample, TONE_CHECK } from "./src/contract"; + +/** How long rendered audio may play before it has to carry the tone: several analyser windows. */ +const GRACE_MS = 3000; + +type Observation = Pick; + +/** Judge the expected tones after giving the running player a bounded grace to fill its analyser. */ +export class ToneCheck { + #expected: { codec: string; channels: number }; + #previous?: { at: number; playing: boolean }; + #played = 0; + + constructor(expected: { codec: string; channels: number }) { + this.#expected = expected; + } + + /** Wait through buffering, accept the tone, or fail after enough audio has played to judge it. */ + read(state: Observation): "waiting" | "heard" | "failed" { + const playing = state.audioContext === "running" && !state.audioStalled; + // Samples use the page's clock, so polling the same observation cannot spend the grace. + // Both ends must be playing: a refill or suspension contributes no playback time. + if (playing && this.#previous?.playing) this.#played += state.at - this.#previous.at; + this.#previous = { at: state.at, playing }; + if (!playing) return "waiting"; + + if ( + state.audioCodec?.split(".")[0] === this.#expected.codec && + state.audioChannels === this.#expected.channels && + state.tones?.length === this.#expected.channels && + state.tones.every(({ rms, share }) => rms >= TONE_CHECK.minRms && share >= TONE_CHECK.minShare) + ) { + return "heard"; + } + return this.#played >= GRACE_MS ? "failed" : "waiting"; + } +} diff --git a/test/justfile b/test/justfile index 5134a3ac50c..cda790b2459 100644 --- a/test/justfile +++ b/test/justfile @@ -22,6 +22,7 @@ harness_browser_tests := ''' ./open-gop.browser.ts ./publish-ui.browser.ts ./signals-dom.browser.ts + ./tone.test.ts ./src/contract.test.ts ./src/resume.test.ts '''