diff --git a/js/net/src/lite/datagram-tracks.test.ts b/js/net/src/lite/datagram-tracks.test.ts index 22df152a2ea..4b874b14726 100644 --- a/js/net/src/lite/datagram-tracks.test.ts +++ b/js/net/src/lite/datagram-tracks.test.ts @@ -3,14 +3,18 @@ import { yieldNow } from "@moq/clock"; import { hostTime } from "@moq/clock/test"; import { Signal } from "@moq/signals"; import { accept, connect } from "../connection/index.ts"; +import { SessionCode } from "../error.ts"; +import { hooks } from "../internal.ts"; import { createMockTransportPair, type MockTransport } from "../mock.ts"; import { Producer as Origin } from "../origin.ts"; import * as Path from "../path.ts"; import { sendOrder } from "../priority.ts"; +import { Writer } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import { Budget } from "../util/yield.ts"; import { wireOf } from "../wire.ts"; -import { ALPN_04, ALPN_05, ALPN_06, ALPN_07_WIP } from "./version.ts"; +import { DataType } from "./stream.ts"; +import { ALPN_04, ALPN_05, ALPN_06, ALPN_07_WIP, Version } from "./version.ts"; const url = new URL("https://localhost/datagrams"); const path = Path.from("live"); @@ -492,6 +496,71 @@ for (const alpn of [ALPN_05, ALPN_06, ALPN_07_WIP]) { }); } +// A peer's protocol violation fails the session at once, without the drain a close gets. A lite-05/06 +// datagram track whose SUBSCRIBE_FINISH was accepted keeps that end, even while its FIN lags. +for (const [alpn, version] of [ + [ALPN_05, Version.DRAFT_05], + [ALPN_06, Version.DRAFT_06], +] as const) { + test(`${alpn}: a duplicate SETUP after a datagram track's end keeps the end`, async () => { + const source = new Origin(); + const publication = source.route(path, { hidden: true }); + const datagrams = publication.broadcast.createDatagrams("audio"); + publication.announce(); + const pair = createMockTransportPair(alpn); + // Once armed, the publisher's FINs on the request streams it accepts never leave. + let holding = false; + const holdFin = (writable: WritableStream) => { + const writer = writable.getWriter(); + return new WritableStream({ + write: (bytes) => writer.write(bytes), + abort: (reason) => writer.abort(reason), + close: () => (holding ? new Promise(() => {}) : writer.close()), + }); + }; + Object.defineProperty(pair.server, "incomingBidirectionalStreams", { + value: pair.server.incomingBidirectionalStreams.pipeThrough( + new TransformStream({ + transform: (bidi, controller) => + controller.enqueue({ readable: bidi.readable, writable: holdFin(bidi.writable) }), + }), + ), + }); + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client }), + accept({ url, transport: pair.server, publish: source.consume() }), + ]); + const remote = wireOf(client).consume(path); + const reader = remote.datagrams("audio"); + reader.setPriority(5); + const accepted = Promise.withResolvers(); + const endTrack = hooks.endTrack; + const spy = spyOn(hooks, "endTrack").mockImplementation((...args) => { + endTrack(...args); + accepted.resolve(); + }); + try { + await reader.info(); + while (datagrams.priority.peek() !== 5) await datagrams.priority.changed(); + holding = true; + datagrams.finish(); + await accepted.promise; + const writer = await Writer.open(pair.server, { version }); + // The client claims the Setup Stream on its type, so the type alone is the duplicate. + await writer.u53(DataType.Setup); + expect((await pair.client.closed).closeCode).toBe(SessionCode.ProtocolViolation); + expect(await reader.recv()).toBeUndefined(); + } finally { + spy.mockRestore(); + reader.close(); + remote.close(); + client.abort(); + server.abort(); + source.close(); + } + }); +} + // Holds back what the publisher writes on the next request stream it accepts once armed, until // released: that subscription's answer stays outstanding while everything else flows. function holdNextAnswer(transport: MockTransport): { arm(): void; release(): void } { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index df7eaa6aada..051672e0dee 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -2808,9 +2808,12 @@ export class Subscriber { close(err: Error = new SessionError(SessionCode.Cancel)) { this.#closed.abort(err); - for (const { track, finished } of this.#subscribes.values()) { + for (const { track, finished, end, datagrams } of this.#subscribes.values()) { if (finished) track.finish(); track.abort(err); + // Lite-05/06: the stream track's end is its datagrams' end. An abort after the declared + // end settled ends it cleanly, even while the FIN that would finalize it is in flight. + if (datagrams && end !== undefined && hooks.trackReleased(track).peek() === null) datagrams.ended = true; } this.#subscribes.clear(); diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index f448acb0bc2..b7cc766a6a8 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -139,6 +139,11 @@ publisher's write budget yielded under load while writing SUBSCRIBE_FINISH, past the case's microtask drain. [moq-dev/moq#8000](https://github.com/moq-dev/moq/pull/8000) stubbed that budget, merging after the quarantine was written. The case was re-enabled in [moq-dev/moq#8110](https://github.com/moq-dev/moq/pull/8110). +Its follow-up: a peer's protocol violation, such as a duplicate SETUP, fails the +session without the drain a close gets, so a lite-05/06 datagram track whose +SUBSCRIBE_FINISH arrived before its FIN aborted with the session's error. Its +datagrams now finish whenever the stream track's settled end survives the abort, +fixed by [moq-dev/moq#8157](https://github.com/moq-dev/moq/pull/8157). `sh/dispatch-query.test.py`'s terminate test was never load-bound. A TERM landing just after the dispatcher's `wait -n` returned left Bash marking the @@ -153,6 +158,5 @@ Public API: none. Wire: none. ## Required - [Fallback playback](/quest/m0/test-flakes-2/ietf22-fallback.md) - a browser viewer follows a fallback within the media lane's bound without dropping its session -- [The datagram-end close follow-up](/quest/m0/test-flakes-2/datagram-end-close.md) - investigate whether duplicate SETUP can override an accepted lite-06 SUBSCRIBE_END during shutdown - [moq-transcode's idle-rung cancel test survives load](/quest/m0/test-flakes-2/transcode-idle-rung-cancel.md) - `cancelling_an_idle_rung_reports_cancel` stops seeing `serve` return before its abort lands - [moq-gst's sink-restart pad test survives load](/quest/m0/test-flakes-2/gst-sink-restart-pads.md) - `a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times` keeps its pads under load diff --git a/quest/m0/test-flakes-2/datagram-end-close.md b/quest/m0/test-flakes-2/datagram-end-close.md deleted file mode 100644 index 3cfc55dbf34..00000000000 --- a/quest/m0/test-flakes-2/datagram-end-close.md +++ /dev/null @@ -1,105 +0,0 @@ -# [S] datagram-tracks' end-before-close case passes under load - -## Goal - -`a session close right after a datagram track's end keeps the end` in -`js/net/src/lite/datagram-tracks.test.ts` passes under load for every draft in -multi-file runs, fixed at its cause under the -[epic](/quest/m0/test-flakes-2/README.md)'s rules. Done when its `test.skip` -is removed with its assertions unchanged. - -## Plan - -The original end/close case was re-enabled by PR #8110 after PR #8000 -stubbed the publisher write budget. The additional duplicate-SETUP -investigation below remains open. - -Added 2026-11-06 (maintainer's standing rule that flakes join this epic). -[moq-dev/moq#7986](https://github.com/moq-dev/moq/pull/7986) and -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) both hit it in -their checks without touching the code it covers, and each quarantined it with -every assertion kept; this file merges their two quests. It came with -[moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951)'s new case. - -Reproductions, 2026-11-06, with six CPU loops on four cores: - -- On #7986's head and on `origin/main`, from `js/net`: `bun test - src/lite/datagram-tracks.test.ts` failed 3 of 4 loaded runs there and 3 of - 3 on `main`. The same file filtered to this case passed 10 of 10 idle and 8 - of 8 loaded runs, so the file's other cases are what make the window miss. -- On #7670 at `13a544e41`: 6 of 10 runs, and 4 of 8 on `main` at - `6ca2ed5e3`, always the lite-06 case, with `subscribe error: id=1 - broadcast=live track=audio error=remote error: 0`. After quarantining - lite-06 alone, lite-05 failed 1 of 8 further loaded runs the same way, so the - whole case is quarantined and the fix must cover all three drafts. -- The failure rejects with `Session: remote error: 0` at `error.ts`'s - `closeError` from `connection.ts`'s `abort`: `reader.recv()` rejected - instead of ending. - -The test ends the publisher's track, then burns microtasks so the end lands -"all within this host task", then closes the session and expects `recv()` to -report the end rather than the close. That holds only when the end's whole -path runs in microtasks, and the failure says it did not: the close reached -the subscriber while `entry.end` was still unset, so `Subscriber.close` -aborted the datagram track. That is the correct answer for a close that -precedes the end, so the assertion is fine and the assumption about when the -end has landed is the suspect. Lead, unverified: a host task still -outstanding, such as the `yieldNow` (`clock.task`) on the publisher's serve -loop or at the start of the FIN handler's `#datagramsRouted`, is starved by -the microtask chain. Lite-06 fits: its SUBSCRIBE carries no range list, so -fewer writes precede the publisher's read of the end. Confirm which host task -is outstanding before choosing the fix; a later host task landing in the -burst, rather than a leftover one, would need a different answer. - -Fix it by waiting on an event that proves the subscriber read the end before -closing the session, rather than a microtask count: for example capture the -publisher's `SUBSCRIBE_FINISH` write through the mock as `holdNextAnswer` -does, or await a hook the subscriber calls on the response. Keep the close -between that evidence and `recv()`, so the case still covers the close that -lands after the end. Raising the microtask count, adding a wall-clock wait, or -retrying the test are not fixes. - -Reproduce first: temporarily remove the case's `test.skip`, then run the -whole file beside CPU loops, as above. Keep the failing output before -changing the test or the end path, and restore the skip until the fix. - -### Additional lite-06 investigation - -Planned 2026-11-06 under the maintainer's rule that a known flaky test is -quarantined immediately and its investigation planned. `Check` failed on this -case in -[run 42200239263](https://github.com/moq-dev/moq/actions/runs/42200239263/job/127833434287) -on the unrelated [moq-dev/moq#7981](https://github.com/moq-dev/moq/pull/7981), -where it was the only failing case in `js/net`. The unchanged module reproduced -it once in eight runs: `cd js/net && bun test src/lite/datagram-tracks.test.ts`. - -The case spins 1000 microtasks after `datagrams.finish()`, then aborts the -client, expecting the reader's `recv()` to return `undefined` because the end -already arrived. On failure the reader surfaces the session close instead -(`Session: remote error: 0`). The spin assumes the mock delivers the end in -microtasks, but the subscriber's response loop yields through `@moq/clock`'s -`yieldNow` (a host task) once its budget's 4 ms slice expires, which a loaded -runner can reach before the end is read. Whether that is only the fixture's -wait or a real gap between receiving the end and delivering it is not -established: diagnose before changing the wait or production code, and do not -keep the case passing by loosening the assertion. Base subsequently quarantined -all three protocol versions, as described above. - -Decided 2026-11-06 (maintainer, /quest-unblock): investigate the additional -[CodeRabbit finding](https://github.com/moq-dev/moq/pull/7981#discussion_r4491539836) -here: an accepted lite-06 `SUBSCRIBE_END` followed by duplicate SETUP can be -overridden by session failure before subscription finalization. Reproduce this -ordering deterministically and preserve the accepted end during shutdown. - -Public API: none. Wire: none. - -## Related - -- [moq-dev/moq#8000](https://github.com/moq-dev/moq/pull/8000) edited this - case: it stubs `Budget.prototype.poll` so the publisher's write budget never - yields a host task, and raises the microtask count to 10000. The quarantined - case keeps both. Stubbing the budget removes one suspected outstanding - `yieldNow`, so check whether it is the cause before choosing the fix; the - raised count alone is not one. On 2026-11-06, ten loaded runs (six busy - loops pinned to four of 24 cores) passed with and without the stub, so that - setup no longer reproduces it and cannot tell. diff --git a/quest/m1/ts-reliability/pcr-jump-flag.md b/quest/m1/ts-reliability/pcr-jump-flag.md index 382181145c8..53893ff678e 100644 --- a/quest/m1/ts-reliability/pcr-jump-flag.md +++ b/quest/m1/ts-reliability/pcr-jump-flag.md @@ -8,6 +8,6 @@ Validate actual-PCR-jump flags in the remaining TS recovery interop coverage. Decided 2026-11-06 (maintainer, /quest-unblock): ✅ Continue the actual-PCR-jump approach. Not chosen: rethink; drop the PR. -Decided 2026-11-06 (maintainer, /quest-unblock): ✅ Reproduce and fix the Rust `with_delay` schedule replacement in this PR. Hand the unrelated JS decoded-end/session-failure finding to [datagram-end-close](/quest/m0/test-flakes-2/datagram-end-close.md). Not chosen: fixing both findings here. +Decided 2026-11-06 (maintainer, /quest-unblock): ✅ Reproduce and fix the Rust `with_delay` schedule replacement in this PR. Hand the unrelated JS decoded-end/session-failure finding to the datagram-end close quest, fixed by [moq-dev/moq#8157](https://github.com/moq-dev/moq/pull/8157). Not chosen: fixing both findings here. Decided 2026-11-06 (maintainer, /quest-unblock): ✅ Construction-only delay supersedes runtime updates. TS Export and Release take delay when constructed, so retained state never changes its delay basis. Not chosen: full runtime retiming; landing without the fix.