From 0f676c45a1245b08fa90a3c5d22cc8cfa7db34e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 11:49:34 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/test-flakes-2/reload-close-drain.md Quest-Claim: 7e6f8735d4ace01d From 376e971392e830c587eb68899b83e5f4f7c08ff8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:27:12 -0800 Subject: [PATCH 2/3] fix(net): a lite-07 subscriber FIN during a reply write still drains the final group The subscriber FINs once it read the finished track's tail, which can land while a reply write such as SUBSCRIBE_FINISH still yields on the write budget. That exit stopped owing the opened group streams, so a draining close aborted a final group whose FIN was still in flight. Every exit of the serving loop now owes them. Restores reload.test.ts's quarantined close-drain test and adds a deterministic lite-07 case. Co-Authored-By: Claude Opus 5.5 --- js/net/src/connection/reload.test.ts | 3 +- js/net/src/lite/connection.test.ts | 91 +++++++++++++++++ js/net/src/lite/publisher.ts | 18 ++-- quest/m0/test-flakes-2/README.md | 8 +- quest/m0/test-flakes-2/reload-close-drain.md | 100 ------------------- 5 files changed, 107 insertions(+), 113 deletions(-) delete mode 100644 quest/m0/test-flakes-2/reload-close-drain.md diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 4b36caa5dcd..066835bc2f6 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -941,8 +941,7 @@ test("a retired GOAWAY predecessor still closes by its handover cap", async () = // A publication only a token's grant covers keeps that grant while the session drains, so // closing the loop ends the tokens after the drain rather than cutting the final group. -// Quarantined pending quest/m0/test-flakes-2/reload-close-drain.md. -test.skip("closing the loop keeps the tokens until the session drains", async () => { +test("closing the loop keeps the tokens until the session drains", async () => { const original = globalThis.WebTransport; const clock = new Manual(); const installed = clock.install(); diff --git a/js/net/src/lite/connection.test.ts b/js/net/src/lite/connection.test.ts index a59006e6042..292ba9445fe 100644 --- a/js/net/src/lite/connection.test.ts +++ b/js/net/src/lite/connection.test.ts @@ -215,6 +215,38 @@ function holdFins(transport: MockTransport, streams: "uni" | "incoming-bidi") { }; } +// Once enabled, every write on a bidirectional stream the peer opens reaches the peer before +// the writer resumes a host task later, as when the write budget yields: a reply delivered +// while its writer still waits. +function yieldReplies(transport: MockTransport) { + let enabled = false; + const wrap = (writable: WritableStream) => { + const writer = writable.getWriter(); + return new WritableStream({ + async write(bytes) { + await writer.write(bytes); + if (enabled) await yieldNow(); + }, + abort: (reason) => writer.abort(reason), + close: () => writer.close(), + }); + }; + Object.defineProperty(transport, "incomingBidirectionalStreams", { + value: transport.incomingBidirectionalStreams.pipeThrough( + new TransformStream({ + transform(stream, controller) { + controller.enqueue({ readable: stream.readable, writable: wrap(stream.writable) }); + }, + }), + ), + }); + return { + enable: () => { + enabled = true; + }, + }; +} + // Buffer incoming unidirectional streams outside the transport, independently of when the // connection accepts them: its accept budget may yield until after the session closes. QMux // keeps a finished stream's unread bytes across a close, so the reads still owed land afterwards. @@ -360,6 +392,65 @@ for (const [alpn, version] of [ } }); + // A lite-07 subscriber FINs once it read the tail, which can land while SUBSCRIBE_FINISH's + // write still waits; the final group's FIN still holds the close. + if (keepsSubscribeOpen(version)) { + test(`a subscriber FIN during the end's write still drains the final group over ${versionName(version)}`, async () => { + const pair = createMockTransportPair(alpn); + const fin = holdFins(pair.server, "uni"); + const replies = yieldReplies(pair.server); + const origin = new Producer(); + const broadcastRoute = origin.route(Path.from("room")); + const broadcast = broadcastRoute.broadcast; + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); + const url = new URL("https://localhost/test"); + const client = new Connection({ url, quic: pair.client, version }); + const server = new Connection({ url, quic: pair.server, version, publish: origin.consume(), client: false }); + const remote = wireOf(client).consume(Path.from("room")); + const track = remote.track("video").subscribe().ordered(); + let closed = false; + void pair.server.closed.then(() => { + closed = true; + }); + try { + const first = producer.appendGroup(); + first.writeFrame(textFrame("first")); + first.finish(); + const received = await track.nextGroup(); + expect(await received?.readString()).toBe("first"); + expect(await received?.readString()).toBeUndefined(); + received?.close(); + + fin.enable(); + const last = producer.appendGroup(); + last.writeFrame(textFrame("last")); + last.finish(); + expect(await (await track.nextGroup())?.readString()).toBe("last"); + await fin.reached; + replies.enable(); + producer.finish(); + producer.close(); + const closing = server.close(); + + expect(await track.nextGroup()).toBeUndefined(); + await settle(); + expect(closed).toBe(false); + + fin.release(); + await closing; + expect(closed).toBe(true); + } finally { + fin.release(); + track.close(); + remote.close(); + client.abort(); + server.abort(); + broadcast.close(); + origin.close(); + } + }); + } + // The peer's close can overtake the final group or, where it is the track's only end, the // SUBSCRIBE FIN it sent first; the closing session reads them. for (const overtaken of hasTrackStream(version) ? ["group"] : ["group", "fin"]) { diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index b78da9de6fc..2f03f82b7c9 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1675,16 +1675,7 @@ export class Publisher { if (!(await sendEnd())) return false; continue; } - if (!(await drained())) { - // A lite-07 subscriber FINs once it read the finished track's tail, or once - // it wants nothing more. Either way the reply FIN goes out now and the groups - // still waiting for a slot are cancelled, but an opened stream may be a final - // group whose FIN is still being acknowledged, so a draining close still - // waits for it; a reader that wants none of it resets it. - if (keepsSubscribeOpen(this.version)) - void this.owe(Promise.allSettled(groups).then(() => undefined)); - return false; - } + if (!(await drained())) return false; if (pendingDrops.length > 0) continue; // Lite-07 keeps the stream open past SUBSCRIBE_FINISH: an update may still ask // for the ended track's history, and the subscriber FINs once it wants nothing @@ -1768,6 +1759,13 @@ export class Publisher { } } finally { if (!finished) unsubscribe(); + // A lite-07 subscriber FINs once it read the finished track's tail, or once it wants + // nothing more, even while a reply is still being written. Either way the groups still + // waiting for a slot are cancelled, but an opened stream may be a final group whose FIN + // is still being acknowledged, so a draining close still waits for it; a reader that + // wants none of it resets it. + if (keepsSubscribeOpen(this.version) && groups.size > 0) + void this.owe(Promise.allSettled(groups).then(() => undefined)); priority.close(); } } diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 2682dacac1b..2757491a58a 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -86,6 +86,13 @@ explains. That batch predates #7658's merge at 15:56Z, and no queue run since has failed it, so it needs no child of its own (noted 2026-11-03, `/quest-plan`). +The JS reload close-drain test lost a race in the lite-07 publisher: the +subscriber's FIN after reading the tail could land while a reply write, such as +SUBSCRIBE_FINISH, still yielded on the write budget. That exit stopped owing the +opened group streams, so a draining close aborted a final group whose FIN was +still in flight. Every exit now owes them, fixed by +[moq-dev/moq#8047](https://github.com/moq-dev/moq/pull/8047). + Added 2026-11-03 (maintainer, `/quest-plan`): five more merge-queue failures from 2026-11-03 and early 2026-11-04 UTC, one child each. @@ -106,7 +113,6 @@ Public API: none. Wire: none. ## Required - [Fallback playback](/quest/m0/test-flakes-2/ietf22-fallback.md) - a browser viewer follows a fallback within the media lane's bound without dropping its session -- [reload.test.ts's close-drain tests pass under load](/quest/m0/test-flakes-2/reload-close-drain.md) - the drain cases in `js/net`'s reload tests pass in loaded multi-file runs - [dispatch-query's terminate test survives load](/quest/m0/test-flakes-2/dispatch-query-terminate.md) - `test_terminating_the_run_stops_the_running_phase` passes reliably under host load, re-enabled once fixed - [Diagnose and re-enable the FFI datagram connection test](/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md) - diagnose the default-connection timeout and re-enable the quarantined test - [Test PulseAudio servers stay off the desktop's D-Bus](/quest/m0/test-flakes-2/pulse-dbus.md) - concurrent media runs on one desktop never collide on `org.PulseAudio1` diff --git a/quest/m0/test-flakes-2/reload-close-drain.md b/quest/m0/test-flakes-2/reload-close-drain.md deleted file mode 100644 index 0eed3b88f9f..00000000000 --- a/quest/m0/test-flakes-2/reload-close-drain.md +++ /dev/null @@ -1,100 +0,0 @@ -# [S] reload.test.ts's close-drain tests pass under load - -## Goal - -`closing the loop keeps the tokens until the session drains` in -`js/net/src/connection/reload.test.ts`, and any drain case sharing its race, -passes under load in multi-file runs, fixed at its cause under the -[epic](/quest/m0/test-flakes-2/README.md)'s rules. Done when the PR explains -the race and shows the repeated multi-file run that reproduced it now passing. - -## Plan - -Planned 2026-11-04 with the maintainer. While iterating -[moq-dev/moq#7090](https://github.com/moq-dev/moq/pull/7090), its agent saw -`closing the loop keeps the tokens until the session drains` fail once in a -four-file batch, then pass 13 runs with and without that PR's change. Its -failure output wasn't kept. The other drain cases in the file also run on a -manual clock and wait for the old session to close: `teardown by … drains the -session` shares its held-FIN fixture (`holdFins`), and `a URL change connects -the replacement while the old session drains until …` waits out a drain -deadline. Check them for the same race. - -The gated `just ci origin/main` on the JS socket branch reproduced this on -2026-11-05: `fin.closed` was already true after advancing 999 ms, where the -assertion expected false. The standalone full `js/net` run passed this case. -[moq-dev/moq#7884](https://github.com/moq-dev/moq/pull/7884) quarantines it -with every assertion retained; restore it when this quest fixes the race. - -Reproduce first: run `reload.test.ts` together with the rest of `js/net` under -CPU contention and `--rerun-each`, as the epic's earlier JS fixes did -([moq-dev/moq#7658](https://github.com/moq-dev/moq/pull/7658) fixed -`lite/connection.test.ts`'s close-drain fixture race that way). If it doesn't -reproduce in a reasonable number of runs, report that and the runs tried -rather than changing the test blind. - -Decided 2026-11-05 (maintainer, /quest-plan): keep this quest and investigate -the next captured failure with logs. Abandoning it was rejected: the agent on -[moq-dev/moq#7764](https://github.com/moq-dev/moq/pull/7764) saw -`closing the loop keeps the tokens until the session drains` fail once in a -full-suite run on 2026-11-05. It reported `expect(fin.closed).toBe(false)` -failing after a 999 ms clock advance, then three passing isolated runs. -This sighting shows the flake still occurs despite the passing reruns below; -it does not establish the cause. Keep this wait in the quest so the -evidence-only [moq-dev/moq#7933](https://github.com/moq-dev/moq/pull/7933) -can land without a speculative test or production change. - -[moq-dev/moq#7764](https://github.com/moq-dev/moq/pull/7764) also disables -the test (`test.skip`, "load-sensitive fin.closed assertion; investigate drain -scheduling before restoring") under the repo's disable-a-flaky-test policy, and -says no follow-up quest was created; this quest is that follow-up. Once that -skip lands, restoring the test is part of this quest: a skipped test cannot -meet the Goal or Done condition, and failures can only be captured by -re-enabling it (for example, with per-dial diagnostics) in a multi-file run. - -On the next captured failure, check the dial count and which client closed -first. In `closing the loop keeps the tokens until the session drains`, the -fixture overwrites shared `fin.hold` on every dial and lets any -client's close set shared `fin.closed`, but reads the track from `dials[0]`. -An extra dial could therefore hold FINs on another client or report another -client's close. The `teardown by …` fixture does not share this hazard: it -sets `fin.hold` and `fin.closed` only on the first dial -(`if (dials.length === 0)`). This is an unverified hypothesis, not an established cause; -check it alongside setup-group completion before choosing a fix. - -Public API: none. Wire: none. - -### Reproduction attempts - -2026-11-04, recorded in [moq-dev/moq#7933](https://github.com/moq-dev/moq/pull/7933): -no reproduction on the unchanged claim based on `origin/main`. - -- `quest gate bun test --cwd js/net --rerun-each 20 --only-failures` - finished with 58,645 passes, 40 skips, and 95 failures across 121 files - (476.88 seconds). All 20 reload-file rounds passed. The package - batch also exposes unrelated rerun failures in the benchmark import test - and connection fallback tests; it is not a clean package validation. -- A four-file batch (`connection/reload`, `connection/pool`, `connection/close`, - and `lite/connection`) filtered to `closing the loop|teardown by|a URL change` - with `--rerun-each 5` passed 40 tests, zero failures. A scratch preload - stubbed `performance.now()` to advance 5 ms on each budget clock read, - forcing the shared stream budgets to yield at each clock check. This - stresses budget scheduling only: it neither advances the test's manual - clock nor forces an extra dial or another client's early close, so it - does not rule out those paths. -- The earlier lite fixture fix suggests checking setup-group completion, - but neither run supports changing this fixture yet. No test or production - code changed, per the reproduce-first requirement. - -## Waiting - -- moq-dev/moq: capture the next reload drain failure with logs before choosing - a fix. If #7764's skip has landed, no full-suite failure can occur; restore - the test with per-dial diagnostics and reproduce under multi-file load instead. Check full-suite and merge-queue failure reports for the test names - above; retain the full command, commit, and assertion/stack output in the - report, plus any available session logs. The fixture currently emits no - dial/close diagnostics; those logs are not required to resume investigation. - Once captured, link that report here, remove this waiting item, and reproduce - the failure under multi-file load, capturing dial count and per-client close - state to test the hypothesis above. Passing isolated reruns alone do not - clear this condition. From 2abbf3f1a496268d5f4b635c5daf0efd14c116e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:33:38 -0800 Subject: [PATCH 3/3] style(net): format the lite-07 drain test Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/connection.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/js/net/src/lite/connection.test.ts b/js/net/src/lite/connection.test.ts index 292ba9445fe..a8454ae930b 100644 --- a/js/net/src/lite/connection.test.ts +++ b/js/net/src/lite/connection.test.ts @@ -405,7 +405,13 @@ for (const [alpn, version] of [ const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); const url = new URL("https://localhost/test"); const client = new Connection({ url, quic: pair.client, version }); - const server = new Connection({ url, quic: pair.server, version, publish: origin.consume(), client: false }); + const server = new Connection({ + url, + quic: pair.server, + version, + publish: origin.consume(), + client: false, + }); const remote = wireOf(client).consume(Path.from("room")); const track = remote.track("video").subscribe().ordered(); let closed = false;