From 9b32c0134e7f10ec108f0816ba2f912d054e90ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 00:58:04 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m0/test-flakes-2/ffi-datagram-roundtrip.md Quest-Claim: aca6caa1fe19041d From ac1d911837ff22ef3ecef92fe2800ddc2d7ff7a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 01:03:08 -0800 Subject: [PATCH 2/5] test(ffi): quarantine datagram flake pending root-cause evidence Co-Authored-By: GPT-6.1 Sol --- .../test-flakes-2/ffi-datagram-roundtrip.md | 29 +++++++++++++++++++ rs/moq-ffi/src/test.rs | 1 + 2 files changed, 30 insertions(+) diff --git a/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md b/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md index 36f4ca9c5a8..e1b8bc519db 100644 --- a/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md +++ b/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md @@ -19,3 +19,32 @@ datagram answer race planned in [moq-dev/moq#7929](https://github.com/moq-dev/moq/pull/7929) (`quest/m1/datagram-tracks/answer-race.md`), say so here and wait on that quest instead. + +Investigation 2026-11-05 on `main` at `419b799c75`: + +- The unchanged three connection cases pass locally. With temporary diagnostic + prints at `DatagramSend::send` and `Subscriber::route_datagram`, + `quest gate just rs test -p moq-ffi --lib -E 'test(datagrams_cross)' + --stress-count 100 --no-capture` passes all 300 cases. Every send returns + `Ok(())`, and all 300 received datagrams find a route. The diagnostic + prints were removed. This does not establish the cause of the CI timeout. +- All three tests await `consumer.info()` before appending. In + `DatagramServe::poll_serve`, the answer decodes before `request.accept` + exposes that info; route registration follows synchronously on that driver + before its next datagram receive pass. The answer race in + [moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951) is real, + but was not reproduced by these tests. Do not claim it caused this timeout. +- `datagrams_cross_a_default_connection` is disabled pending diagnosis, per + the repository's flaky-test rule. Its requested siblings have not failed + in this investigation and remain enabled. No retry or longer wait was added. + +## Waiting + +- moq-dev/moq: capture a failing run with send and route diagnostics to + distinguish transport loss from an unroutable datagram or another lifecycle + bug. Recommendation: rerun the ignored case under the CI workload after + [the answer-race fix](https://github.com/moq-dev/moq/pull/7951) lands, + retaining diagnostics if it fails. A passing stress run is not proof of + transport loss. Once the cause is confirmed, apply the corresponding Plan + above and re-enable the test. No workflow dispatch or host change is + authorized for this investigation. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 29f84a9d701..68ca499d3e0 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -6041,6 +6041,7 @@ async fn datagram_connection( /// A native datagram-only reader reaches a publisher on the default lite-06 path. #[tokio::test] +#[ignore = "flaky under CI load; tracked in quest/m0/test-flakes-2/ffi-datagram-roundtrip.md"] async fn datagrams_cross_a_default_connection() { let (broadcast, remote, session, _server) = datagram_connection("sensor", vec![]).await; let producer = broadcast From e2b1633c3b57435d24a4a5024b130034482420c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 01:05:46 -0800 Subject: [PATCH 3/5] test(ffi): register the quarantined test with its investigation quest Co-Authored-By: GPT-6.1 Sol --- sh/gh/test-inventory.allow | 1 + 1 file changed, 1 insertion(+) diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 6f44c275b3f..8562a330974 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -3,6 +3,7 @@ # orphan missing here, and fails on an entry that has a caller. # Flaky under load; the quest makes it reliable, then removes its `#[ignore]`. +ignore moq-ffi::test::datagrams_cross_a_default_connection /quest/m0/test-flakes-2/ffi-datagram-roundtrip.md ignore moq-gst::source::imp::session_tests::restarts_under_reconnects_interleave_on_one_pad /quest/m0/test-flakes-2/gst-reconnect-interleave.md # Need a GPU, a desktop, or a device that no CI runner has. From 3f0842283b2e356c0f79e883d46df93b81759704 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 08:50:41 -0800 Subject: [PATCH 4/5] docs(quest): record datagram quarantine decision and diagnostic wait Co-Authored-By: GPT-6.1 Sol --- .../test-flakes-2/ffi-datagram-roundtrip.md | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md b/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md index e1b8bc519db..799e382a5f2 100644 --- a/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md +++ b/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md @@ -1,24 +1,28 @@ -# [S] moq-ffi's datagram tests don't hinge on one datagram +# [S] Diagnose and re-enable the FFI datagram connection test ## Goal `datagrams_cross_a_default_connection` and its `requested_` sibling -(`rs/moq-ffi/src/test.rs`) pass reliably under CI load, without depending on -a single unreliable datagram arriving. +(`rs/moq-ffi/src/test.rs`) pass reliably under CI load after the timeout +cause is confirmed and fixed. ## Plan Decided 2026-11-05 (maintainer, follow-up from [moq-dev/moq#7914](https://github.com/moq-dev/moq/pull/7914)): ✅ m0. +Decided 2026-11-05 (maintainer, /quest-plan): ✅ Keep only +`datagrams_cross_a_default_connection` quarantined and land the quarantine in +[moq-dev/moq#7954](https://github.com/moq-dev/moq/pull/7954). Re-diagnose under +CI load after [moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951) +lands. Re-enabling now is rejected. Do not add repeated sending without +evidence of ordinary loss. + The test sends one datagram over a real connection and waits 10 s for it; -it timed out in Check on #7914 while passing 30 of 30 runs on an idle host. -Confirm the cause first. If it's loss, keep writing datagrams until the first -arrives and assert on what arrives, rather than waiting longer. If it's the -datagram answer race planned in -[moq-dev/moq#7929](https://github.com/moq-dev/moq/pull/7929) -(`quest/m1/datagram-tracks/answer-race.md`), say so here and wait on that -quest instead. +it timed out in Check on +[moq-dev/moq#7914](https://github.com/moq-dev/moq/pull/7914) while passing +30 of 30 runs on an idle host. Confirm the cause with send/route diagnostics +before choosing the fix and re-enabling the test. Investigation 2026-11-05 on `main` at `419b799c75`: @@ -40,11 +44,13 @@ Investigation 2026-11-05 on `main` at `419b799c75`: ## Waiting -- moq-dev/moq: capture a failing run with send and route diagnostics to - distinguish transport loss from an unroutable datagram or another lifecycle - bug. Recommendation: rerun the ignored case under the CI workload after - [the answer-race fix](https://github.com/moq-dev/moq/pull/7951) lands, - retaining diagnostics if it fails. A passing stress run is not proof of - transport loss. Once the cause is confirmed, apply the corresponding Plan - above and re-enable the test. No workflow dispatch or host change is - authorized for this investigation. +- moq-dev/moq: wait for + [moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951) to merge, then + capture a failing run under CI load with send/route diagnostics. Check the + PR's merged state with `gh pr view 7951 --json state,mergedAt`; once merged, + rerun the ignored case under the CI workload with diagnostics at + `DatagramSend::send` and `Subscriber::route_datagram`, retaining a failing + trace. Use it to distinguish ordinary transport loss, a route miss, or + another lifecycle bug and choose the corresponding fix before re-enabling. + Passing stress runs alone do not clear this wait or establish transport loss. + No workflow dispatch or host change is authorized for this investigation. From 2685afa56a79c0e3a36da9f97a499e1d94a1bb02 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:16:31 -0800 Subject: [PATCH 5/5] docs(quest): clarify how to resume datagram diagnostics Co-Authored-By: GPT-6.1 Sol --- quest/m0/test-flakes-2/README.md | 2 +- quest/m0/test-flakes-2/ffi-datagram-roundtrip.md | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 6c21678a2fd..4ae8c3471db 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -101,4 +101,4 @@ Public API: none. Wire: none. - [reload.test.ts's close-drain tests pass under load](/quest/m0/test-flakes-2/reload-close-drain.md) - the drain cases in `js/net`'s reload tests pass in loaded multi-file runs - [dispatch-query's terminate test survives load](/quest/m0/test-flakes-2/dispatch-query-terminate.md) - `test_terminating_the_run_stops_the_running_phase` passes reliably under host load, re-enabled once fixed - [release.test.ts never reaches the network](/quest/m0/test-flakes-2/release-test-offline.md) - the JSR release test fakes its registry lookup and passes offline -- [moq-ffi's datagram tests don't hinge on one datagram](/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md) - the default-connection datagram tests pass under CI load +- [Diagnose and re-enable the FFI datagram connection test](/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md) - diagnose the default-connection timeout and re-enable the quarantined test diff --git a/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md b/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md index 799e382a5f2..8629ae1993e 100644 --- a/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md +++ b/quest/m0/test-flakes-2/ffi-datagram-roundtrip.md @@ -48,9 +48,12 @@ Investigation 2026-11-05 on `main` at `419b799c75`: [moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951) to merge, then capture a failing run under CI load with send/route diagnostics. Check the PR's merged state with `gh pr view 7951 --json state,mergedAt`; once merged, - rerun the ignored case under the CI workload with diagnostics at - `DatagramSend::send` and `Subscriber::route_datagram`, retaining a failing - trace. Use it to distinguish ordinary transport loss, a route miss, or + run the ignored case locally under CI-like concurrent build/test load, + with diagnostics at `DatagramSend::send` and `Subscriber::route_datagram`, + retaining a failing trace. Explicitly include the quarantine with + `quest gate just rs test -p moq-ffi --lib + -E 'test(datagrams_cross_a_default_connection)' --run-ignored all --no-capture`; + normal Check and queue runs intentionally skip it. Use it to distinguish ordinary transport loss, a route miss, or another lifecycle bug and choose the corresponding fix before re-enabling. Passing stress runs alone do not clear this wait or establish transport loss. No workflow dispatch or host change is authorized for this investigation.