From 80574e0bf179710c60b704d5e178e85ec9480c37 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 11:49:55 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m0/test-flakes-2/ffi-concurrent-ready.md Quest-Claim: 6c2935df8ae39e0b From 4d5fdb11370565010e4de4eb8b38bd8e74dc47b1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:22:30 -0800 Subject: [PATCH 2/2] fix(ffi): the concurrent-ready test lets woken calls run between reads The old drain read until the group ended, which an unfinished group never does. ffi_caught_up returned without yielding whenever the FFI runtime answered before the test thread polled it, so under load the woken ready calls missed their turn until every frame was read, and the next read waited forever. The test now reads at most what was written and yields after each read, so it is re-enabled. Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-2/README.md | 12 ++++--- .../m0/test-flakes-2/ffi-concurrent-ready.md | 33 ------------------- rs/moq-ffi/src/test.rs | 7 ++-- sh/gh/test-inventory.allow | 1 - 4 files changed, 10 insertions(+), 43 deletions(-) delete mode 100644 quest/m0/test-flakes-2/ffi-concurrent-ready.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 2682dacac1b..be788b9515f 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -97,9 +97,14 @@ timeout, one child. Added 2026-11-05 (maintainer, `/quest-complete` follow-ups): moq-net's pool-debt flight test, the Python audio playout-delay test, and the io_uring -tests' shared locked-memory budget, one child each. On 2026-11-06, moq-ffi's -concurrent-ready test, quarantined here instead of in the unrelated -[moq-dev/moq#7981](https://github.com/moq-dev/moq/pull/7981). +tests' shared locked-memory budget, one child each. + +moq-ffi's concurrent-ready test drained its reader until the group ended, which +an unfinished group never does. Its `ffi_caught_up` returned without yielding +whenever the FFI runtime answered first, so under load the woken `ready` calls +missed their turn until every frame was read. Fixed by reading at most what was +written and yielding after each read, in +[moq-dev/moq#8053](https://github.com/moq-dev/moq/pull/8053). Public API: none. Wire: none. @@ -113,4 +118,3 @@ Public API: none. Wire: none. - [moq-net's pool-debt flight test holds its bound in parallel runs](/quest/m0/test-flakes-2/pool-debt-flight.md) - `pool_debt_sheds_a_flight_without_its_cached_group` stays under `2 * POOL` when it shares a process with moq-net's other tests - [The Python audio playout-delay test reads its frames](/quest/m0/test-flakes-2/audio-playout-delay.md) - `test_audio_consumer_reports_its_playout_delay` stops timing out on its first read in the merge queue - [io_uring tests fit the shared locked-memory budget](/quest/m0/test-flakes-2/uring-memlock.md) - concurrent moq-relay and moq-uring test runs stop exhausting the user's `RLIMIT_MEMLOCK`, fixed at the cause rather than by raising the limit -- [The FFI concurrent-ready test wakes both calls under load](/quest/m0/test-flakes-2/ffi-concurrent-ready.md) - `concurrent_ready_calls_all_wake` stops stalling its reader drain under load and is re-enabled diff --git a/quest/m0/test-flakes-2/ffi-concurrent-ready.md b/quest/m0/test-flakes-2/ffi-concurrent-ready.md deleted file mode 100644 index 814cb07794b..00000000000 --- a/quest/m0/test-flakes-2/ffi-concurrent-ready.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] The FFI concurrent-ready test wakes both calls under load - -## Goal - -`concurrent_ready_calls_all_wake` in `rs/moq-ffi/src/test.rs` passes -reliably in loaded runs, fixed at its cause under the -[epic](/quest/m0/test-flakes-2/README.md)'s rules, and its `#[ignore]` and -`sh/gh/test-inventory.allow` entry are removed. Done when the PR explains -the stall and shows the loaded run that reproduced it now passing. - -## Plan - -Added 2026-11-06 (maintainer's standing rule that flakes join this epic, -`/quest-complete`). The `/quest-iterate` agent on -[moq-dev/moq#7981](https://github.com/moq-dev/moq/pull/7981) saw it time out -in a local `quest gate just ci` while draining the reader during the two -concurrent `ready` calls; it passed on unchanged `main`, and #7981 doesn't -touch moq-ffi. Quarantined in -[moq-dev/moq#8024](https://github.com/moq-dev/moq/pull/8024) with every -assertion kept. - -Lead, unverified: the drain loop reads until both parked `ready` calls have -finished, but it checks them only between reads. If the reader takes the -last of the 25 frames before both calls finish (their wakes run on the FFI -runtime, which lags under load), the next `read_frame` waits on a group that -gets no more frames and never finishes, so it times out as "the read -stalled". Check whether that is the failure before choosing between waiting -on the calls once the window has room and fixing a lost wake in the group. - -Reproduce first: run the test repeatedly with `--run-ignored all` beside -`just check --all` load under `quest gate`, and keep the failing output. - -Public API: none. Wire: none. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index f37f22400d5..67c1952a740 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -9455,7 +9455,6 @@ async fn ready_after_abort_is_closed() { /// Two calls waiting on the same group both wake once the reader drains, though the group /// keeps only one writer waker. #[tokio::test] -#[ignore = "flaky under load: the reader can drain every frame before both ready calls finish; see quest/m0/test-flakes-2/ffi-concurrent-ready.md"] async fn concurrent_ready_calls_all_wake() { let broadcast = MoqBroadcastProducer::new().unwrap(); let track = broadcast.publish_track("tcp".into(), None).unwrap(); @@ -9484,9 +9483,8 @@ async fn concurrent_ready_calls_all_wake() { ]; assert!(waits.iter().all(|wait| !wait.is_finished()), "the window was not full"); - // Every read wakes both calls. Run them before the next read, and read no more than - // was written: draining the window faster than the runtime polls a woken call must - // not leave this loop waiting on a group nobody writes again. + // A read that makes room wakes both calls, which run when this task yields. Read no + // more than was written: the group never ends, so a read past its last frame waits forever. for _ in 0..24 { if waits.iter().all(|wait| wait.is_finished()) { break; @@ -9497,7 +9495,6 @@ async fn concurrent_ready_calls_all_wake() { .unwrap() .expect("the group ended early"); tokio::task::yield_now().await; - ffi_caught_up().await; } for wait in waits { kio::clock::timeout(TIMEOUT, wait) diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 45e9fe71542..90bd29be6e8 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -4,7 +4,6 @@ # Flaky under load; the quest makes it reliable, then removes its `#[ignore]`. ignore moq-ffi::test::datagrams_cross_a_default_connection /quest/m0/test-flakes-2/ffi-datagram-roundtrip.md -ignore moq-ffi::test::concurrent_ready_calls_all_wake /quest/m0/test-flakes-2/ffi-concurrent-ready.md ignore moq-ffi::test::requested_datagrams_cross_a_default_connection /quest/m0/test-flakes-2/ffi-datagram-roundtrip.md # Need a GPU, a desktop, or a device that no CI runner has.