From f90244224105edb757e51872d2c64d6d889c94d5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 16:32:49 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m0/kio-waiter-records.md Quest-Claim: d8f083c4aa28855d From f9fc15c991bde8f105649c31a2ebcd21fa587703 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 17:12:04 -0800 Subject: [PATCH 2/6] fix(net): preserve errored history across metadata polling order Keep a successful metadata answer separate from the copy outcome, join both catalog handshakes before teardown, and raise waiter records to 16. Co-Authored-By: GPT-6.1 Sol --- rs/kio/benches/waiter.rs | 2 +- rs/kio/src/waiter.rs | 6 +- rs/moq-net/src/model/origin.rs | 61 ++++++++++++++++--- rs/moq-net/src/model/resume.rs | 8 +-- rs/moq-net/src/util/idle.rs | 2 +- rs/moq-net/tests/integration/quiet_catalog.rs | 51 ++++++++-------- rs/moq-stats/src/aggregate.rs | 6 +- 7 files changed, 90 insertions(+), 46 deletions(-) diff --git a/rs/kio/benches/waiter.rs b/rs/kio/benches/waiter.rs index 9fb08a8cd4b..d565ac6f2b4 100644 --- a/rs/kio/benches/waiter.rs +++ b/rs/kio/benches/waiter.rs @@ -122,7 +122,7 @@ fn bench_cancel(c: &mut Criterion) { fn bench_park_cycle(c: &mut Criterion) { let mut g = c.benchmark_group("waiter_park_cycle"); // Include broad polls that remain parked on many quiet lists. - for &lists in &[1usize, 2, 4, 8, 16] { + for &lists in &[1usize, 2, 4, 8, 16, 32] { g.bench_with_input(BenchmarkId::from_parameter(lists), &lists, |b, &n| { let cx = Context::from_waker(Waker::noop()); let mut park = Park::default(); diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index e2dc3bc5404..7d22543dfb7 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -47,8 +47,8 @@ pub struct Waiter { noop: bool, } -/// Lists a waiter remembers being parked on. A relay's busiest tasks park on up to 8. -const PARKED: usize = 8; +/// Lists a waiter remembers being parked on. +const PARKED: usize = 16; impl Waiter { /// Create a new waiter from an async [`Waker`]. @@ -82,7 +82,7 @@ impl Waiter { /// Register this waiter with a [`WaiterList`] for future notification. /// /// Delegates to [`WaiterList::register`], a no-op while the list still holds it. - /// Only the first 8 lists at once are remembered: a waiter kept across polls + /// Only the first 16 lists at once are remembered: a waiter kept across polls /// that also sits on more appends a duplicate to those extras on every call. pub fn register(&self, list: &mut WaiterList) { list.register(self); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 3753d041fbe..ad3486d8e0c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4163,15 +4163,8 @@ async fn serve_front(task: FrontTask) -> Vec { let closing = sources.get(&source).is_some_and(|s| s.is_closing()); let Some(io) = tracks.get_mut(&name) else { continue }; let Some(asked) = io.query.take() else { continue }; - // A copy that is already aborted cannot be spliced; its error is - // the source's answer for the track. - let result = match result { - Ok(info) => match asked.copy.poll_complete(&kio::Waiter::noop()) { - Poll::Ready(Err(err)) => Err(err), - _ => Ok(info), - }, - Err(err) => Err(err), - }; + // Known metadata still resolves after an abort: the copy's readers drain + // its finished history before reporting the outcome, as local readers do. // Staged only while the track has a reader: without one the machine // will not splice, and a held copy would keep the source subscribed. if let Ok(info) = &result @@ -7752,6 +7745,56 @@ mod tests { .collect() } + /// A track whose outcome is already an error still serves its finished history through + /// a front: learning the info and the outcome in one driver poll must not refuse it. + #[test] + fn an_errored_copy_serves_history_before_its_outcome() { + let now = Instant::ZERO; + let waiter = kio::Waiter::noop(); + let (producer, mut driver) = Producer::new(Config::new(origin(1))); + let server = producer + .requests_route("room", Route::default().with_hops(hops(&[10])), None) + .unwrap(); + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.reserve_track("video").unwrap(); + + let mut requesting = producer.consume().request("room", None); + driver.poll(now, &waiter).unwrap(); + let Poll::Ready(Ok(request)) = server.poll_requested_broadcast(&waiter) else { + panic!("the route receives the request"); + }; + request.serve(&broadcast); + driver.poll(now, &waiter).unwrap(); + let Poll::Ready(Ok(remote)) = requesting.poll_ok(&waiter) else { + panic!("the broadcast resolves"); + }; + let subscribing = remote.track("video").unwrap().subscribe(None); + driver.poll(now, &waiter).unwrap(); + assert!(subscribing.poll_ok(&waiter).is_pending()); + + // The query is already in flight when metadata and the abort arrive together. + let track = track.accept(track::Info::timed()); + let mut group = track.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"history".as_ref()).unwrap(); + group.finish().unwrap(); + track.abort(Error::App(7)).unwrap(); + driver.poll(now, &waiter).unwrap(); + let Poll::Ready(Ok(mut subscriber)) = subscribing.poll_ok(&waiter) else { + panic!("the errored copy's info resolves"); + }; + let Poll::Ready(Ok(Some(mut group))) = subscriber.poll_recv_group(&waiter) else { + panic!("the finished history is still readable"); + }; + let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { + panic!("the history's frame is still readable"); + }; + assert_eq!(frame.payload.as_ref(), b"history"); + assert!(matches!( + subscriber.poll_recv_group(&waiter), + Poll::Ready(Err(Error::App(7))) + )); + } + /// Ending a logical track must finish its metadata producer as well as its routes, /// without losing the source's final group or warning about an unfinished writer. #[kio::test] diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 112d10ce6d6..f987d006ca6 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -510,9 +510,9 @@ struct Copy { feeds: bool, } -// Boxed: a subscriber is twice a pending answer, and a copy holds one for its whole life. +// Boxed: a copy holds one for its whole life, and neither arm should pad every copy. enum Sub { - Pending(kio::Pending), + Pending(Box>), Ready(Box), } @@ -1280,7 +1280,7 @@ impl Reader { let wire = self.mirrored.clamped(floor, None, &resolved); self.copies.push(Copy { generation, - sub: Sub::Pending(track.subscribe(wire)), + sub: Sub::Pending(Box::new(track.subscribe(wire))), track, floor, from: self.mirrored.ranges.clone(), @@ -1994,7 +1994,7 @@ impl Reader { let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { generation, - sub: Sub::Pending(track.subscribe(wire)), + sub: Sub::Pending(Box::new(track.subscribe(wire))), floor, from: self.mirrored.ranges.clone(), resolved, diff --git a/rs/moq-net/src/util/idle.rs b/rs/moq-net/src/util/idle.rs index 75f55c57558..531c495c3f5 100644 --- a/rs/moq-net/src/util/idle.rs +++ b/rs/moq-net/src/util/idle.rs @@ -172,7 +172,7 @@ mod test { fn a_waiter_parked_past_its_records_is_retired() { let task = kio::Waiter::noop(); let mut slot: Option> = None; - for (lists, retired) in [(1, false), (9, true)] { + for (lists, retired) in [(1, false), (17, true)] { let mut idle = Idle::start(&mut slot, &task); let mut parked: Vec<_> = (0..lists).map(|_| kio::WaiterList::new()).collect(); let waiter = idle.waiter(); diff --git a/rs/moq-net/tests/integration/quiet_catalog.rs b/rs/moq-net/tests/integration/quiet_catalog.rs index 9855715de07..aebcf867c67 100644 --- a/rs/moq-net/tests/integration/quiet_catalog.rs +++ b/rs/moq-net/tests/integration/quiet_catalog.rs @@ -167,32 +167,33 @@ async fn resume_then_fresh(version: Version, in_process: bool) -> Result<(), Str let _fresh_link = link(version, &relay, &fresh).await; let resume_remote = request(&resuming).await; - // Held until the scenario returns. Dropping it would cancel the upstream - // subscription and let the fresh reader open a new one at the live edge. - let _resume = kio::clock::sim::spawn(async move { - let _sub = resume_remote - .track("catalog.json") - .unwrap() - .subscribe(track::Subscription::default().with_range(1..)) + // Drive both readers to their answers before tearing down either link. Demand reaches + // the publisher before the resume's handshake necessarily resolves. + let (_resume, _fresh) = futures::try_join!( + async { + resume_remote + .track("catalog.json") + .unwrap() + .subscribe(track::Subscription::default().with_range(1..)) + .await + .map_err(|err| format!("resume subscribe: {err}")) + }, + async { + // The fresh reader widens the existing upstream subscription. + let resumed = vec![track::Range::groups(1..)]; + kio::clock::timeout(TIMEOUT, async { + while track.subscription().map(|sub| sub.ranges).as_ref() != Some(&resumed) { + track.subscription_changed().await.unwrap(); + } + }) .await - .expect("resume subscribe"); - kio::clock::sleep(Duration::from_secs(60)).await; - }); - - // Wait for the resume to reach the publisher, so the fresh reader widens it - // rather than opening its own floorless subscription. - let resumed = vec![track::Range::groups(1..)]; - kio::clock::timeout(TIMEOUT, async { - while track.subscription().map(|sub| sub.ranges).as_ref() != Some(&resumed) { - track.subscription_changed().await.unwrap(); - } - }) - .await - .map_err(|_| "the resume never reached the publisher")?; + .map_err(|_| "the resume never reached the publisher")?; - let remote = request(if in_process { &relay } else { &fresh }).await; - read_snapshot(&remote) - .await - .map_err(|err| format!("fresh reader: {err}"))?; + let remote = request(if in_process { &relay } else { &fresh }).await; + read_snapshot(&remote) + .await + .map_err(|err| format!("fresh reader: {err}")) + } + )?; Ok(()) } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 290c807b91b..8ddc2f104df 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -698,7 +698,7 @@ enum Reader { /// Waiting for the node's listing to offer the track. Waiting, /// Awaiting the subscription handshake. - Subscribing(Pending), + Subscribing(Box>), /// Reading frames. Boxed: the snapshot consumer dwarfs the other variants, /// and one lives per node in a map. Active(Box>), @@ -1001,7 +1001,7 @@ fn advance( } }, Poll::Ready(Ok(broadcast)) => match broadcast.track(name) { - Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), + Ok(track) => node.reader = Reader::Subscribing(Box::new(track.subscribe(None))), Err(err) => { tracing::debug!(?err, name, "stats: node missing track"); return changed | node.depart(); @@ -1030,7 +1030,7 @@ fn advance( } node.offered = false; match listing.broadcast().track(name) { - Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), + Ok(track) => node.reader = Reader::Subscribing(Box::new(track.subscribe(None))), Err(err) => { tracing::debug!(?err, name, "stats: node missing track"); return changed | node.depart(); From 10a16ed790f9d720c8dd0123d8f2858b91754fa6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 17:23:33 -0800 Subject: [PATCH 3/6] chore(quest): complete the waiter records quest Co-Authored-By: GPT-6.1 Sol --- .github/relay-cpu-bisect.md | 2 +- quest/m0/README.md | 1 - quest/m0/kio-waiter-records.md | 35 ----------------------- quest/m1/release-perf/relay-cpu-bisect.md | 3 +- 4 files changed, 2 insertions(+), 39 deletions(-) delete mode 100644 quest/m0/kio-waiter-records.md diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 12a1a385969..cc8ac84d530 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -109,7 +109,7 @@ Against `main` at [619df6e1b3](https://github.com/moq-dev/moq/commit/619df6e1b3) `Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. Each time the cache aged out an old group, the eviction also landed with every reader, all of which had already taken it, and forced each one to re-run the full check. The fast path judges a group the arrival walk serves without the hole, backfill, and ledger passes, skips the subscription, fills, and answers when nothing is asked or owed, and starts each reader's eviction inbox above what it handed out. It recovers most of [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s and [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)'s cost. -kio's `WaiterList::register` is the session path's second-largest self cost: 7.2k per group. A waiter records 8 lists, and a relay session's control and serving tasks park on more, so every list past the eighth takes a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was reverted: with it, two moq-net integration tests (`errored_end::an_errored_end_keeps_its_history_across_a_hop`, `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it`) fail, and they pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records those reads stay parked, and something they depend on doesn't wake them. That is now an m0 correctness quest, [a waiter's record limit changes no behavior](../quest/m0/kio-waiter-records.md), and the per-group costs this report attributes and nothing here cut are accepted in the [release-perf epic](../quest/m1/release-perf/README.md). +kio's `WaiterList::register` was the session path's second-largest self cost: 7.2k per group. A waiter recorded 8 lists, and a relay session's control and serving tasks parked on more, so every list past the eighth took a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was initially reverted after two moq-net integration tests failed. [moq-dev/moq#7789](https://github.com/moq-dev/moq/pull/7789) fixes the front's metadata/abort ordering and the quiet-catalog test's detached handshake, then raises the record limit to 16; its paired session measurement is recorded in that PR. Forcing every idle gate to walk still reproduced both failures, so the suspected idle-gate wake dependency did not explain them. The per-group costs this report attributes and nothing here cut are accepted in the [release-perf epic](../quest/m1/release-perf/README.md). On the quiet test host (run.we4umM, one-minute load at most 4.7), chat over io\_uring went from 154.5k \[154.1-155.0] instructions per frame on `main` to 148.9k \[148.3-150.8] with the probe and window changes (-3.6%), four rounds each, every cell at 20,000 frames/s with no loss. The relay's QUIC, crypto, and syscalls are most of its cost, so the model's -23% shows as a few percent there. diff --git a/quest/m0/README.md b/quest/m0/README.md index b0090933bb2..25a135c7f0d 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -35,7 +35,6 @@ since cutting segments from first-frame pts waits on nothing in m1. - [Backport the joined-head FETCH race fix](/quest/m0/backport-fill-head-race.md) - a `release` relay stops FETCHing a group head its own SUBSCRIBE is bringing, which left live readers hanging - [CI cost and latency](/quest/m0/ci/README.md) - cloud CI runs only in a batched merge gate as one `just ci` pass per batch, expensive checks run nightly and gate `release`, caches move to R2, and runners are chosen last - [More tests under load](/quest/m0/test-flakes-2/README.md) - load-only and queue failures, one quest per flake, fixed at the cause, since each one fails a whole batch -- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - two moq-net tests stop failing when kio's waiters remember 16 lists, and the limit becomes a tuning knob worth 3.9% per delivered group - [Backport the relay and auth hardening to release](/quest/m0/backport-release-hardening.md) - `release`'s relay refuses dot-segment paths, serves its HTTP routes with main's response hardening, and withdraws advertisements a peer starves of send credit; its auth client refuses redirects - [Release's MKV importer bounds what it buffers](/quest/m0/backport-mkv-buffer.md) - on `release`, an MKV input that declares a huge element and trickles its bytes no longer grows the importer's memory without limit - [HLS export on a declared grid](/quest/m0/hls-media-grid.md) - the catalog's `hls` field and record tracks go away, and the export numbers segments on `media.join.interval` from media tracks diff --git a/quest/m0/kio-waiter-records.md b/quest/m0/kio-waiter-records.md deleted file mode 100644 index f4225dae312..00000000000 --- a/quest/m0/kio-waiter-records.md +++ /dev/null @@ -1,35 +0,0 @@ -# [M] A waiter's record limit changes no behavior - -## Goal - -Raising kio's `PARKED`, the number of lists a `kio::Waiter` remembers parking on, from 8 to 16 changes only what it costs: no moq-net test or relay behavior depends on the limit. Today two moq-net integration tests fail at 16 and pass at 8, so a missed wake, or a test leaning on one by accident, hides behind it. Find and fix that, then set the limit to 16 and take the 3.9% per delivered group the chat fan-out measured. - -## Plan - -Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): m0, as a correctness bug rather than a performance tweak, since "the fact that tests fail if we change a constant is a HUGE red flag". - -Reproduce: set `const PARKED: usize = 16;` in `rs/kio/src/waiter.rs`, then - -```bash -nix develop --command cargo nextest run -p moq-net --all-features \ - -E 'test(an_errored_end_keeps_its_history_across_a_hop) | test(a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it)' -``` - -- `errored_end::an_errored_end_keeps_its_history_across_a_hop` fails with `subscribe: Stream(App(7))` at `rs/moq-net/tests/integration/errored_end.rs:35`. -- `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it` fails with `resume subscribe: Unroutable` at `rs/moq-net/tests/integration/quiet_catalog.rs:178`. -- Both pass at 8, on [0e9b7ae6bb](https://github.com/moq-dev/moq/commit/0e9b7ae6bb4d22b50da244a3c1146acd9bb91f5a) and on `main` at the time. - -The lead: `rs/moq-net/src/util/idle.rs` keeps a read's waiter, and reports the read parked without walking it again, only while every list it parked on is recorded. Past the limit the waiter is marked lost and retired, and the read walks again. At 8 the reads in these tests overflowed and walked on every poll; at 16 they stay parked, so a change they depend on doesn't wake them, through a list not registered with this waiter or a wake that only a walk noticed. Find the read and the change, and fix the wake or the test's hidden dependency, not the limit. - -Then set the limit to 16: - -- `util::idle::test::a_waiter_parked_past_its_records_is_retired` parks on 9 lists, the old limit plus one; make it the limit plus one. -- The larger waiter makes clippy's `large_enum_variant` fire on `Sub` in `rs/moq-net/src/model/resume.rs` and `Reader` in `rs/moq-stats/src/aggregate.rs`; box their pending variants, as [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)'s reverted commits [101ed5a806](https://github.com/moq-dev/moq/commit/101ed5a806) and [1a5ceeca84](https://github.com/moq-dev/moq/commit/1a5ceeca84) did. -- Measure with `just bench-fanout "" session "64 512"`. On [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), 16 took the session scenario from 115,000 to 110,524 instructions per group at 64 viewers and from 113,200 to 108,747 at 512 (-3.9%); 12 and 32 each cost about 1-2% more than 16, since every registration searches the records. - -Public API: none expected. Wire: none. - -## Related - -- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - found this, measuring the waiter's registration cost -- [Task sets release memory after a burst](/quest/m2/performance-survey/task-capacity.md) - a task slot's `Park` holds a whole `Waiter`, which 16 records grow diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index c64a984f82d..d21e260551d 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -36,7 +36,7 @@ Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev - Accept all of them now. - Keep investigating all of them. 2. The waiter's list records: - - ✅ A follow-up at m0, as a correctness bug: [a waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md). + - ✅ A correctness follow-up at m0, completed in [moq-dev/moq#7789](https://github.com/moq-dev/moq/pull/7789): the metadata/abort and test teardown races are fixed, and waiters remember 16 lists. - Leave the limit at 8. 3. `group::Consumer` moves: - ✅ A follow-up in this epic: [a read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md). @@ -54,7 +54,6 @@ Public API: none expected. Wire: none. ## Related -- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - found here; the session's waiter registration cost waits on it - [A read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md) - found here; the session's largest remaining self cost - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - the nightly series that should have caught this growth From e6690144ca14f9ec8c48a56b4ed26c5a65a47708 Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:13:22 +0000 Subject: [PATCH 4/6] fix(ci): report cancelled phases when wait returns a lane status A cancelled lane can make the EXIT trap's wait return 143, and set -e then skips report, so a terminated run loses its timings. Tolerate the wait status so the report and cleanup always run. --- sh/dispatch.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 60494879ad8..21637ba55c8 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -534,7 +534,8 @@ lone= next=0 failed=() failed_status=0 -trap 'cancel; wait; report; rm -rf "$logs" "$changed" "$js_changed"' EXIT +# A cancelled lane can make wait return its status; the report must still run. +trap 'cancel; wait || true; report; rm -rf "$logs" "$changed" "$js_changed"' EXIT trap 'exit 130' INT trap 'exit 143' TERM From 83b3f53a4d23b8f32797b29cf21363b96113e97c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 21:29:01 -0800 Subject: [PATCH 5/6] fix(net): keep the incumbent when a replacement aborts without history An aborted copy with delivered history still splices so its readers drain it, but one that delivered nothing is a refusal again, leaving a draining incumbent serving instead of ending the track on the newcomer's error. Drop the dispatch.sh trap change, which belongs to the dispatch-query terminate quest, and record it there as a lead. Update the m2 quests that quoted the old PARKED value and waiter size. Co-Authored-By: Claude Opus 5.5 --- .../test-flakes-2/dispatch-query-terminate.md | 7 +++ quest/m2/perf/kio-waiter-costs.md | 4 +- quest/m2/performance-survey/task-capacity.md | 6 +-- rs/moq-net/src/model/origin.rs | 48 ++++++++++++++++++- sh/dispatch.sh | 3 +- 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/quest/m0/test-flakes-2/dispatch-query-terminate.md b/quest/m0/test-flakes-2/dispatch-query-terminate.md index 617fea4286d..bbcaf6ac066 100644 --- a/quest/m0/test-flakes-2/dispatch-query-terminate.md +++ b/quest/m0/test-flakes-2/dispatch-query-terminate.md @@ -15,3 +15,10 @@ that PR does not touch `sh/`. It is skipped meanwhile, per AGENTS.md's rule to disable a flaky test at once. Reproduce it under load (repeated runs beside a CPU-heavy job), find the deadline or ordering it races on, and assert on the event instead of a wait, as the epic's rules say. + +A lead to verify: pullfrog reproduced a failure 1 in 420 runs on bash 5.3.20, +where the EXIT trap's `wait` returns a cancelled lane's 143 and `set -e` skips +`report`. Its fix, `wait || true` in `sh/dispatch.sh`'s EXIT trap +([e6690144ca](https://github.com/moq-dev/moq/commit/e6690144ca14f9ec8c48a56b4ed26c5a65a47708)), +passed 420/420 but was dropped from +[moq-dev/moq#7789](https://github.com/moq-dev/moq/pull/7789) as out of scope. diff --git a/quest/m2/perf/kio-waiter-costs.md b/quest/m2/perf/kio-waiter-costs.md index afce903e351..9945f393eda 100644 --- a/quest/m2/perf/kio-waiter-costs.md +++ b/quest/m2/perf/kio-waiter-costs.md @@ -4,7 +4,7 @@ Three kio waiter costs are measured and fixed where they are real, each with before/after instruction counts: -- A long-lived waiter parked on more than `PARKED` (8) lists is no longer discarded and rebuilt on every poll, measured on a relay session driver. +- A long-lived waiter parked on more than `PARKED` (16) lists is no longer discarded and rebuilt on every poll, measured on a relay session driver. - IETF fetch serving keeps one close watch per fetch stream instead of building one per chunk. - A short-lived `kio::log` costs measurably less than about 1.3k instructions to set up and clean up, or the measurement shows why it can't. @@ -14,7 +14,7 @@ From [moq-dev/moq#7556](https://github.com/moq-dev/moq/pull/7556) (a measured no Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m2, in the perf epic, one quest for the three, since they share kio's waiter and log internals and one measurement setup. -- **`Park::hold`'s retire rule** (`rs/kio/src/waiter.rs`): `hold` replaces a waiter that is not `reusable()`, and a waiter parked on more than `PARKED = 8` lists is not. #7556 traced about 4 `Arc` per group on a session driver (`Handle::run` / `time::run`) to this: the new waiter's first registration lands on the session-close `Bridge` (moq-uring `adapter.rs` and moq-tokio `owned.rs` `poll_closed`). A relay session driver parks on many more lists, so measure there first (allocations and instructions per delivered group over `just bench-runtime` or a session bench). Then choose between raising `PARKED`, changing the retire rule, or leaving it, on the numbers. +- **`Park::hold`'s retire rule** (`rs/kio/src/waiter.rs`): `hold` replaces a waiter that is not `reusable()`, and a waiter parked on more than `PARKED = 16` lists is not ([moq-dev/moq#7789](https://github.com/moq-dev/moq/pull/7789) raised it from 8 to fix subscription ordering bugs). #7556 traced about 4 `Arc` per group on a session driver (`Handle::run` / `time::run`) to this: the new waiter's first registration lands on the session-close `Bridge` (moq-uring `adapter.rs` and moq-tokio `owned.rs` `poll_closed`). A relay session driver parks on many more lists, so measure there first (allocations and instructions per delivered group over `just bench-runtime` or a session bench). Then choose between raising `PARKED`, changing the retire rule, or leaving it, on the numbers. - **IETF fetch close watch** (`rs/moq-net/src/ietf/publisher.rs`): the fetch chunk loop pins a fresh `stream.closed()` per chunk (about lines 2387, 2450, and 2488), so each chunk builds a `kio::wait` whose waiter registers on the send stream's close `Bridge`. Hoist it to one per fetch stream. #7556 did not measure this path; a FETCH delivery bench with many chunks per stream shows the before/after. - **`kio::log` lifetime cost**: #7531 attributed about 1.3k instructions per short-lived log to its epoch pin, flush, and collect, plus segment retirement on drop. That matters to any design holding one log per stream or per group. Profile a create/append/drop microbench, and cut what doesn't need to run for a log that never grew past its first segment. diff --git a/quest/m2/performance-survey/task-capacity.md b/quest/m2/performance-survey/task-capacity.md index 05cf3dd973a..0a7b081ffbb 100644 --- a/quest/m2/performance-survey/task-capacity.md +++ b/quest/m2/performance-survey/task-capacity.md @@ -10,7 +10,7 @@ capacity is worth what it saves. [kio::Tasks](/rs/kio/src/task.rs) keeps every slot it ever grew: the slab entry (`Option>`, whose `Park` holds an inline `Waiter` of about -100 B), the per-slot waker allocation, and the wake bitset. The CPU half of +164 B), the per-slot waker allocation, and the wake bitset. The CPU half of this quest landed in [moq-dev/moq#7513](https://github.com/moq-dev/moq/pull/7513). A three-level wake index means an owner poll visits only woken words, so poll cost no longer depends on the historical peak. The memory half remains. @@ -26,8 +26,8 @@ Decided 2026-11-01 by the maintainer on The [`task_capacity` bench](/rs/kio/benches/task_capacity.rs) reports retained bytes after a burst. With an 8-byte task, the set keeps about -184 B per peak slot whatever is still live: 188,912 B after a 1,024-task -burst and 12,085,304 B after 65,536. A real task adds `size_of::()` +248 B per peak slot whatever is still live: 254,448 B after a 1,024-task +burst and 16,279,608 B after 65,536. A real task adds `size_of::()` per slot, so measure the actual consumers (`GroupServe`, the io_uring worker's `Task`, the session sets) before judging the cost. Dropping the set returns the heap to baseline, so this is retained capacity, not a leak. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index a4ba77ba76e..ac8cccf0377 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4219,8 +4219,16 @@ async fn serve_front(task: FrontTask) -> Vec { let closing = sources.get(&source).is_some_and(|s| s.is_closing()); let Some(io) = tracks.get_mut(&name) else { continue }; let Some(asked) = io.query.take() else { continue }; - // Known metadata still resolves after an abort: the copy's readers drain - // its finished history before reporting the outcome, as local readers do. + // An aborted copy with history still splices, so its readers drain that + // history before the outcome, as local readers do. One that delivered + // nothing is a refusal, which leaves a draining incumbent serving. + let result = match result { + Ok(info) => match asked.copy.poll_complete(&kio::Waiter::noop()) { + Poll::Ready(Err(err)) if asked.copy.latest_sequence().is_none() => Err(err), + _ => Ok(info), + }, + Err(err) => Err(err), + }; // Staged only while the track has a reader: without one the machine // will not splice, and a held copy would keep the source subscribed. if let Ok(info) = &result @@ -13638,6 +13646,42 @@ mod tests { .expect("open"); } + /// The newcomer answers "video" with matching metadata but aborts before delivering a + /// group, all before the front polls the answer: that is a refusal, so the incumbent + /// keeps serving rather than the track ending on the newcomer's error. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_aborted_without_history_keeps_the_incumbent() { + let (producer, _server, _upstream, old, mut subscription) = remote_front().await; + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + request.accept(crate::track::Info::timed()).abort(Error::App(7)).unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); + group.finish().unwrap(); + let mut group = next_group(&mut subscription) + .await + .expect("the incumbent keeps serving") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"still"); + + // The incumbent's clean end is the track's, not the newcomer's error. + old.finish().unwrap(); + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + } + /// The newcomer refuses "video", so the track drains the incumbent. The reader leaves /// (the park drops the incumbent's copy), a third source that also lacks the track /// takes over, and the reader returns: it gets an outcome rather than waiting on the diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 3d26f233643..c68c4bb0c40 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -534,8 +534,7 @@ lone= next=0 failed=() failed_status=0 -# A cancelled lane can make wait return its status; the report must still run. -trap 'cancel; wait || true; report; rm -rf "$logs" "$changed" "$js_changed"' EXIT +trap 'cancel; wait; report; rm -rf "$logs" "$changed" "$js_changed"' EXIT trap 'exit 130' INT trap 'exit 143' TERM From 0d5b096c7d45fdab0797ba857ca9e64b2a598d62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 21:31:07 -0800 Subject: [PATCH 6/6] style: format the incumbent test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ac8cccf0377..62a5a409208 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -13663,7 +13663,10 @@ mod tests { .await .expect("the front asked the newcomer") .expect("request"); - request.accept(crate::track::Info::timed()).abort(Error::App(7)).unwrap(); + request + .accept(crate::track::Info::timed()) + .abort(Error::App(7)) + .unwrap(); for _ in 0..10 { kio::clock::sim::yield_now().await; }