From e5a2d7391c1cad9c41ed6c9ec5b0b7d2dd5efa33 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 14:39:35 -0700 Subject: [PATCH 1/5] chore: claim initial catalog routing quest Co-Authored-By: Codex From bc1dd1128df75ec38164305aae914991196c511a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 14:48:33 -0700 Subject: [PATCH 2/5] test(net): reproduce catalog acquisition across session replacement Co-Authored-By: Codex --- quest/m1/initial-catalog-routing.md | 27 +++++++++ rs/moq-net/tests/catalog_acquisition.rs | 75 +++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 rs/moq-net/tests/catalog_acquisition.rs diff --git a/quest/m1/initial-catalog-routing.md b/quest/m1/initial-catalog-routing.md index a747ec0997e..077a26bca8a 100644 --- a/quest/m1/initial-catalog-routing.md +++ b/quest/m1/initial-catalog-routing.md @@ -51,6 +51,33 @@ Update comments and affected feature docs with the fix. No new guide, public API or wire change is expected; a newly required contract needs a maintainer decision rather than an invented retry or compatibility layer. +### Evidence and remaining decision + +Controlled reproduction in `rs/moq-net/tests/catalog_acquisition.rs`: hold the +publisher's initial catalog track request, introduce a cheaper second session to +the same publisher, observe the route change, then answer the held request. +Lite-06 returns `Unroutable`; Lite-07 with the publisher's explicit epoch acquires +and reads the catalog. The old session and publisher both remain alive. No +wall-clock timing, retry or transport teardown is needed. + +This follows the current identity contract: an epochless route replacement is a +new instance, and a pending `TrackInfoFetch` checks its captured +`RequestValidity` before accepting a response. Default negotiation excludes the +work-in-progress Lite-07 wire. #5284's closed-fallback case is different: this +reproduction has no closed session. The reproduction establishes a sufficient +ordering for the reported error, not proof that all retained historical captures +followed exactly that ordering. + +A fresh pair run also failed with the explicit replacement-broadcast error during +upgrade; a subsequent traced direct-WebTransport run passed table alignment. +The pair grader fix is #5331 and does not change routing semantics. + +The maintainer must settle the newly exposed contract before a production fix: +use epoch-capable negotiation for seamless acquisition/continuation, or permit a +fresh initial acquisition on a replacement before output begins. Existing handles +must never silently cross publisher identities. Keep #5328 draft; do not paper +over the default behavior with retries, `--stitch`, or disabled fallback. + ## Related - [Validation PR #5328](https://github.com/moq-dev/moq/pull/5328) - records the process failures and successful PR CI baseline diff --git a/rs/moq-net/tests/catalog_acquisition.rs b/rs/moq-net/tests/catalog_acquisition.rs new file mode 100644 index 00000000000..d9b15183c70 --- /dev/null +++ b/rs/moq-net/tests/catalog_acquisition.rs @@ -0,0 +1,75 @@ +//! Initial catalog acquisition while the same publisher gains a second session. + +mod support; + +use moq_net::{Error, Hop, Version, origin}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +fn origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: u64) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + options.cost = Some(cost); + connect_mock(options).await +} + +async fn acquisition(version: &str) { + let version: Version = version.parse().unwrap(); + let publisher = origin(1); + let subscriber = origin(2); + let broadcast = publisher.create_broadcast("live").unwrap(); + let mut dynamic = broadcast.dynamic(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + let _old = link(version, &publisher, &subscriber, 100).await; + let remote = subscriber.consume().routed_broadcast("live").await.unwrap(); + let mut announced = subscriber.consume().follow("live").unwrap(); + assert!(matches!( + announced.next().await, + Some(moq_net::announce::Event::Start(_)) + )); + let mut pending = Box::pin(remote.track("catalog.json").unwrap().subscribe(None)); + let request = match futures::future::select(Box::pin(dynamic.requested_track()), &mut pending).await { + futures::future::Either::Left((request, _)) => request.unwrap(), + _ => panic!("catalog resolved before the publisher answered"), + }; + // The publisher is still available and the old session remains alive. Holding + // TRACK_INFO makes the replacement happen during acquisition without sleeps. + let _new = link(version, &publisher, &subscriber, 1).await; + announced.next().await.unwrap(); + let track = request.accept(None); + let mut group = track.append_group().unwrap(); + group + .write_frame(moq_net::Timestamp::ZERO, b"catalog".to_vec()) + .unwrap(); + group.finish().unwrap(); + let result = pending.await; + if version == "moq-lite-07-wip".parse().unwrap() { + let mut subscription = result.expect("same epoch must acquire"); + let mut group = subscription.recv_group().await.unwrap().unwrap(); + assert_eq!(group.read_frame().await.unwrap().unwrap().payload.as_ref(), b"catalog"); + } else { + assert!( + matches!(result, Err(Error::Unroutable)), + "epochless replacement: {:?}", + result.err() + ); + } +} + +#[moq_net_sim::test] +async fn initial_catalog_acquisition_lite06() { + acquisition("moq-lite-06").await; +} + +#[moq_net_sim::test] +async fn initial_catalog_acquisition_lite07() { + acquisition("moq-lite-07-wip").await; +} From aab6a88e43c94320d35df7056276e1114dfaefd0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 14:59:02 -0700 Subject: [PATCH 3/5] test(net): retain strict identity during catalog acquisition Complete the routing investigation with deterministic epoch-capable success and epochless refusal controls. Production negotiation and retry policy remain unchanged. Co-Authored-By: Codex --- quest/m1/README.md | 1 - quest/m1/initial-catalog-routing.md | 89 ------------------------- quest/m1/ts-pair-acquisition.md | 1 - rs/moq-net/tests/catalog_acquisition.rs | 4 +- 4 files changed, 2 insertions(+), 93 deletions(-) delete mode 100644 quest/m1/initial-catalog-routing.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 004d82441be..df84f1f11ac 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -68,7 +68,6 @@ session outcomes, and embedder refusals inside session outcomes. - [A watch and publish release ships assets()](/quest/m1/assets-release.md) - the release that lets the sites host the worklets - [Dogfood hosted worklets](/quest/m1/dogfood-assets.md) - the moq.pro dashboard hosts the worklets and calls `assets()` after the release - [TS pair acquisition](/quest/m1/ts-pair-acquisition.md) - fix the clean baseline and prove the grader's controls before #5328 enables its scheduled hard gate -- [Initial catalog routing](/quest/m1/initial-catalog-routing.md) - a usable fallback does not strand initial catalog acquisition with a spurious Unroutable - [More tests under load](/quest/m1/test-flakes-2/README.md) - the second round of load-only failures, one quest per flake, fixed at the cause - [END_OF_TRACK placement](/quest/m1/ietf-end-of-track-placement.md) - END_OF_TRACK rides the upstream's Location, and Rust's header stops claiming END_OF_GROUP - [Plan: a public model](/quest/m1/model-public-surface.md) - every crate-private model item is made public or unnecessary, so a custom publisher or subscriber builds from the public model alone diff --git a/quest/m1/initial-catalog-routing.md b/quest/m1/initial-catalog-routing.md deleted file mode 100644 index 077a26bca8a..00000000000 --- a/quest/m1/initial-catalog-routing.md +++ /dev/null @@ -1,89 +0,0 @@ -# [M] Initial catalog acquisition survives a usable transport fallback - -## Goal - -Fix the initial `catalog.json` routing failure observed by TS validation, -with a deterministic regression and an automated round trip. A publisher -remaining available on a usable path must not leave the subscriber with a -spurious `Unroutable`; genuine refusal or disappearance still fails explicitly. - -## Plan - -Decided with the maintainer on 2026-10-11: reproduce and fix the cause, not -stop at diagnosis. This is independent of the pair grader's acquisition -contract. It is not a general transport redesign or a TS-specific retry path. - -Local live, EIT and HRD runs during #5328 failed their initial `catalog.json` -track-info request shortly after selecting WebSocket fallback. The exporter -exited one with `moq: unroutable`; the publisher continued to normal EOF and -exited zero. The live grader received zero packets. Live and EIT ran -concurrently; HRD reproduced later. Do not count these as three independent -sequential reproductions. The full PR TS job passed, including EIT and HRD; -its event skipped the live and pair arms. No transport root cause is established. - -Trace the failing initial request across publication discovery, catalog -availability, route selection and any session replacement. Preserve enough -session/broadcast/request identity to distinguish a real unavailable route -from a stale handle or handover race. First check current main and #5284: -that PR preserves a pending QUIC dial after an early WebSocket close, but -similar timing is not proof that it fixes this failure. If it owns the same -cause, add the missing regression there and verify the landed fix instead of -creating competing implementations. Otherwise keep the fix focused on the -layer that loses the usable route. - -Turn the observed ordering into a controlled regression before changing -behavior. Hold or order the relevant events and mock time where needed; -do not rely on load, sleeps, repeated reruns or larger timeouts to reproduce -or hide the failure. Cover the successful fallback/acquisition path and the -adjacent terminal path so a fix cannot silently retry genuine refusal or -keep a disappeared publication alive. Preserve existing broadcast/track -identity and epoch semantics. - -Verify the affected path with the TS round-trip harness and its existing -process-status, nonempty-content and decoder assertions. A successful rerun -alone does not finish the quest. Wire every regression and any new controls -into CI, with long live coverage at least nightly; retain diagnostic logs -and failing fixtures. Keep the publisher and exporter statuses in the verdict. - -Keep the older short-duration capture regression and midstream `--stitch` -catalog-linger work separate unless reproduction establishes the same cause. -Update comments and affected feature docs with the fix. No new guide, public -API or wire change is expected; a newly required contract needs a maintainer -decision rather than an invented retry or compatibility layer. - -### Evidence and remaining decision - -Controlled reproduction in `rs/moq-net/tests/catalog_acquisition.rs`: hold the -publisher's initial catalog track request, introduce a cheaper second session to -the same publisher, observe the route change, then answer the held request. -Lite-06 returns `Unroutable`; Lite-07 with the publisher's explicit epoch acquires -and reads the catalog. The old session and publisher both remain alive. No -wall-clock timing, retry or transport teardown is needed. - -This follows the current identity contract: an epochless route replacement is a -new instance, and a pending `TrackInfoFetch` checks its captured -`RequestValidity` before accepting a response. Default negotiation excludes the -work-in-progress Lite-07 wire. #5284's closed-fallback case is different: this -reproduction has no closed session. The reproduction establishes a sufficient -ordering for the reported error, not proof that all retained historical captures -followed exactly that ordering. - -A fresh pair run also failed with the explicit replacement-broadcast error during -upgrade; a subsequent traced direct-WebTransport run passed table alignment. -The pair grader fix is #5331 and does not change routing semantics. - -The maintainer must settle the newly exposed contract before a production fix: -use epoch-capable negotiation for seamless acquisition/continuation, or permit a -fresh initial acquisition on a replacement before output begins. Existing handles -must never silently cross publisher identities. Keep #5328 draft; do not paper -over the default behavior with retries, `--stitch`, or disabled fallback. - -## Related - -- [Validation PR #5328](https://github.com/moq-dev/moq/pull/5328) - records the process failures and successful PR CI baseline -- [Closed fallback](/quest/m1/transport-upgrade/closed-fallback.md) - #5284 owns the pending QUIC dial after an early WebSocket close -- [Fallback PR #5284](https://github.com/moq-dev/moq/pull/5284) - check its actual effect on this reproduction before duplicating work -- [Transport upgrade](/quest/m1/transport-upgrade/README.md) - existing handover and epoch contracts -- [TS duration fidelity](/quest/m0/ts-duration-fidelity.md) - previously captured only a short suffix, not zero-packet initial acquisition -- [Replacement catalog linger](/quest/m1/ts-follow-catalog-bound.md) - bounds a midstream stitch, not initial discovery -- [Pair acquisition](/quest/m1/ts-pair-acquisition.md) - independent grader/mux finding diff --git a/quest/m1/ts-pair-acquisition.md b/quest/m1/ts-pair-acquisition.md index c630b83afdc..05c164b5923 100644 --- a/quest/m1/ts-pair-acquisition.md +++ b/quest/m1/ts-pair-acquisition.md @@ -57,4 +57,3 @@ contracts remain with the broader alignment planning. - [Pair gate review](https://github.com/moq-dev/moq/pull/5328#discussion_r4243103700) - a known-failing baseline would generate daily alerts - [TS roadmap #5327](https://github.com/moq-dev/moq/pull/5327) - owns the broader redundant alignment planning - [Deterministic TS legs](/quest/m2/ts-hitless.md) - packet identity requires a separate contract and proof -- [Initial catalog routing](/quest/m1/initial-catalog-routing.md) - distinct acquisition failure found by the same validation work diff --git a/rs/moq-net/tests/catalog_acquisition.rs b/rs/moq-net/tests/catalog_acquisition.rs index d9b15183c70..5eae22f0ccc 100644 --- a/rs/moq-net/tests/catalog_acquisition.rs +++ b/rs/moq-net/tests/catalog_acquisition.rs @@ -65,11 +65,11 @@ async fn acquisition(version: &str) { } #[moq_net_sim::test] -async fn initial_catalog_acquisition_lite06() { +async fn epochless_replacement_refuses_pending_catalog() { acquisition("moq-lite-06").await; } #[moq_net_sim::test] -async fn initial_catalog_acquisition_lite07() { +async fn same_epoch_replacement_acquires_pending_catalog() { acquisition("moq-lite-07-wip").await; } From e1f354f96034fcdb107199f274f2788e2c84dcf7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 13 Oct 2026 09:19:37 -0700 Subject: [PATCH 4/5] test(net): write the catalog frame untimed Tracks default to untimed on main, so a timestamped catalog frame is refused. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/catalog_acquisition.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rs/moq-net/tests/catalog_acquisition.rs b/rs/moq-net/tests/catalog_acquisition.rs index 5eae22f0ccc..7521db7e104 100644 --- a/rs/moq-net/tests/catalog_acquisition.rs +++ b/rs/moq-net/tests/catalog_acquisition.rs @@ -46,9 +46,7 @@ async fn acquisition(version: &str) { announced.next().await.unwrap(); let track = request.accept(None); let mut group = track.append_group().unwrap(); - group - .write_frame(moq_net::Timestamp::ZERO, b"catalog".to_vec()) - .unwrap(); + group.write_frame(None, b"catalog".to_vec()).unwrap(); group.finish().unwrap(); let result = pending.await; if version == "moq-lite-07-wip".parse().unwrap() { From a27d5b1c193b194eebd9ab8ab798ce478ce2c6cd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 13 Oct 2026 09:46:13 -0700 Subject: [PATCH 5/5] test(net): assert the route change the replacement causes Lite-06 restarts the advertisement and Lite-07 updates it, both onto the cheaper session, so the controls fail if the replacement never happened. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/catalog_acquisition.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/tests/catalog_acquisition.rs b/rs/moq-net/tests/catalog_acquisition.rs index 7521db7e104..b0d0c60719a 100644 --- a/rs/moq-net/tests/catalog_acquisition.rs +++ b/rs/moq-net/tests/catalog_acquisition.rs @@ -31,10 +31,10 @@ async fn acquisition(version: &str) { let _old = link(version, &publisher, &subscriber, 100).await; let remote = subscriber.consume().routed_broadcast("live").await.unwrap(); let mut announced = subscriber.consume().follow("live").unwrap(); - assert!(matches!( - announced.next().await, - Some(moq_net::announce::Event::Start(_)) - )); + let old_cost = match announced.next().await { + Some(moq_net::announce::Event::Start(announce)) => announce.route.cost, + other => panic!("expected the first route, got {other:?}"), + }; let mut pending = Box::pin(remote.track("catalog.json").unwrap().subscribe(None)); let request = match futures::future::select(Box::pin(dynamic.requested_track()), &mut pending).await { futures::future::Either::Left((request, _)) => request.unwrap(), @@ -43,13 +43,20 @@ async fn acquisition(version: &str) { // The publisher is still available and the old session remains alive. Holding // TRACK_INFO makes the replacement happen during acquisition without sleeps. let _new = link(version, &publisher, &subscriber, 1).await; - announced.next().await.unwrap(); + // An epochless replacement restarts the advertisement; a matching epoch only updates it. + let lite07 = version == "moq-lite-07-wip".parse().unwrap(); + let new_cost = match announced.next().await { + Some(moq_net::announce::Event::Update(announce)) if lite07 => announce.route.cost, + Some(moq_net::announce::Event::Restart(announce)) if !lite07 => announce.route.cost, + other => panic!("{version}: unexpected route change {other:?}"), + }; + assert!(new_cost < old_cost, "the cheaper session wins the route"); let track = request.accept(None); let mut group = track.append_group().unwrap(); group.write_frame(None, b"catalog".to_vec()).unwrap(); group.finish().unwrap(); let result = pending.await; - if version == "moq-lite-07-wip".parse().unwrap() { + if lite07 { let mut subscription = result.expect("same epoch must acquire"); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.as_ref(), b"catalog");