From dd2a488bbce3a175b509f4296c7935fbcdae5e09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 05:55:30 -0700 Subject: [PATCH 1/7] chore(quest): claim quest/m2/route-health/design.md Quest-Claim: ba7e1f38d2e5327c From 794fb34d00d79f38e9a8dc7e714c6d3227b1e04b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 06:03:30 -0700 Subject: [PATCH 2/7] test(net): measure delivery stalls that transport failure misses Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/route_change.rs | 16 +- rs/moq-net/tests/integration/route_health.rs | 282 +++++++++++++++++++ 3 files changed, 291 insertions(+), 8 deletions(-) create mode 100644 rs/moq-net/tests/integration/route_health.rs diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index da3b6acc2b0..b6362a785a2 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -47,6 +47,7 @@ mod relay_upstream_close; mod replacement; mod request_caps; mod route_change; +mod route_health; mod routing_contracts; mod segmented_frame; mod session_close; diff --git a/rs/moq-net/tests/integration/route_change.rs b/rs/moq-net/tests/integration/route_change.rs index f7a7ca97ef9..93b5be81e01 100644 --- a/rs/moq-net/tests/integration/route_change.rs +++ b/rs/moq-net/tests/integration/route_change.rs @@ -26,7 +26,7 @@ use moq_net::{Error, Hop, Timestamp, Version, announce, origin, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. -fn timed() -> moq_net::track::Info { +pub(super) fn timed() -> moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } @@ -35,19 +35,19 @@ const TEST_TIMEOUT: Duration = Duration::from_secs(30); /// Frames per group. const FRAMES: u64 = 4; -fn produce_origin(hop: u64) -> origin::Producer { +pub(super) fn produce_origin(hop: u64) -> origin::Producer { let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); support::harness::spawn(driver); producer } /// Have `to` pull everything `from` publishes. -async fn link(version: Version, from: &origin::Producer, to: &origin::Producer) -> MockPair { +pub(super) async fn link(version: Version, from: &origin::Producer, to: &origin::Producer) -> MockPair { priced_link(version, from, to, None).await } /// [`link`], with `to` pricing the link at `cost` when given. -async fn priced_link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: Option) -> MockPair { +pub(super) async fn priced_link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: Option) -> MockPair { let mut options = MockConnectOptions::new(version); options.server_publish = Some(from.consume()); options.client_subscribe = Some(to.clone()); @@ -55,7 +55,7 @@ async fn priced_link(version: Version, from: &origin::Producer, to: &origin::Pro connect_mock(options).await } -fn payload(group: u64, frame: u64) -> Vec { +pub(super) fn payload(group: u64, frame: u64) -> Vec { format!("{group}.{frame}").into_bytes() } @@ -65,7 +65,7 @@ fn tagged(name: &str, group: u64, frame: u64) -> Vec { } /// Let every task settle. Time is paused, so this returns once the runtime is idle. -async fn settle() { +pub(super) async fn settle() { moq_clock::sleep(Duration::from_millis(500)).await; } @@ -178,10 +178,10 @@ impl Topology { } /// A frame as the reader saw it: its group, then its timestamp and payload. -type Delivery = (u64, moq_net::Result<(Option, Vec)>); +pub(super) type Delivery = (u64, moq_net::Result<(Option, Vec)>); /// Read every group in full, reporting each frame as it arrives. -fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { +pub(super) fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { let (tx, rx) = mpsc::unbounded(); moq_clock::sim::spawn(async move { loop { diff --git a/rs/moq-net/tests/integration/route_health.rs b/rs/moq-net/tests/integration/route_health.rs new file mode 100644 index 00000000000..0e7d67377b8 --- /dev/null +++ b/rs/moq-net/tests/integration/route_health.rs @@ -0,0 +1,282 @@ +#![cfg(feature = "async")] + +//! Delivery stalls that neither transport failure nor reconnect recovers, measured on +//! the same-epoch diamond `P -> O -> M1/M2 -> E` that route-health standbys target. +//! +//! `E` reads through `M1`, and `M2` is an eligible same-epoch route priced to stand +//! by. These fixtures pin what `E` gets today; a route-health standby on `M2` is meant +//! to change each of them. + +use super::route_change::{Delivery, link, payload, priced_link, produce_origin, read, settle, timed}; + +use std::collections::BTreeSet; +use std::time::Duration; + +use futures::channel::mpsc; +use moq_net::{Error, Timestamp, Version, origin, track}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(60); + +/// Frames per one-second group. +const FRAMES: u64 = 10; +const STEP: Duration = Duration::from_millis(100); + +/// moq-relay's default QUIC idle timeout: how long a peer that vanished without a +/// close keeps its sessions. +const IDLE_TIMEOUT: Duration = Duration::from_secs(10); + +/// The group `M1` is killed in, after `E` read its first [`KILLED_AT`] frames. Off the +/// group boundary, so no group's age sits exactly on a budget. +const KILLED: u64 = 4; +const KILLED_AT: u64 = 3; + +/// Collect the frames the reader has delivered so far. A group that fails, such as +/// one cut as too old, just stops contributing frames. +fn drain(rx: &mut mpsc::UnboundedReceiver, into: &mut BTreeSet<(u64, u64)>) { + while let Ok((group, frame)) = rx.try_recv() { + let Ok((_, payload)) = frame else { continue }; + let payload = String::from_utf8(payload).unwrap(); + let (_, frame) = payload.split_once('.').unwrap(); + into.insert((group, frame.parse().unwrap())); + } +} + +/// `M1` is killed without a close partway through group [`KILLED`]. `O` notices at +/// the idle timeout, and `E` notices `detect` after the kill and resumes on the cold `M2`. +/// +/// Returns the groups after the killed one that `E` did not read in full. +async fn silent_kill(budget: Duration, detect: Duration) -> Vec { + let version: Version = "moq-lite-07-wip".parse().unwrap(); + let (p, o, m1, m2, e) = ( + produce_origin(1), + produce_origin(2), + produce_origin(3), + produce_origin(4), + produce_origin(5), + ); + let broadcast = p.create_broadcast("live").unwrap(); + let track = broadcast.create_track("video", timed()).unwrap(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + let p_o = link(version, &p, &o).await; + let o_m1 = link(version, &o, &m1).await; + let o_m2 = link(version, &o, &m2).await; + let m1_e = link(version, &m1, &e).await; + + let consumer = e.consume(); + consumer.routed("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); + let prefs = track::Subscription::default().with_max_delay(budget); + let mut rx = read(remote.track("video").unwrap().subscribe(prefs).await.unwrap()); + let m2_e = priced_link(version, &m2, &e, Some(1_000)).await; + settle().await; + + let kill = KILLED * FRAMES + KILLED_AT; + let e_notices = kill + (detect.as_millis() / STEP.as_millis()) as u64; + let o_notices = kill + (IDLE_TIMEOUT.as_millis() / STEP.as_millis()) as u64; + // Run three groups past whichever notices last, so the resumed feed settles. + let groups = e_notices.max(o_notices) / FRAMES + 3; + + let mut seen = BTreeSet::new(); + let mut tick = 0; + for sequence in 0..groups { + let mut group = track.append_group().unwrap(); + for frame in 0..FRAMES { + let timestamp = Timestamp::from_millis(tick * STEP.as_millis() as u64).unwrap(); + group.write_frame(timestamp, payload(sequence, frame)).unwrap(); + moq_clock::sleep(STEP).await; + tick += 1; + if tick == kill { + drain(&mut rx, &mut seen); + assert!(seen.contains(&(KILLED, KILLED_AT - 1)), "read up to the kill"); + for pair in [&o_m1, &m1_e] { + pair.client_transport.set_latency(Duration::from_secs(3600)); + } + } + if tick == o_notices { + o_m1.server.abort(Error::Cancel); + o_m1.client.abort(Error::Cancel); + } + if tick == e_notices { + m1_e.server.abort(Error::Cancel); + m1_e.client.abort(Error::Cancel); + } + } + group.finish().unwrap(); + } + settle().await; + drain(&mut rx, &mut seen); + drop((broadcast, p_o, o_m2, m2_e)); + + // The rest of the killed group is left out: whether `E` keeps reading it depends on + // whether `O`'s copy went idle before the resume, not on the detection delay. + (KILLED + 1..groups) + .filter(|sequence| (0..FRAMES).any(|frame| !seen.contains(&(*sequence, frame)))) + .collect() +} + +/// A group is stale once its successor arrived a full budget ago, so a resume at +/// `kill + detect` loses every group older than `detect - budget`: a silent kill costs +/// about `detect - budget` of media, and nothing once detection beats the budget. Today +/// detection is the idle timeout. A standby observing `M2` could detect after a frame +/// interval plus the paths' skew. +#[moq_clock::test] +async fn silent_kill_loses_what_detection_outlasts() { + let cases = [ + // (budget, detection, groups lost) + (Duration::from_secs(2), IDLE_TIMEOUT, (5..=11).collect::>()), + (Duration::from_secs(4), IDLE_TIMEOUT, (5..=9).collect()), + (Duration::from_secs(4), Duration::from_secs(6), vec![5]), + (Duration::from_secs(2), Duration::from_secs(4), vec![5]), + (Duration::from_secs(2), Duration::from_secs(3), vec![]), + (Duration::from_secs(2), Duration::from_millis(500), vec![]), + ]; + for (budget, detect, expected) in cases { + let lost = moq_clock::timeout(TEST_TIMEOUT, silent_kill(budget, detect)) + .await + .expect("timed out"); + assert_eq!(lost, expected, "budget {budget:?}, detection {detect:?}"); + } +} + +/// How `M1` stops delivering payload to `E` while every session stays up. +#[derive(Clone, Copy, Debug)] +enum Stall { + /// `O` runs out of stream credit toward `M1`, so `M1`'s copy receives no group. + Upstream, + /// `M1`'s group streams toward `E` park on stream credit `E` never grants. + Downstream, + /// `M1`'s group headers reach `E`, but no frame behind them does. + Headers, +} + +/// What a reader saw over a stall: frames read, groups that failed, the largest it +/// reported if that advanced, and whether its subscription ended. +#[derive(Debug, Default)] +struct Observed { + frames: usize, + failed: usize, + largest: Option, + ended: bool, +} + +/// Drain what a reader saw since the last call, and the newest largest it reports. +fn observe(rx: &mut mpsc::UnboundedReceiver, sub: &mut track::Subscriber) -> Observed { + let mut observed = Observed::default(); + while let Ok(delivery) = rx.try_recv() { + match delivery { + (_, Ok(_)) => observed.frames += 1, + (u64::MAX, Err(_)) => observed.ended = true, + (_, Err(_)) => observed.failed += 1, + } + } + let waiter = moq_net::kio::Waiter::noop(); + while let std::task::Poll::Ready(Some(largest)) = sub.poll_largest(&waiter) { + observed.largest = Some(largest); + } + observed +} + +/// `E` reads through `M1` while `W` reads the same track through `M2`, and `M1` stalls +/// per `stall` after group 1 while `P` keeps writing for `span`. +/// +/// Returns what `E` and `W` each observed after the stall began. +async fn payload_stall(stall: Stall, span: Duration) -> (Observed, Observed) { + let version: Version = "moq-lite-07-wip".parse().unwrap(); + let (p, o, m1, m2, e, w) = ( + produce_origin(1), + produce_origin(2), + produce_origin(3), + produce_origin(4), + produce_origin(5), + produce_origin(6), + ); + let broadcast = p.create_broadcast("live").unwrap(); + let track = broadcast.create_track("video", timed()).unwrap(); + broadcast + .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .unwrap(); + let p_o = link(version, &p, &o).await; + let o_m1 = link(version, &o, &m1).await; + let o_m2 = link(version, &o, &m2).await; + let m1_e = link(version, &m1, &e).await; + let m2_w = link(version, &m2, &w).await; + + async fn subscribe(origin: &origin::Producer) -> track::Subscriber { + let consumer = origin.consume(); + consumer.routed("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); + let prefs = track::Subscription::default().with_max_delay(Duration::from_secs(2)); + remote.track("video").unwrap().subscribe(prefs).await.unwrap() + } + let mut e_sub = subscribe(&e).await; + let mut e_rx = read(subscribe(&e).await); + let mut w_sub = subscribe(&w).await; + let mut w_rx = read(subscribe(&w).await); + let m2_e = priced_link(version, &m2, &e, Some(1_000)).await; + settle().await; + + let mut tick = 0u64; + let mut write = async |groups: u64| { + for _ in 0..groups { + let sequence = tick / FRAMES; + let mut group = track.append_group().unwrap(); + for frame in 0..FRAMES { + let timestamp = Timestamp::from_millis(tick * STEP.as_millis() as u64).unwrap(); + group.write_frame(timestamp, payload(sequence, frame)).unwrap(); + moq_clock::sleep(STEP).await; + tick += 1; + } + group.finish().unwrap(); + } + }; + write(2).await; + settle().await; + let before = observe(&mut e_rx, &mut e_sub); + assert_eq!(before.frames, 2 * FRAMES as usize, "E read through M1 before the stall"); + observe(&mut w_rx, &mut w_sub); + + match stall { + Stall::Upstream => o_m1.server_transport.withhold_unis(), + Stall::Downstream => m1_e.server_transport.withhold_unis(), + Stall::Headers => m1_e.server_transport.split_unis(), + } + write(span.as_secs()).await; + settle().await; + + let seen = (observe(&mut e_rx, &mut e_sub), observe(&mut w_rx, &mut w_sub)); + drop((broadcast, p_o, o_m1, o_m2, m1_e, m2_w, m2_e)); + seen +} + +/// `M1` stops delivering payload for 30 seconds while every session stays up, so no idle +/// timeout fires and nothing reconnects. `E` reads nothing and its subscription neither +/// ends nor moves, while `W` reads every frame through `M2`. When only `M1`'s group +/// headers arrive, `E`'s largest keeps pace with `W`'s and its groups fail as too old: +/// comparing the two routes' largest positions finds nothing wrong. +#[moq_clock::test] +async fn a_payload_stall_with_live_sessions_is_never_recovered() { + const SPAN: Duration = Duration::from_secs(30); + // The stall starts after group 1, and `P` writes one group per second through it. + let newest = 1 + SPAN.as_secs(); + for stall in [Stall::Upstream, Stall::Downstream, Stall::Headers] { + let (e, w) = moq_clock::timeout(TEST_TIMEOUT, payload_stall(stall, SPAN)) + .await + .expect("timed out"); + assert_eq!(e.frames, 0, "{stall:?}: E read through the stall"); + assert!(!e.ended, "{stall:?}: E's subscription ended"); + assert_eq!(w.frames, (SPAN.as_secs() * FRAMES) as usize, "{stall:?}: W lost media"); + assert_eq!(w.largest.map(|largest| largest.group), Some(newest), "{stall:?}"); + match stall { + Stall::Upstream | Stall::Downstream => { + assert_eq!(e.largest, None, "{stall:?}: E's largest moved"); + assert_eq!(e.failed, 0, "{stall:?}"); + } + Stall::Headers => { + assert_eq!(e.largest.map(|largest| largest.group), Some(newest), "{stall:?}"); + assert!(e.failed > 0, "{stall:?}: no group was cut"); + } + } + } +} From 3a31c550cdfead1d68620c698299631aac932c4c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 06:14:00 -0700 Subject: [PATCH 3/7] quest: record measured route-health stalls and propose the health contract Co-Authored-By: Claude Opus 5.5 --- quest/m2/route-health/design.md | 102 +++++++++---------- quest/m2/route-health/recovery.md | 9 +- rs/moq-net/tests/integration/route_change.rs | 7 +- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/quest/m2/route-health/design.md b/quest/m2/route-health/design.md index 06a22be2d84..fcf0c55641b 100644 --- a/quest/m2/route-health/design.md +++ b/quest/m2/route-health/design.md @@ -9,65 +9,57 @@ do not add speculative production code or choose arbitrary timing thresholds. ## Plan -The [line's decisions](/quest/m2/route-health/README.md) are settled. Start by -reproducing a stall that transport failure and ordinary reconnect miss; if -none exists, bring abandoning the line back to the maintainer. A killed relay -in a same-epoch diamond qualifies until the idle timeout fires -([#6062](https://github.com/moq-dev/moq/issues/6062)); measure the media it -loses and how far a standby shortens that, starting from -[#6183](https://github.com/moq-dev/moq/pull/6183)'s mock-time diamond in -`rs/moq-net/tests/integration/route_change.rs`. Preserve -application priorities, automatic health-driven switching and failback, a -payload-free steady standby, and bounded payload overlap to verify delivery. -The mechanism is entirely excluded for epochless broadcasts. +The [line's decisions](/quest/m2/route-health/README.md) are settled. The +stall the line needs is reproduced in `rs/moq-net/tests/integration/route_health.rs` +on the same-epoch diamond `P -> O -> M1/M2 -> E`: -Keep three facts separate: what the source reports, what payload the local -reader can actually use, and what that reader is demanding. Existing -`largest()` planning includes remote hints, so comparing two largest values -alone can miss stalled media with healthy control traffic. Group/frame position -also does not prove completion of an earlier frame or availability of a gap. +- A silent kill of `M1` costs about `detection - max_delay` of media, by whole + groups. At the 10s default idle timeout a 2s budget loses 7 groups and a 4s + budget 5; detection under the budget loses none. +- `M1` withholding payload while every session stays up (no upstream stream + credit, no downstream credit, or group headers without frames) starves `E` + for 30s with no error, no route change, and no recovery, while `W` reads + every frame through `M2`. With headers only, `E`'s `largest()` keeps pace with + `W`'s while every group fails as too old, so comparing largest positions + finds nothing wrong. -Work through `Subscription.meta: true` over ordinary group/frame framing. -Define when progress is emitted, frame completion, legitimately empty frames, -mode updates racing group streams, and promotion resuming at a missing byte or -frame rather than skipping payload represented only by metadata. Reconcile -with planned ranges and SUBSCRIBE_DROP. Metadata must never satisfy a media -read, populate a false payload cache, or advance a media resume cursor. +Lowering the idle timeout shrinks only the first case; nothing transport-level +recovers the second. Recovery turns these fixtures into resumes through `M2`. -Prove end-to-end demand aggregation: metadata-only readers cause no upstream -payload pull, mixed readers retain the payload demand they actually need, and -metadata observers do not recursively create alternate monitoring branches. -Standby observations must remain tied to the candidate being evaluated rather -than silently following the active route through ordinary aggregation. Different -first hops do not imply independent paths; include common upstreams and a -candidate returning through the requester. +The remaining counterexamples (quiet publishers, backpressure, exploration, +requalification, metadata cost) need the metadata mode, so they belong to the +implementation quests' tests. The proposals below are pending the maintainer; +once settled, fold them into [Metadata subscriptions](/quest/m2/route-health/meta.md), +[Track route selection](/quest/m2/route-health/selection.md), and +[Automatic recovery](/quest/m2/route-health/recovery.md), then delete this quest. -Prove bounded exploration beyond the initial next-best standby when active -and standby share a failure. One standby is a concurrency bound, not a fixed -choice: define how outstanding demand and insufficient evidence advance through -other eligible candidates without treating silence as failure, opening every -alternative, or polling quiet tracks forever. Test a healthy third route behind -two stalled candidates, all candidates quiet, and candidate churn; measure -observation cost and time to discover useful progress. Settle scheduling and -budgets from these fixtures before implementation. - -Test quiet publishers, empty groups, partial frames, out-of-order delivery, -receiver backpressure, metadata advancing while payload stalls, cached/stale -reports, epoch changes, and simultaneous failure/recovery at several hops. -Measure when evidence is sufficient to try a candidate, prove payload before -retiring usable old copies, and prevent immediate bounce back to a preferred -but still broken route. Recovery checks must be demand-driven and bounded; -reusing a fixed idle PROBE loop would recreate #5382. - -Per-track selection is the target. Rust TrackIo/resume already splices a -single track, while Front still chooses a broadcast-wide source. Share identity -and candidate handles; do not allocate a full route table/front per track. -If the authorized broadcast-level fallback is needed, record why and update -the affected tests and public behavior consistently. - -Exact framing and thresholds follow this evidence. Bring consequential API or -wire choices back to quest-plan before implementation. Publish synthetic -fixtures and retained-state/control-work measurements in existing CI lanes. +- Evidence: a stall is the active's lag behind the standby, measured on + payload the active delivered against positions the standby reported, growing + past its own healthy baseline by one frame interval of the track while the + reader demands it. The baseline matters: a standby that is normally faster + must not override application priority. Never compare largest values, and + never count source silence or a reader that stopped reading. A false trigger + costs one bounded payload trial, so evidence may be sensitive. +- Framing: a metadata group stream says so in its own header, so a stream + opened before a mode update is read under the mode it was opened in. + Metadata FRAMEs keep timestamp and declared size and carry no bytes, so a + real empty frame stays distinguishable. Promotion re-requests payload from + the reader's first missing frame; observed metadata never fills a cache or + advances a resume cursor. +- Aggregation: a relay pulls payload upstream only for payload demand, and + serves metadata from a payload copy it already holds. Only payload demand + creates a standby, so metadata observers never branch further. A standby is + pinned to its candidate's session, not routed through the front. +- Exploration: the standby slot moves to the next eligible candidate only + while the reader's demand is outstanding and neither active nor standby has + shown progress for the reader's own max delay, doubling that wait per move + until real progress resets it. Without demand nothing explores; without a + max delay, fall back to the publisher's max age, or explore never. +- Requalification: a demoted preferred route becomes the standby. Once its + metadata catches up, it trials payload from the next group boundary and wins + back the track after delivering that whole group within its old healthy + baseline. Each failed trial doubles the groups before the next one, and + metadata alone never requalifies it. ## Related diff --git a/quest/m2/route-health/recovery.md b/quest/m2/route-health/recovery.md index f32bc0591e7..618d6a3730a 100644 --- a/quest/m2/route-health/recovery.md +++ b/quest/m2/route-health/recovery.md @@ -36,11 +36,18 @@ idle connections, or arbitrary hold-down as a correctness substitute. Cancel checks with reader demand, terminal track state, epoch replacement, and session closure. A newer same-epoch connection is not automatically a healthier route. +`rs/moq-net/tests/integration/route_health.rs` pins today's stalls: a silent +kill loses about `detection - max_delay` of media, and a payload stall with live +sessions is never recovered, even while `largest()` keeps pace. Flip both into +resumes through the standby; detection under the reader's max delay loses no +group. + Mock-time tests cover all Health design counterexamples, including a preferred route whose metadata advances but media remains blocked, failed promotion, recovery that repeatedly degrades, a shared active/standby failure with a healthy third route, bounded exploration when every candidate is quiet, -multi-hop simultaneous failover, and no +a quiet publisher, receiver backpressure, stale or replayed reports, an epoch +change, multi-hop simultaneous failover, and no standby for epochless broadcasts. Run through Rust/JS interop and existing CI. Measure tracks, routes, and readers on both axes; retain the idle-session poll-count regressions and report overlap, switch counts, and memory. diff --git a/rs/moq-net/tests/integration/route_change.rs b/rs/moq-net/tests/integration/route_change.rs index 93b5be81e01..6205828be4c 100644 --- a/rs/moq-net/tests/integration/route_change.rs +++ b/rs/moq-net/tests/integration/route_change.rs @@ -47,7 +47,12 @@ pub(super) async fn link(version: Version, from: &origin::Producer, to: &origin: } /// [`link`], with `to` pricing the link at `cost` when given. -pub(super) async fn priced_link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: Option) -> MockPair { +pub(super) async fn priced_link( + version: Version, + from: &origin::Producer, + to: &origin::Producer, + cost: Option, +) -> MockPair { let mut options = MockConnectOptions::new(version); options.server_publish = Some(from.consume()); options.client_subscribe = Some(to.clone()); From 6db59aee77f654b983a9bf5cc171706062a89523 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 21:52:52 -0800 Subject: [PATCH 4/7] quest: settle the route-health contract and delete the design quest Fold the 2026-11-02 decisions into the line README, meta.md, selection.md, and recovery.md: a meta: true standby on the existing SUBSCRIBE, a switch when the active lags the standby past the reader's max_delay (2s default), per-stream mode headers with payload-less metadata FRAMEs, exploration on the same budget with doubling, the next-best standby, and no automatic return to a demoted preferred route (reversing 2026-10-13). Co-Authored-By: Claude Opus 5.5 --- quest/m2/route-health/README.md | 58 +++++++++++++++++++------ quest/m2/route-health/design.md | 67 ----------------------------- quest/m2/route-health/meta.md | 50 +++++++++++++--------- quest/m2/route-health/recovery.md | 69 +++++++++++++++++++----------- quest/m2/route-health/selection.md | 15 ++++--- 5 files changed, 127 insertions(+), 132 deletions(-) delete mode 100644 quest/m2/route-health/design.md diff --git a/quest/m2/route-health/README.md b/quest/m2/route-health/README.md index c37fddb05d8..b25a7c7a09b 100644 --- a/quest/m2/route-health/README.md +++ b/quest/m2/route-health/README.md @@ -5,7 +5,7 @@ Endpoints and relays detect stalled delivery on an actively demanded track by comparing it with a next-best route observing group/frame progress without payload. The shared routing model automatically resumes through a working -alternate and returns to a recovered preferred route. This applies only to +alternate and keeps the track there until that route fails. This applies only to broadcasts with an explicit matching epoch: epochless broadcasts cannot resume and keep their existing Restart behavior. @@ -14,8 +14,12 @@ and keep their existing Restart behavior. Decided with the maintainer, 2026-10-13: - Ranked in m2, after clustering stability, federation, and P2P. Health - design must first reproduce a stall that transport failure and ordinary - reconnect do not recover, or the line is abandoned. + design had to reproduce a stall that transport failure and ordinary + reconnect do not recover, or the line would be abandoned. It did + ([moq-dev/moq#7096](https://github.com/moq-dev/moq/pull/7096)): + `rs/moq-net/tests/integration/route_health.rs` pins a silent kill that loses + about `detection - max_delay` of media, and a payload stall on live sessions + that is never recovered while `largest()` keeps pace. - A route that dies without a close counts as such a stall until the QUIC idle timeout fires (decided 2026-10-20, from [#6062](https://github.com/moq-dev/moq/issues/6062)): a killed relay @@ -28,7 +32,7 @@ Decided with the maintainer, 2026-10-13: wins only when routes otherwise match. Stalled delivery is addressed by evidence, not a general newest-connection preference. - Observe at most one eligible same-epoch standby at a time per demanded - track, initially the next-best route, aggregating downstream readers. If + track, the next-best route, aggregating downstream readers. If active and standby provide insufficient evidence, bounded exploration must reach other eligible candidates; a shared failure must not hide a working third route. Silence alone still does not establish failure. Metadata watchers @@ -51,18 +55,44 @@ Decided with the maintainer, 2026-10-13: - Automatically let a stalled preferred route yield to a healthy lower-ranked eligible route. Relative metadata progress is evidence to test a replacement, not proof it can deliver the needed media. Brief payload overlap is allowed - during promotion and recovery verification; steady standby carries no payload. -- Automatically return to the preferred route after useful media delivery - demonstrates recovery. Metadata alone does not requalify it. Establish - evidence and anti-oscillation rules from controlled failures, not an arbitrary - delay or a blanket transport reconnect. Preserve ordinary transport failures. + while verifying a promotion; steady standby carries no payload. +- Never automatically return to a demoted preferred route (decided + 2026-11-02, reversing the 2026-10-13 decision to return after useful media + demonstrated recovery). The track stays on the working route until that + route fails, by transport failure or by the same stall evidence. Payload + trials that let a recovered route win the track back, with uncapped or + capped doubling backoff, were rejected: the track is already being + delivered, and a return would trade a working route for one that just + failed, at the cost of overlap, possible oscillation, and a backoff + schedule. Preserve ordinary transport failures. - No health standby, automatic resume, or recovery promotion for epochless broadcasts, even when the path, node, or connection looks unchanged. +Decided with the maintainer, 2026-11-02, from the measured stalls: + +- Evidence: the standby is a `Subscription.meta: true` subscription on the + existing SUBSCRIBE; "SUBSCRIBE_META" was shorthand, not a new message. + Switch when the active's delivered media timestamp lags the standby's + reported timestamp by more than the reader's max delay, while the reader + demands. That budget is what the reader already accepts losing, so a + standby that is normally a little faster never overrides application + priority, and no per-route baseline or fraction constant is needed. A + reader without a max delay uses a fixed 2s. Never compare `largest()` + values, and never count source silence or a reader that stopped reading. +- Exploration: when neither active nor standby shows progress for the + reader's max delay (or the same 2s), the standby slot moves to the next + eligible candidate, doubling the wait per move. One constant drives both + the switch and exploration; it replaces falling back to the publisher's + max age. +- Standby choice: the next-best route. Exploration already bounds the cost of + an upstream it shares with the active, so preferring the candidate with + the fewest shared hops was rejected as extra machinery. + The epic's remaining work is end-to-end validation across a multi-hop relay network and alternate sessions representing CDN/direct paths: stalled video moves while healthy audio remains, -control advances while payload stalls, the recovered primary resumes without -oscillation, and both quiet routes stay unknown rather than falsely failing. +control advances while payload stalls, a recovered preferred route does not +take the track back while the working route delivers, and both quiet routes +stay unknown rather than falsely failing. Test epochless exclusion, an active/standby shared bottleneck with a healthy third route, simultaneous relay decisions, loss/reordering, cancelled demand, and stale/replayed progress. Wire these into @@ -70,15 +100,15 @@ CI. Report actual media overlap, control traffic, wakes, switches, and memory across tracks, candidate connections, and downstream readers. Extend the [P2P harness](/quest/m2/p2p/harness.md) with real browser/native cases: metadata-only standby, stalled payload with advancing control, same-epoch -promotion with bounded overlap, verified failback, and epochless exclusion. +promotion with bounded overlap, no failback while the working route +delivers, and epochless exclusion. Add a route-health section to the routing concept page. ## Required -- [Health design](/quest/m2/route-health/design.md) - prove useful progress, aggregation, switching evidence, and bounded recovery - [Metadata subscriptions](/quest/m2/route-health/meta.md) - the current lite wip and the shared model observe frames without payload in Rust and JS - [Track route selection](/quest/m2/route-health/selection.md) - share broadcast identity/candidates while tracks choose sources independently -- [Automatic recovery](/quest/m2/route-health/recovery.md) - epoch-only promotion and evidence-based failback in endpoints and relays +- [Automatic recovery](/quest/m2/route-health/recovery.md) - epoch-only promotion in endpoints and relays, staying on the working route ## Related diff --git a/quest/m2/route-health/design.md b/quest/m2/route-health/design.md deleted file mode 100644 index fcf0c55641b..00000000000 --- a/quest/m2/route-health/design.md +++ /dev/null @@ -1,67 +0,0 @@ -# [M] Prove active/passive delivery health - -## Goal - -A reviewed design and controlled-time fixtures define metadata-only progress, -local stall evidence, promotion, and recovery for epoch-bound tracks at both -endpoints and relays. Update the implementation quests with measured choices; -do not add speculative production code or choose arbitrary timing thresholds. - -## Plan - -The [line's decisions](/quest/m2/route-health/README.md) are settled. The -stall the line needs is reproduced in `rs/moq-net/tests/integration/route_health.rs` -on the same-epoch diamond `P -> O -> M1/M2 -> E`: - -- A silent kill of `M1` costs about `detection - max_delay` of media, by whole - groups. At the 10s default idle timeout a 2s budget loses 7 groups and a 4s - budget 5; detection under the budget loses none. -- `M1` withholding payload while every session stays up (no upstream stream - credit, no downstream credit, or group headers without frames) starves `E` - for 30s with no error, no route change, and no recovery, while `W` reads - every frame through `M2`. With headers only, `E`'s `largest()` keeps pace with - `W`'s while every group fails as too old, so comparing largest positions - finds nothing wrong. - -Lowering the idle timeout shrinks only the first case; nothing transport-level -recovers the second. Recovery turns these fixtures into resumes through `M2`. - -The remaining counterexamples (quiet publishers, backpressure, exploration, -requalification, metadata cost) need the metadata mode, so they belong to the -implementation quests' tests. The proposals below are pending the maintainer; -once settled, fold them into [Metadata subscriptions](/quest/m2/route-health/meta.md), -[Track route selection](/quest/m2/route-health/selection.md), and -[Automatic recovery](/quest/m2/route-health/recovery.md), then delete this quest. - -- Evidence: a stall is the active's lag behind the standby, measured on - payload the active delivered against positions the standby reported, growing - past its own healthy baseline by one frame interval of the track while the - reader demands it. The baseline matters: a standby that is normally faster - must not override application priority. Never compare largest values, and - never count source silence or a reader that stopped reading. A false trigger - costs one bounded payload trial, so evidence may be sensitive. -- Framing: a metadata group stream says so in its own header, so a stream - opened before a mode update is read under the mode it was opened in. - Metadata FRAMEs keep timestamp and declared size and carry no bytes, so a - real empty frame stays distinguishable. Promotion re-requests payload from - the reader's first missing frame; observed metadata never fills a cache or - advances a resume cursor. -- Aggregation: a relay pulls payload upstream only for payload demand, and - serves metadata from a payload copy it already holds. Only payload demand - creates a standby, so metadata observers never branch further. A standby is - pinned to its candidate's session, not routed through the front. -- Exploration: the standby slot moves to the next eligible candidate only - while the reader's demand is outstanding and neither active nor standby has - shown progress for the reader's own max delay, doubling that wait per move - until real progress resets it. Without demand nothing explores; without a - max delay, fall back to the publisher's max age, or explore never. -- Requalification: a demoted preferred route becomes the standby. Once its - metadata catches up, it trials payload from the next group boundary and wins - back the track after delivering that whole group within its old healthy - baseline. Each failed trial doubles the groups before the next one, and - metadata alone never requalifies it. - -## Related - -- [Routes and announces](/quest/m1/cluster-routing/routes.md) - candidate identity, identity-named requests, and transit priority semantics -- [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - mode changes must preserve requested positions and demand diff --git a/quest/m2/route-health/meta.md b/quest/m2/route-health/meta.md index 1b0b37badc7..2eb85d39501 100644 --- a/quest/m2/route-health/meta.md +++ b/quest/m2/route-health/meta.md @@ -8,26 +8,34 @@ chains forward and aggregate that demand without secretly pulling full media. ## Plan -Decided 2026-10-13: prefer a `meta: true` field on `Subscription`, defaulting -to normal payload delivery. It must be a live stream: `largest()` on a -payload-less standby is only the SUBSCRIBE_OK snapshot, and a one-shot -TRACK_STATUS on announcement races the active route. It does not gate lite-07. Preserve normal group/frame progression with -payload omitted. Follow the proven mode/framing contract from Health design, -including updates on an existing subscription. Mirror the public field and -semantics across Rust, JS, and any bindings exposing Subscription. - -An explicit metadata mode distinguishes omitted payload from real empty -frames. Metadata-only state cannot satisfy media readers, masquerade as a -complete cached frame, or advance their resume cursor. A mode change to media -must request actual bytes at the required position, including previously -observed metadata positions. Handle updates and already-open group streams -without interpreting old-mode frames under the new mode. - -Aggregate full and metadata readers separately: real payload demand wins -only where needed, and each downstream metadata subscriber still receives -no payload. When only metadata demand remains, release full upstream demand. +Decided 2026-10-13: a `meta: true` field on `Subscription`, defaulting to +normal payload delivery, carried by the existing SUBSCRIBE (confirmed +2026-11-02: "SUBSCRIBE_META" was shorthand, not a new message). It must be a +live stream: `largest()` on a payload-less standby is only the SUBSCRIBE_OK +snapshot, and a one-shot TRACK_STATUS on announcement races the active route. +It does not gate lite-07. Preserve normal group/frame progression with +payload omitted. Mirror the public field and semantics across Rust, JS, and +any bindings exposing Subscription. + +Framing, decided 2026-11-02: each group stream marks its metadata mode in +its own header, so a stream opened before a mode update is read under the +mode it was opened in. Keeping the mode only on the subscription was +rejected: the receiver would race updates against already-open streams. +Metadata FRAMEs keep the timestamp and declared size and carry no bytes, so +a real empty frame stays distinguishable from omitted payload. Promotion to +payload is a SUBSCRIBE update on the same subscription, re-requesting bytes +from the reader's first missing frame, including positions it observed as +metadata. Metadata-only state never satisfies a media reader, fills a cache +as a complete frame, or advances a resume cursor. + +Aggregation, settled 2026-11-02: a relay pulls payload upstream only for +payload demand, and serves metadata from a payload copy it already holds. +Only payload demand creates a health standby, so metadata observers never +branch further. Each downstream metadata subscriber still receives no +payload. When only metadata demand remains, release full upstream demand. Remove all observation demand when the last reader leaves. Preserve ranges, -priority, drop/end semantics, and retention without one global scan per update. +priority, drop/end semantics, and retention without one global scan per +update. Reuse the live-largest model where its semantics fit, keeping remote reports separate from useful received media. Standby health uses this feature only @@ -46,9 +54,9 @@ Run `just check`, `just drafts check`, and `just test interop --all`; add the new mode to cross-language interop and existing CI. Public API: `Subscription.meta` and any corresponding binding surface. -Wire: a metadata delivery mode in the current lite wip; exact encoding is settled by Health design. +Wire: a metadata delivery mode in the current lite wip: the SUBSCRIBE field, +a mode marker in each group stream header, and payload-less metadata FRAMEs. ## Required -- [Health design](/quest/m2/route-health/design.md) - progress and mode-transition contract - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - final subscription range shape this extends diff --git a/quest/m2/route-health/recovery.md b/quest/m2/route-health/recovery.md index 618d6a3730a..8e1c5a97c2d 100644 --- a/quest/m2/route-health/recovery.md +++ b/quest/m2/route-health/recovery.md @@ -3,38 +3,55 @@ ## Goal Endpoints and relays automatically move a stalled track to a verified working -same-epoch alternate, then return to the preferred route after demonstrated -recovery, while preserving logical readers and bounded control/media overhead. +same-epoch alternate, and keep it there until that route fails, while +preserving logical readers and bounded control/media overhead. This includes a route that dies without a close: the track moves before the QUIC idle timeout declares it dead. Epochless broadcasts never enter this active/passive mechanism. ## Plan -Implement the evidence rules established by Health design. Observe the next-best -eligible same-epoch candidate using metadata-only demand, shared across downstream -readers. Apply the proven bounded exploration rule when that standby is -uninformative, so a healthy third candidate remains discoverable while only -one steady standby is observed at a time. Compare remote progress, actual -useful payload, and outstanding demand; -neither source silence nor local backpressure alone establishes a route failure. +Decided with the maintainer 2026-11-02; the line README records why. +Observe the next-best eligible same-epoch candidate with a `meta: true` +subscription, shared across downstream readers. While the reader demands the +track, switch when the active's delivered media timestamp lags the standby's +reported timestamp by more than the reader's max delay, or a fixed 2s for a +reader without one. Compare payload the active actually delivered against +timestamps the standby reported, never `largest()` values. Neither source +silence nor a reader that stopped reading or applies backpressure establishes +a route failure. At a relay the reader is the aggregated subscription, so +its budget is the largest downstream max delay the aggregate already +carries. Open question for this quest: when one aggregated reader has no +max delay, the aggregate has none either and falls to 2s, which may be +shorter than another reader's budget; decide whether such a reader should +instead leave the others' budget in force. -Health may temporarily exclude an otherwise preferred route, allowing a -lower-priority or more expensive eligible route to serve. Keep that evidence +Exploration: while demand is outstanding and neither active nor standby +shows progress for that same budget, move the standby slot to the next +eligible candidate, doubling the wait per move until real progress resets +it. A healthy third candidate stays discoverable behind an upstream the +active and standby share, while only one steady standby is observed at a +time. Without demand nothing explores. + +Health may exclude an otherwise preferred route from a track, allowing a +lower-priority or more expensive eligible route to serve it. Keep that evidence local to the affected delivery choice, not a global route-price change or publisher END. Do not close a shared connection because one track stalled. -Metadata progress justifies trying a candidate. Promote to payload at the -needed position and demonstrate useful delivery before retiring a still-usable -active copy. Brief overlap during promotion/recovery is approved; no steady -duplicate payload pull. Reuse resume's retained in-flight copies and deduplication. +Metadata progress justifies trying a candidate. Promote it to payload with +an update on the same subscription, from the reader's first missing frame, +and demonstrate useful delivery before retiring a still-usable active copy. +Brief overlap during promotion is approved; no steady duplicate payload +pull. Reuse resume's retained in-flight copies and deduplication. -Automatically requalify a recovered preferred route using real delivery, -not metadata alone, and restore ordinary priority/cost ordering. Follow the -measured anti-oscillation rule; no unbounded retry loop, permanent polling of -idle connections, or arbitrary hold-down as a correctness substitute. Cancel -checks with reader demand, terminal track state, epoch replacement, and session -closure. A newer same-epoch connection is not automatically a healthier route. +Never automatically return the track to a demoted preferred route (decided +2026-11-02, reversing the 2026-10-13 line decision to return after useful +media demonstrated recovery). The track stays on the working route until +that route fails, by transport failure or by the same stall evidence; no +payload trials, requalification backoff, or permanent polling of idle +connections. Cancel checks with reader demand, terminal track state, epoch +replacement, and session closure. A newer same-epoch connection is not +automatically a healthier route. `rs/moq-net/tests/integration/route_health.rs` pins today's stalls: a silent kill loses about `detection - max_delay` of media, and a payload stall with live @@ -42,10 +59,12 @@ sessions is never recovered, even while `largest()` keeps pace. Flip both into resumes through the standby; detection under the reader's max delay loses no group. -Mock-time tests cover all Health design counterexamples, including a preferred -route whose metadata advances but media remains blocked, failed promotion, -recovery that repeatedly degrades, a shared active/standby failure with a -healthy third route, bounded exploration when every candidate is quiet, +Mock-time tests cover a preferred route whose metadata advances but media +remains blocked, failed promotion, a standby that is normally faster by less +than the budget and so never takes the track, a recovered preferred route +that does not win the track back until the working route fails, a reader +without a max delay using the 2s default, a shared active/standby failure +with a healthy third route, bounded exploration when every candidate is quiet, a quiet publisher, receiver backpressure, stale or replayed reports, an epoch change, multi-hop simultaneous failover, and no standby for epochless broadcasts. Run through Rust/JS interop and existing CI. diff --git a/quest/m2/route-health/selection.md b/quest/m2/route-health/selection.md index 341799d109a..6c50f7c2087 100644 --- a/quest/m2/route-health/selection.md +++ b/quest/m2/route-health/selection.md @@ -17,13 +17,19 @@ health-specific reader. The maintainer permits broadcast-level fallback only if the design work demonstrates disproportionate complexity and records it. Keep candidate resolution, epoch/authorization checks and requester exclusions -shared. Select per-track copies from those candidates, with standby observations -bound to the candidate being measured. Preserve subscription aggregation and -bounded ownership when tracks, candidates, or downstream readers leave. +shared. Select per-track copies from those candidates. A standby is the +next-best eligible candidate (decided 2026-11-02), pinned to that +candidate's session rather than routed through the front, so its +observations measure the candidate itself. Only payload demand creates a +standby. Selection also lets Automatic recovery move the standby slot to the +next eligible candidate, and keeps a promoted copy active until it fails: +nothing here returns a track to a higher-ranked route on its own. Preserve +subscription aggregation and bounded ownership when tracks, candidates, or +downstream readers leave. Normal selection still uses application priority, cost, deterministic hashing, and newest only on otherwise identical routes. This quest provides selection -and lifetime machinery; Automatic recovery owns the detector and requalification. +and lifetime machinery; Automatic recovery owns the detector and exploration. Never create resumable multi-route state for an epochless broadcast. Test independent track switches, changed/missing epoch, a disappearing candidate, @@ -36,5 +42,4 @@ report any required surface change. No independent wire extension planned. ## Required -- [Health design](/quest/m2/route-health/design.md) - the proven selection boundary - [Routes and announces](/quest/m1/cluster-routing/routes.md) - shared candidate and identity model From 20b4158b5a7b2d2b0dae1c2fe03b664d5c8c6190 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 09:24:12 -0800 Subject: [PATCH 5/7] test(moq-net): pin the age cut in the headers-only payload stall Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/route_health.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/tests/integration/route_health.rs b/rs/moq-net/tests/integration/route_health.rs index 4305768f868..d39d215ef00 100644 --- a/rs/moq-net/tests/integration/route_health.rs +++ b/rs/moq-net/tests/integration/route_health.rs @@ -150,12 +150,12 @@ enum Stall { Headers, } -/// What a reader saw over a stall: frames read, groups that failed, the largest it -/// reported if that advanced, and whether its subscription ended. +/// What a reader saw over a stall: frames read, why each failed group failed, the +/// largest it reported if that advanced, and whether its subscription ended. #[derive(Debug, Default)] struct Observed { frames: usize, - failed: usize, + failed: Vec, largest: Option, ended: bool, } @@ -167,7 +167,7 @@ fn observe(rx: &mut mpsc::UnboundedReceiver, sub: &mut track::Subscrib match delivery { (_, Ok(_)) => observed.frames += 1, (u64::MAX, Err(_)) => observed.ended = true, - (_, Err(_)) => observed.failed += 1, + (_, Err(err)) => observed.failed.push(err), } } let waiter = kio::Waiter::noop(); @@ -269,11 +269,16 @@ async fn a_payload_stall_with_live_sessions_is_never_recovered() { match stall { Stall::Upstream | Stall::Downstream => { assert_eq!(e.largest, None, "{stall:?}: E's largest moved"); - assert_eq!(e.failed, 0, "{stall:?}"); + assert!(e.failed.is_empty(), "{stall:?}: {:?}", e.failed); } Stall::Headers => { assert_eq!(e.largest.map(|largest| largest.group), Some(newest), "{stall:?}"); - assert!(e.failed > 0, "{stall:?}: no group was cut"); + assert!(!e.failed.is_empty(), "{stall:?}: no group was cut"); + assert!( + e.failed.iter().all(|err| matches!(err, Error::Old)), + "{stall:?}: a group was cut for something other than age: {:?}", + e.failed + ); } } } From 43d61cd0dc1fa2d3c598a5d179e126f9bb51d20d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 09:26:09 -0800 Subject: [PATCH 6/7] quest: close the route-health gaps the review found Keep metadata-only demand live upstream after a relay demotes it, define the switch rule before the active's first frame, and state what exploration does when every candidate is quiet. Co-Authored-By: Claude Opus 5.5 --- quest/m2/route-health/README.md | 17 ++++++++++++----- quest/m2/route-health/meta.md | 16 +++++++++++----- quest/m2/route-health/recovery.md | 13 +++++++++---- rs/moq-net/tests/integration/route_health.rs | 7 +++++-- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/quest/m2/route-health/README.md b/quest/m2/route-health/README.md index b25a7c7a09b..ee0e871895e 100644 --- a/quest/m2/route-health/README.md +++ b/quest/m2/route-health/README.md @@ -77,13 +77,20 @@ Decided with the maintainer, 2026-11-02, from the measured stalls: demands. That budget is what the reader already accepts losing, so a standby that is normally a little faster never overrides application priority, and no per-route baseline or fraction constant is needed. A - reader without a max delay uses a fixed 2s. Never compare `largest()` - values, and never count source silence or a reader that stopped reading. + reader without a max delay uses a fixed 2s. Until the active delivers its + first frame, its delivered timestamp is that of the first frame the reader + still needs, as the standby reported it, so a route that never delivers is + caught on the same budget. Never compare `largest()` values, and never + count source silence or a reader that stopped reading. - Exploration: when neither active nor standby shows progress for the reader's max delay (or the same 2s), the standby slot moves to the next - eligible candidate, doubling the wait per move. One constant drives both - the switch and exploration; it replaces falling back to the publisher's - max age. + eligible candidate, doubling the wait per move. That bounds exploration: + one standby at a time, and moves that slow down geometrically. With every + candidate quiet, the slot keeps cycling through the eligible candidates at + the doubling wait and fails nothing, since silence is not failure; the + track stays on the active until real progress resets the wait or demand + ends. One constant drives both the switch and exploration; it replaces + falling back to the publisher's max age. - Standby choice: the next-best route. Exploration already bounds the cost of an upstream it shares with the active, so preferring the candidate with the fewest shared hops was rejected as extra machinery. diff --git a/quest/m2/route-health/meta.md b/quest/m2/route-health/meta.md index 2eb85d39501..7f4fb91eec3 100644 --- a/quest/m2/route-health/meta.md +++ b/quest/m2/route-health/meta.md @@ -29,10 +29,14 @@ metadata. Metadata-only state never satisfies a media reader, fills a cache as a complete frame, or advances a resume cursor. Aggregation, settled 2026-11-02: a relay pulls payload upstream only for -payload demand, and serves metadata from a payload copy it already holds. -Only payload demand creates a health standby, so metadata observers never -branch further. Each downstream metadata subscriber still receives no -payload. When only metadata demand remains, release full upstream demand. +payload demand. Metadata-only demand still travels upstream as a live +`meta: true` subscription, so group and frame headers keep arriving after +any payload copy the relay holds is exhausted; while such a copy is live, +the relay serves metadata from it. Only payload demand creates a health +standby, so metadata observers never branch further. Each downstream +metadata subscriber still receives no payload. When only metadata demand +remains, demote the upstream subscription to metadata with an update: drop +the payload demand and keep the metadata demand. Remove all observation demand when the last reader leaves. Preserve ranges, priority, drop/end semantics, and retention without one global scan per update. @@ -45,7 +49,9 @@ fetching and discarding media. Explicitly report unsupported mode; ordinary media and epochless Restart continue unchanged. An IETF counterpart is not required for this lite feature. -Tests: metadata-only multi-hop traffic has no media payload; mixed demand; +Tests: metadata-only multi-hop traffic has no media payload; a relay whose +last payload reader leaves keeps forwarding live headers to its metadata +readers; mixed demand; promotion/demotion during an open group; empty frames; late old-mode frames; resume at a metadata-observed position; cancellation; refusal on unsupported versions. Sweep tracks, readers, and relay depth for bytes, wakes and memory. diff --git a/quest/m2/route-health/recovery.md b/quest/m2/route-health/recovery.md index 8e1c5a97c2d..96d2d4563dd 100644 --- a/quest/m2/route-health/recovery.md +++ b/quest/m2/route-health/recovery.md @@ -16,7 +16,9 @@ Observe the next-best eligible same-epoch candidate with a `meta: true` subscription, shared across downstream readers. While the reader demands the track, switch when the active's delivered media timestamp lags the standby's reported timestamp by more than the reader's max delay, or a fixed 2s for a -reader without one. Compare payload the active actually delivered against +reader without one. Until the active delivers its first frame, measure it +from the standby-reported timestamp of the first frame the reader still +needs. Compare payload the active actually delivered against timestamps the standby reported, never `largest()` values. Neither source silence nor a reader that stopped reading or applies backpressure establishes a route failure. At a relay the reader is the aggregated subscription, so @@ -29,7 +31,8 @@ instead leave the others' budget in force. Exploration: while demand is outstanding and neither active nor standby shows progress for that same budget, move the standby slot to the next eligible candidate, doubling the wait per move until real progress resets -it. A healthy third candidate stays discoverable behind an upstream the +it. With every candidate quiet, the slot keeps cycling at that doubling wait +and nothing fails: the track stays on the active. A healthy third candidate stays discoverable behind an upstream the active and standby share, while only one steady standby is observed at a time. Without demand nothing explores. @@ -60,11 +63,13 @@ resumes through the standby; detection under the reader's max delay loses no group. Mock-time tests cover a preferred route whose metadata advances but media -remains blocked, failed promotion, a standby that is normally faster by less +remains blocked, an active route that never delivers its first frame, +failed promotion, a standby that is normally faster by less than the budget and so never takes the track, a recovered preferred route that does not win the track back until the working route fails, a reader without a max delay using the 2s default, a shared active/standby failure -with a healthy third route, bounded exploration when every candidate is quiet, +with a healthy third route, every candidate quiet (one standby at a time, +doubling waits, no route or track failed), a quiet publisher, receiver backpressure, stale or replayed reports, an epoch change, multi-hop simultaneous failover, and no standby for epochless broadcasts. Run through Rust/JS interop and existing CI. diff --git a/rs/moq-net/tests/integration/route_health.rs b/rs/moq-net/tests/integration/route_health.rs index d39d215ef00..63e7d8b47f5 100644 --- a/rs/moq-net/tests/integration/route_health.rs +++ b/rs/moq-net/tests/integration/route_health.rs @@ -13,7 +13,7 @@ use std::collections::BTreeSet; use std::time::Duration; use futures::channel::mpsc; -use moq_net::{Error, Timestamp, Version, origin, track}; +use moq_net::{Error, StreamError, Timestamp, Version, origin, track}; const TEST_TIMEOUT: Duration = Duration::from_secs(60); @@ -275,7 +275,10 @@ async fn a_payload_stall_with_live_sessions_is_never_recovered() { assert_eq!(e.largest.map(|largest| largest.group), Some(newest), "{stall:?}"); assert!(!e.failed.is_empty(), "{stall:?}: no group was cut"); assert!( - e.failed.iter().all(|err| matches!(err, Error::Old)), + // `E` cuts a group itself, or `M1` resets it with the same reason. + e.failed + .iter() + .all(|err| matches!(err, Error::Old | Error::Stream(StreamError::Old))), "{stall:?}: a group was cut for something other than age: {:?}", e.failed ); From 3f1ba40e01181b2d8e4f880a1dbc3a99944e1921 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Mon, 2 Nov 2026 12:23:46 -0800 Subject: [PATCH 7/7] test(net): follow route instance API in health fixtures --- rs/moq-net/tests/integration/route_health.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/tests/integration/route_health.rs b/rs/moq-net/tests/integration/route_health.rs index 63e7d8b47f5..a7a83dfdb66 100644 --- a/rs/moq-net/tests/integration/route_health.rs +++ b/rs/moq-net/tests/integration/route_health.rs @@ -57,7 +57,7 @@ async fn silent_kill(budget: Duration, detect: Duration) -> Vec { let broadcast = p.create_broadcast("live").unwrap(); let track = broadcast.create_track("video", timed()).unwrap(); broadcast - .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .announce(origin::Route::default().with_instance(moq_net::Instance::random(), 0)) .unwrap(); let p_o = link(version, &p, &o).await; let o_m1 = link(version, &o, &m1).await; @@ -194,7 +194,7 @@ async fn payload_stall(stall: Stall, span: Duration) -> (Observed, Observed) { let broadcast = p.create_broadcast("live").unwrap(); let track = broadcast.create_track("video", timed()).unwrap(); broadcast - .announce(origin::Route::default().with_epoch(moq_net::Epoch::mint())) + .announce(origin::Route::default().with_instance(moq_net::Instance::random(), 0)) .unwrap(); let p_o = link(version, &p, &o).await; let o_m1 = link(version, &o, &m1).await;