From 8f31a7a51b458141a776be985d4ce97e47ce412b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:52:18 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m1/simulcast-rung-disable.md Quest-Claim: a36ed79e1aa3d923 From 7b5f66f7e39d9a8b29b21c005fcf154b3ef11540 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:58:29 -0800 Subject: [PATCH 2/6] test(ffi): prove simulcast disable and resume on a shaped uplink Co-Authored-By: GPT-6.1 Sol --- .config/nextest.toml | 5 + .github/workflows/nightly.yml | 3 +- Cargo.lock | 1 + rs/justfile | 4 + rs/moq-ffi/Cargo.toml | 1 + rs/moq-ffi/src/test.rs | 2 + rs/moq-ffi/src/test/rung.rs | 232 ++++++++++++++++++++++++++++++++++ 7 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 rs/moq-ffi/src/test/rung.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 993a7ad512a..77e38d96cfd 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -44,6 +44,11 @@ test-group = "v4l2" filter = "package(moq-ffi) & test(=bench::benchmark_ffi)" slow-timeout = { period = "60s" } +# The shaped uplink spends 50s crossing BBR3's max-bandwidth windows and recovering. +[[profile.default.overrides]] +filter = "package(moq-ffi) & test(/^test::rung::/)" +slow-timeout = { period = "60s", terminate-after = 2 } + # The receive-window benchmark sweeps a grid twice, once through a bottleneck # whose rate bounds how fast a run can go, so it outlasts the hang budget. [[profile.default.overrides]] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cbec471b82d..bc9606d9bda 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -336,6 +336,7 @@ jobs: # viewer: diff-independent, since a break can arrive through the relay, # js/net, or the wire between them. # `rs quic-sim` sweeps moq-quic's simulated workloads from a fresh seed; PRs run a fixed 64. + # `rs rung-disable` proves FFI simulcast disable and recovery on a shaped BBR3 uplink. # `test resume-model` sweeps resume against its model from a fresh seed; PRs run a fixed 64. # It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases. # `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback. @@ -352,7 +353,7 @@ jobs: # `rs v4l2` streams moq-video's V4L2 capture from the kernel's vivid # virtual camera, a module load no pull request job should make. # `js worklet-matrix` loads `?worklet` output through the bundlers pinned in test/worklet. - recipe: ["gate", "rs doctest --workspace", "rs send-depth", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] + recipe: ["gate", "rs doctest --workspace", "rs send-depth", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "rs rung-disable", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] include: # Miri interprets kio's and moq-net's in-memory unsafe; too slow for the merge gate. - recipe: rs miri diff --git a/Cargo.lock b/Cargo.lock index 8254a20b071..22803881019 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5182,6 +5182,7 @@ dependencies = [ "moq-json", "moq-mux", "moq-net", + "moq-shaper", "moq-tokio", "moq-video", "pollster", diff --git a/rs/justfile b/rs/justfile index 47bb11e7027..857925863ac 100644 --- a/rs/justfile +++ b/rs/justfile @@ -343,6 +343,10 @@ quic-sim: bash sh/rs/nextest.sh cargo nextest run --locked --release -p moq-quic --lib --run-ignored only --no-capture -E 'test(/^tests::receive_ts::/)' bash sh/rs/nextest.sh cargo nextest run --locked --release -p moq-tokio --test integration --run-ignored only --no-capture --test-threads 1 -E 'test(/^receive_ts::/)' +# Prove the FFI simulcast publisher disables and resumes over a shaped BBR3 uplink. +rung-disable: + just rs test -p moq-ffi --lib --run-ignored only --no-capture -E 'test(/^test::rung::/)' + # Prove downstream futures are Send at the default recursion limit on the # flake's pinned nightly. Nightly runs it; the land gate does not. send-depth: diff --git a/rs/moq-ffi/Cargo.toml b/rs/moq-ffi/Cargo.toml index 612554c52a6..f2e5f404bde 100644 --- a/rs/moq-ffi/Cargo.toml +++ b/rs/moq-ffi/Cargo.toml @@ -68,6 +68,7 @@ uniffi = { workspace = true } url = { workspace = true } [dev-dependencies] +moq-shaper = { path = "../moq-shaper" } # The video tests encode with OpenH264, the one encoder a GPU-less host has, so # they turn it on without the `openh264` feature. moq-video = { path = "../moq-video", features = ["openh264"] } diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 169cc9bdfe0..8f1b5360bb2 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -23,6 +23,8 @@ use std::future::Future; use std::sync::Arc; use std::time::Duration; +mod rung; + /// `value` microseconds. fn us(value: u64) -> MoqTimestamp { MoqTimestamp { diff --git a/rs/moq-ffi/src/test/rung.rs b/rs/moq-ffi/src/test/rung.rs new file mode 100644 index 00000000000..455043491d9 --- /dev/null +++ b/rs/moq-ffi/src/test/rung.rs @@ -0,0 +1,232 @@ +//! The pre-encoded FFI publisher follows real BBR3 grants through a closing and reopening uplink. + +use super::*; +use crate::bandwidth::{MoqRateControl, MoqRatePolicy, MoqRateTarget}; + +const LOW: u64 = 250_000; +const HIGH: u64 = 1_000_000; +const BOUNDARY: u64 = (HIGH + 2 * LOW) / 3; +const NARROW: Duration = Duration::from_secs(8); +const BELOW_LOWEST: Duration = Duration::from_secs(20); +const RECOVER: Duration = Duration::from_secs(32); +const END: Duration = Duration::from_secs(50); +const FRAME: Duration = Duration::from_nanos(1_000_000_000 / 30); + +#[tokio::test] +#[ignore = "real sockets and 50s of shaped uplink; run nightly by `just rs rung-disable`"] +async fn shaped_uplink_disables_and_resumes() { + let server = MoqServer::new(MoqServerConfig { + bind: Some("127.0.0.1:0".into()), + tls: localhost_tls(), + ..Default::default() + }) + .unwrap(); + let addr = server.listen().await.unwrap().parse().unwrap(); + let mut setup = moq_shaper::Preset::load("cellular-step") + .unwrap() + .setup("127.0.0.1:0".parse().unwrap(), addr); + // Leave a full BBR3 max-bandwidth window at each bottleneck. The second drop + // takes even the lowest rendition below its ceiling; only probing can recover. + setup.config.up.jitter = Duration::ZERO; + setup.up.steps = [(NARROW, 500_000), (BELOW_LOWEST, 100_000), (RECOVER, 2_000_000)] + .into_iter() + .map(|(at, bits_per_second)| moq_shaper::Step { + at, + bits_per_second: Some(bits_per_second), + ..Default::default() + }) + .collect(); + let start = kio::clock::now(); + let shaper = moq_shaper::Shaper::bind(setup).await.unwrap(); + let client = MoqClient::new(MoqClientConfig { + bind: Some("127.0.0.1:0".into()), + tls: insecure_tls(), + once: true, + quic: MoqQuicConfig { + congestion_control: Some(MoqCongestionControl::Delay), + ..Default::default() + }, + websocket: MoqWebSocketConfig { + enabled: Some(false), + ..Default::default() + }, + ..Default::default() + }) + .unwrap(); + let (publisher, receiver) = tokio::join!(client.connect(format!("https://{}/", shaper.addr())), async { + server.accept().await.unwrap().unwrap().accept(None, None).await + }); + let publisher = publisher.unwrap(); + let receiver = receiver.unwrap(); + let broadcast = create_announced(&publisher.publish(), "ladder"); + let allocator = publisher.allocator(); + let mut tracks = Vec::new(); + let mut reservations = Vec::new(); + let mut controls = Vec::new(); + for (name, max, lower) in [("low", LOW, None), ("high", HIGH, Some(LOW))] { + let track = MoqMediaTrackProducer::video( + &broadcast, + MoqMediaTarget::Named { + name: Some(name.into()), + }, + MoqVideoInit { + hint: Some(MoqVideoHint { + bitrate: Some(max), + ..Default::default() + }), + lower: tracks.first().cloned(), + ..video_init(MoqVideoFormat::Avc3, h264_init()) + }, + ) + .unwrap(); + reservations.push(allocator.reserve(&track.demand().unwrap(), max)); + controls.push(MoqRateControl::new(MoqRatePolicy { + max_bps: max, + lower_bps: lower, + })); + tracks.push(track); + } + let remote = await_announced(&receiver.consume(), "ladder").await; + let catalog = MoqMediaCatalogConsumer::subscribe(&remote).await.unwrap(); + let first = catalog.next().await.unwrap().expect("catalog ended"); + let low = remote.subscribe_track("low".into(), None).await.unwrap(); + let high = MoqMediaTrackConsumer::subscribe( + &remote, + MoqMediaTrackConfig { + rendition: MoqRendition::Video { + id: "high".into(), + config: first.video["high"].clone(), + }, + subscription: None, + }, + ) + .await + .unwrap(); + let mut ticker = kio::clock::interval(FRAME); + ticker.set_missed_tick_behavior(kio::clock::MissedTickBehavior::Skip); + let mut remote_enabled = Some(first.video["high"].enabled); + let mut initial = false; + let mut disabled = false; + let mut below_lowest = false; + let mut resumed = false; + let mut received = [0u64; 2]; + let mut resumed_frame = false; + let mut stopped_at = None; + let mut resumed_at = None; + let mut previous_second = None; + loop { + tokio::select! { + _ = kio::clock::sleep_until(start + END) => break, + _ = ticker.tick() => { + let elapsed = start.elapsed(); + let timestamp = elapsed.as_micros() as u64; + for (i, ((track, reservation), control)) in tracks.iter().zip(&reservations).zip(&controls).enumerate() { + let was_disabled = control.target() == MoqRateTarget::Disabled; + if let Some(target) = control.update(reservation) { + println!("{elapsed:?} rung {i}: {target:?}, grant {:?}", reservation.rate()); + let enabled = matches!(target, MoqRateTarget::Enabled { .. }); + track.set_enabled(enabled).unwrap(); + if !enabled { + assert_eq!(i, 1, "the lowest rung must never disable"); + assert!(reservation.rate().unwrap() < BOUNDARY); + track.discontinuity().unwrap(); + if elapsed >= NARROW && stopped_at.is_none() { + stopped_at = Some(timestamp); + } + } else if i == 1 && was_disabled && stopped_at.is_some() { + assert!(elapsed >= RECOVER, "resumed before the path reopened"); + assert!(reservation.rate().unwrap() >= BOUNDARY + BOUNDARY / 20, + "resumed without clearing the hysteresis"); + resumed_at.get_or_insert(timestamp); + } + } + if let MoqRateTarget::Enabled { bitrate_bps } = control.target() { + // An IDR access unit padded to the target's frame budget. This exercises + // framing and transport, without making a codec's rate accuracy the test. + let mut payload = h264_init(); + payload.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]); + payload.resize((bitrate_bps / 8 / 30) as usize, 0xff); + track.write_frame(MoqFrame { payload, timestamp: Some(us(timestamp)) }).unwrap(); + } + } + if elapsed < NARROW && remote_enabled == Some(true) && received.iter().all(|count| *count > 0) { + initial = true; + } + if elapsed >= NARROW && elapsed < BELOW_LOWEST && remote_enabled == Some(false) { + disabled = true; + } + if elapsed >= BELOW_LOWEST && elapsed < RECOVER && reservations[0].rate().is_some_and(|rate| rate < LOW) { + assert_eq!(controls[1].target(), MoqRateTarget::Disabled); + assert_eq!(remote_enabled, Some(false)); + below_lowest = true; + } + if elapsed >= RECOVER && resumed_at.is_some() && remote_enabled == Some(true) { + resumed = true; + } + if previous_second != Some(elapsed.as_secs()) { + previous_second = Some(elapsed.as_secs()); + println!("{elapsed:?}: estimate {:?}, grants {:?}, received {received:?}, remote {remote_enabled:?}", + publisher.stats().estimated_send_rate_bps, + reservations.iter().map(|r| r.rate()).collect::>()); + } + }, + update = catalog.next() => { + let update = update.unwrap().expect("catalog ended"); + assert!(update.video["low"].enabled); + assert_eq!(update.video["low"].bitrate, Some(LOW)); + assert_eq!(update.video["high"].bitrate, Some(HIGH)); + remote_enabled = Some(update.video["high"].enabled); + println!("{:?}: remote enabled {remote_enabled:?}", start.elapsed()); + }, + frame = low.read_frame() => { + frame.unwrap().expect("lowest track ended"); + received[0] += 1; + }, + frame = high.next() => { + let read = frame.unwrap().expect("top track ended"); + // Frames queued before disabling may still arrive, but none may have + // been produced in the disabled interval. Decode the container so its + // discontinuity marker isn't mistaken for media. + if let MoqMediaRead::Frame { frame } = read { + let timestamp = micros(frame.timestamp); + if let Some(stopped) = stopped_at { + assert!(timestamp < stopped || resumed_at.is_some_and(|at| timestamp >= at), + "media was produced while disabled: {timestamp}"); + } + if resumed_at.is_some_and(|at| timestamp >= at) { + assert!(frame.keyframe, "the resumed rendition must start with a keyframe"); + resumed_frame = true; + } + received[1] += 1; + } + }, + } + } + let stats = shaper.verify().unwrap(); + println!("shaper: {stats}"); + assert!(stats.up.overflowed > 0, "the finite queue never overflowed"); + assert!( + initial, + "both renditions never delivered before the bottleneck narrowed" + ); + assert!( + disabled, + "the top rendition never advertised disabled on the narrow uplink" + ); + assert!( + below_lowest, + "the grant never fell below the lowest rung while the top stayed disabled" + ); + assert!( + resumed && resumed_frame, + "the top rendition never advertised and delivered its resume" + ); + for track in tracks { + track.finish().unwrap(); + } + broadcast.close().unwrap(); + publisher.cancel(0); + receiver.cancel(0); + client.cancel(); + server.cancel(); +} From 11600bebeb5bdb2e19f1d4f6f97efaf42fa6fdf3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:38:42 -0800 Subject: [PATCH 3/6] test(ffi): drain both renditions and wait for observed recovery Co-Authored-By: GPT-6.1 Sol --- .config/nextest.toml | 4 +-- quest/m1/simulcast-rung-disable.md | 35 ++++++++++++---------- rs/moq-ffi/src/test/rung.rs | 48 +++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 77e38d96cfd..8eb34811253 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -44,10 +44,10 @@ test-group = "v4l2" filter = "package(moq-ffi) & test(=bench::benchmark_ffi)" slow-timeout = { period = "60s" } -# The shaped uplink spends 50s crossing BBR3's max-bandwidth windows and recovering. +# The shaped uplink crosses BBR3's max-bandwidth windows before waiting for recovery. [[profile.default.overrides]] filter = "package(moq-ffi) & test(/^test::rung::/)" -slow-timeout = { period = "60s", terminate-after = 2 } +slow-timeout = { period = "120s", terminate-after = 2 } # The receive-window benchmark sweeps a grid twice, once through a bottleneck # whose rate bounds how fast a run can go, so it outlasts the hang budget. diff --git a/quest/m1/simulcast-rung-disable.md b/quest/m1/simulcast-rung-disable.md index 4d7a25fad3d..b3278c103ac 100644 --- a/quest/m1/simulcast-rung-disable.md +++ b/quest/m1/simulcast-rung-disable.md @@ -1,4 +1,4 @@ -# [M] A shaped uplink proves native rung disable and resume +# [S] Finish validation of the shaped-uplink rung proof ## Goal @@ -11,20 +11,25 @@ the lowest rung. It runs nightly. The JS publisher is ## Plan -The rule and every native publisher landed: `moq_mux::rate` in -[moq-dev/moq#6781](https://github.com/moq-dev/moq/pull/6781) (`Policy::lower` -sets the band boundary `(max + 2 * lower) / 3`; a rung disables only while a -session probes for its reservation), the FFI and C video producers there too, -and OBS through moq-ffi's `MoqRateControl`, `MoqVideoInit.lower`, and -`MoqMediaTrackProducer::set_enabled`, which a pre-encoded publisher follows -the same way. - -- Shape it with a `moq-shaper` profile whose rate steps down below the top - rung and back up (like the `cellular-step` built-in), behind a deep but - finite queue whose overflow drops packets. Assert the disable, the catalog - `enabled` flips, and the resume, not exact timings. -- Run it nightly with an `#[ignore]` test and a recipe in `rs/justfile`, - beside `quic-sim`. +The native rule and publishers have landed. The remaining transport proof is +implemented in `rs/moq-ffi/src/test/rung.rs`, using the pre-encoded FFI path: +`MoqRateControl`, reservations, `MoqVideoInit.lower`, and +`MoqMediaTrackProducer::set_enabled`. `just rs rung-disable` runs the ignored +socket test and is wired into Nightly beside `quic-sim`. + +The shaped run passed, including queue overflow, remote catalog flips, no +media timestamps in the disabled interval, a below-lowest grant, and a resumed +keyframe. The final phase ends on observed recovery, without imposing an exact +recovery time. Existing mock-grant tests cover the built-in encoder's teardown +and reopen. No production controller change is part of this proof. + +Finish local validation and `/quest-review`, then remove this quest once the +full gate passes and the PR can be marked ready. Keep the nightly continuation +changes with [moq-dev/moq#7667](https://github.com/moq-dev/moq/pull/7667). + +## Waiting + +- moq-dev/moq: the full local gate must pass after the existing C/C++ pkg-config fix in [moq-dev/moq#7654](https://github.com/moq-dev/moq/pull/7654) and default Gradle discovery fix in [moq-dev/moq#7632](https://github.com/moq-dev/moq/pull/7632), on a Nix builder that permits user namespaces. The prior gate also failed Dart's native-build freshness assertion; recheck it with the FFI build work in [moq-dev/moq#7666](https://github.com/moq-dev/moq/pull/7666), without importing those independent changes here. ## Related diff --git a/rs/moq-ffi/src/test/rung.rs b/rs/moq-ffi/src/test/rung.rs index 455043491d9..be74fffd63f 100644 --- a/rs/moq-ffi/src/test/rung.rs +++ b/rs/moq-ffi/src/test/rung.rs @@ -9,11 +9,11 @@ const BOUNDARY: u64 = (HIGH + 2 * LOW) / 3; const NARROW: Duration = Duration::from_secs(8); const BELOW_LOWEST: Duration = Duration::from_secs(20); const RECOVER: Duration = Duration::from_secs(32); -const END: Duration = Duration::from_secs(50); +const END: Duration = Duration::from_secs(110); const FRAME: Duration = Duration::from_nanos(1_000_000_000 / 30); #[tokio::test] -#[ignore = "real sockets and 50s of shaped uplink; run nightly by `just rs rung-disable`"] +#[ignore = "real sockets and BBR3 capacity cycles; run nightly by `just rs rung-disable`"] async fn shaped_uplink_disables_and_resumes() { let server = MoqServer::new(MoqServerConfig { bind: Some("127.0.0.1:0".into()), @@ -89,7 +89,18 @@ async fn shaped_uplink_disables_and_resumes() { let remote = await_announced(&receiver.consume(), "ladder").await; let catalog = MoqMediaCatalogConsumer::subscribe(&remote).await.unwrap(); let first = catalog.next().await.unwrap().expect("catalog ended"); - let low = remote.subscribe_track("low".into(), None).await.unwrap(); + let low = MoqMediaTrackConsumer::subscribe( + &remote, + MoqMediaTrackConfig { + rendition: MoqRendition::Video { + id: "low".into(), + config: first.video["low"].clone(), + }, + subscription: None, + }, + ) + .await + .unwrap(); let high = MoqMediaTrackConsumer::subscribe( &remote, MoqMediaTrackConfig { @@ -110,6 +121,8 @@ async fn shaped_uplink_disables_and_resumes() { let mut below_lowest = false; let mut resumed = false; let mut received = [0u64; 2]; + let mut sent = [0u64; 2]; + let mut keyframe = [true; 2]; let mut resumed_frame = false; let mut stopped_at = None; let mut resumed_at = None; @@ -128,25 +141,28 @@ async fn shaped_uplink_disables_and_resumes() { track.set_enabled(enabled).unwrap(); if !enabled { assert_eq!(i, 1, "the lowest rung must never disable"); - assert!(reservation.rate().unwrap() < BOUNDARY); track.discontinuity().unwrap(); + keyframe[i] = true; if elapsed >= NARROW && stopped_at.is_none() { stopped_at = Some(timestamp); } } else if i == 1 && was_disabled && stopped_at.is_some() { assert!(elapsed >= RECOVER, "resumed before the path reopened"); - assert!(reservation.rate().unwrap() >= BOUNDARY + BOUNDARY / 20, - "resumed without clearing the hysteresis"); + assert_eq!(target, MoqRateTarget::Enabled { bitrate_bps: BOUNDARY }, + "resume must start at the band boundary before ramping"); resumed_at.get_or_insert(timestamp); } } if let MoqRateTarget::Enabled { bitrate_bps } = control.target() { - // An IDR access unit padded to the target's frame budget. This exercises - // framing and transport, without making a codec's rate accuracy the test. - let mut payload = h264_init(); - payload.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21]); + // Two-second GOPs, with an IDR on resume. Pad access units to the + // target budget, keeping codec rate accuracy out of this transport test. + let idr = keyframe[i] || sent[i].is_multiple_of(60); + let mut payload = if idr { h264_init() } else { Vec::new() }; + payload.extend_from_slice(&[0, 0, 0, 1, if idr { 0x65 } else { 0x41 }, 0x88, 0x84, 0x21]); payload.resize((bitrate_bps / 8 / 30) as usize, 0xff); track.write_frame(MoqFrame { payload, timestamp: Some(us(timestamp)) }).unwrap(); + keyframe[i] = false; + sent[i] += 1; } } if elapsed < NARROW && remote_enabled == Some(true) && received.iter().all(|count| *count > 0) { @@ -178,9 +194,10 @@ async fn shaped_uplink_disables_and_resumes() { remote_enabled = Some(update.video["high"].enabled); println!("{:?}: remote enabled {remote_enabled:?}", start.elapsed()); }, - frame = low.read_frame() => { - frame.unwrap().expect("lowest track ended"); - received[0] += 1; + frame = low.next() => { + if let MoqMediaRead::Frame { .. } = frame.unwrap().expect("lowest track ended") { + received[0] += 1; + } }, frame = high.next() => { let read = frame.unwrap().expect("top track ended"); @@ -193,7 +210,7 @@ async fn shaped_uplink_disables_and_resumes() { assert!(timestamp < stopped || resumed_at.is_some_and(|at| timestamp >= at), "media was produced while disabled: {timestamp}"); } - if resumed_at.is_some_and(|at| timestamp >= at) { + if !resumed_frame && resumed_at.is_some_and(|at| timestamp >= at) { assert!(frame.keyframe, "the resumed rendition must start with a keyframe"); resumed_frame = true; } @@ -201,6 +218,9 @@ async fn shaped_uplink_disables_and_resumes() { } }, } + if initial && disabled && below_lowest && resumed && resumed_frame { + break; + } } let stats = shaper.verify().unwrap(); println!("shaper: {stats}"); From 00f9fdad6d9f9a9fa40a0757ca129db9bd973bd1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:49:42 -0800 Subject: [PATCH 4/6] quest: retain the real-probe recovery blocker Co-Authored-By: GPT-6.1 Sol --- quest/m1/simulcast-rung-disable.md | 31 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/quest/m1/simulcast-rung-disable.md b/quest/m1/simulcast-rung-disable.md index b3278c103ac..9449ad03ba9 100644 --- a/quest/m1/simulcast-rung-disable.md +++ b/quest/m1/simulcast-rung-disable.md @@ -1,4 +1,4 @@ -# [S] Finish validation of the shaped-uplink rung proof +# [M] A shaped uplink proves native rung disable and resume ## Goal @@ -12,24 +12,33 @@ the lowest rung. It runs nightly. The JS publisher is ## Plan The native rule and publishers have landed. The remaining transport proof is -implemented in `rs/moq-ffi/src/test/rung.rs`, using the pre-encoded FFI path: +staged in `rs/moq-ffi/src/test/rung.rs`, using the pre-encoded FFI path: `MoqRateControl`, reservations, `MoqVideoInit.lower`, and `MoqMediaTrackProducer::set_enabled`. `just rs rung-disable` runs the ignored socket test and is wired into Nightly beside `quic-sim`. -The shaped run passed, including queue overflow, remote catalog flips, no -media timestamps in the disabled interval, a below-lowest grant, and a resumed -keyframe. The final phase ends on observed recovery, without imposing an exact -recovery time. Existing mock-grant tests cover the built-in encoder's teardown -and reopen. No production controller change is part of this proof. - -Finish local validation and `/quest-review`, then remove this quest once the -full gate passes and the PR can be marked ready. Keep the nightly continuation +The proof is not reliable yet. One real run completed the entire cycle, but a +fresh run disabled the top rendition during startup and never delivered it on +the initial wide link. It also stayed disabled after the path reopened. The +fixture drains both media cursors, retains its reservations, and keeps the +state assertions intact. Its final phase ends on observed recovery rather than +an exact recovery time; extending the hang guard did not settle the cold-start +failure. + +Diagnose native probe recovery on the real driver/shaper path, both from cold +start and after a below-lowest grant. Probe traces show the target remains +installed and attempts run, but partial estimate gains do not reliably clear +the rung's resume margin. Small constant-rate deterministic QUIC spikes did +reach that margin, so the root cause is not isolated. No controller change or +instrumentation is part of this attempt. + +Finish the proof at its source, pass local checks, and run `/quest-review` +before removing this quest and marking the PR ready. Keep nightly continuation changes with [moq-dev/moq#7667](https://github.com/moq-dev/moq/pull/7667). ## Waiting -- moq-dev/moq: the full local gate must pass after the existing C/C++ pkg-config fix in [moq-dev/moq#7654](https://github.com/moq-dev/moq/pull/7654) and default Gradle discovery fix in [moq-dev/moq#7632](https://github.com/moq-dev/moq/pull/7632), on a Nix builder that permits user namespaces. The prior gate also failed Dart's native-build freshness assertion; recheck it with the FFI build work in [moq-dev/moq#7666](https://github.com/moq-dev/moq/pull/7666), without importing those independent changes here. +- moq-dev/moq: the full local gate must pass after the existing C/C++ pkg-config fix in [moq-dev/moq#7654](https://github.com/moq-dev/moq/pull/7654) and default Gradle discovery fix in [moq-dev/moq#7632](https://github.com/moq-dev/moq/pull/7632), on a Nix builder that permits user namespaces. The earlier gate also failed Dart's native-build freshness assertion; recheck it with the FFI build work in [moq-dev/moq#7666](https://github.com/moq-dev/moq/pull/7666), without importing those independent changes here. ## Related From 00fbbaca234dd5a1822cf86082d130af2f44afbc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:53:32 -0800 Subject: [PATCH 5/6] quest: clear the passing Dart validation wait Co-Authored-By: GPT-6.1 Sol --- quest/m1/simulcast-rung-disable.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/simulcast-rung-disable.md b/quest/m1/simulcast-rung-disable.md index 9449ad03ba9..db201f0d3af 100644 --- a/quest/m1/simulcast-rung-disable.md +++ b/quest/m1/simulcast-rung-disable.md @@ -38,7 +38,7 @@ changes with [moq-dev/moq#7667](https://github.com/moq-dev/moq/pull/7667). ## Waiting -- moq-dev/moq: the full local gate must pass after the existing C/C++ pkg-config fix in [moq-dev/moq#7654](https://github.com/moq-dev/moq/pull/7654) and default Gradle discovery fix in [moq-dev/moq#7632](https://github.com/moq-dev/moq/pull/7632), on a Nix builder that permits user namespaces. The earlier gate also failed Dart's native-build freshness assertion; recheck it with the FFI build work in [moq-dev/moq#7666](https://github.com/moq-dev/moq/pull/7666), without importing those independent changes here. +- moq-dev/moq: the full local gate must pass after the existing C/C++ pkg-config fix in [moq-dev/moq#7654](https://github.com/moq-dev/moq/pull/7654) and default Gradle discovery fix in [moq-dev/moq#7632](https://github.com/moq-dev/moq/pull/7632), on a Nix builder that permits user namespaces. ## Related From d5aa7e8f272af67da8d93920d2ffe82d35efc5cf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 15:39:08 -0800 Subject: [PATCH 6/6] fix(quic): let a sparse sender's probe climb to its target Two limits held the bandwidth probe short of its target whenever the probe's inflight was only a few packets, as for the lowest simulcast rung probing for the rungs above it: - Each probe packet had to fit a whole segment under the controller's inflight bound, so a flight of a few packets stopped most of a packet short of it: the whole of ProbeBW_UP's gain. The packet that reaches the bound may now cross it. - An attempt stalled two smoothed round trips after its last raise. At a low estimate BBR3 paces a round over several round trips, and the peer's ACK delay holds a sparse sender's acknowledgments back, so rounds that were still raising the estimate were judged misses and sat out four seconds each. An attempt now stalls once two rounds are acknowledged in packets without a raise. The moq-ffi shaped-uplink rung test now passes: the top rendition resumes about six seconds after the path reopens. That completes the quest. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/c/api-gaps.md | 4 - quest/m1/simulcast-rung-disable.md | 47 ------ ...bandwidth-grant-in-moq-audio-instead-of.md | 9 +- quest/m2/obs-multitrack-check.md | 4 - quest/m2/probe/README.md | 3 +- quest/m2/probe/padding.md | 2 +- quest/m2/probe/rung-disable-js.md | 3 +- quest/m2/quic-realtime/README.md | 4 +- rs/moq-congestion/src/lib.rs | 4 +- rs/moq-ffi/src/test/rung.rs | 5 +- rs/moq-quic/src/connection/bandwidth_probe.rs | 155 +++++++++++++----- rs/moq-quic/src/connection/mod.rs | 2 +- rs/moq-quic/src/tests/probe.rs | 74 +++++++-- 14 files changed, 181 insertions(+), 136 deletions(-) delete mode 100644 quest/m1/simulcast-rung-disable.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 24e54e04076..1a7c430733e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -156,7 +156,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [Sockets](/quest/m1/socket/README.md) - a generic dial/accept `Socket` in moq-net, Rust, JS, and Python, with tracks as streams; moq-boy and CARLA move onto it - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate -- [Native rung disable proof](/quest/m1/simulcast-rung-disable.md) - a shaped uplink through moq-ffi proves the top rendition stops and resumes over real BBR3 probing - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; time-stretched browser playback without steady-state underruns and a native trace replay remain - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust diff --git a/quest/m1/c/api-gaps.md b/quest/m1/c/api-gaps.md index 840349d9fbe..9f8e318afd8 100644 --- a/quest/m1/c/api-gaps.md +++ b/quest/m1/c/api-gaps.md @@ -28,7 +28,3 @@ The independent JSON window caller timestamp gap is implemented in Public API: moq-c gains pre-encoded rung disable, lower-rendition ordering, and reservation rate control. Wire: none. - -## Required - -- [OBS rung disable](/quest/m1/simulcast-rung-disable.md) - land the shared importer and FFI prerequisites in [moq-dev/moq#7029](https://github.com/moq-dev/moq/pull/7029) before adding their C ABI counterparts diff --git a/quest/m1/simulcast-rung-disable.md b/quest/m1/simulcast-rung-disable.md deleted file mode 100644 index db201f0d3af..00000000000 --- a/quest/m1/simulcast-rung-disable.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] A shaped uplink proves native rung disable and resume - -## Goal - -One shaped-uplink run through moq-ffi proves the rung rule end to end over -real BBR3 probing: as the bottleneck narrows below a rung's band, the top -rendition stops and is advertised `enabled: false`, and once the bottleneck -opens back up it resumes with hysteresis, including after the grant fell below -the lowest rung. It runs nightly. The JS publisher is -[JS rung disable](/quest/m2/probe/rung-disable-js.md). - -## Plan - -The native rule and publishers have landed. The remaining transport proof is -staged in `rs/moq-ffi/src/test/rung.rs`, using the pre-encoded FFI path: -`MoqRateControl`, reservations, `MoqVideoInit.lower`, and -`MoqMediaTrackProducer::set_enabled`. `just rs rung-disable` runs the ignored -socket test and is wired into Nightly beside `quic-sim`. - -The proof is not reliable yet. One real run completed the entire cycle, but a -fresh run disabled the top rendition during startup and never delivered it on -the initial wide link. It also stayed disabled after the path reopened. The -fixture drains both media cursors, retains its reservations, and keeps the -state assertions intact. Its final phase ends on observed recovery rather than -an exact recovery time; extending the hang guard did not settle the cold-start -failure. - -Diagnose native probe recovery on the real driver/shaper path, both from cold -start and after a below-lowest grant. Probe traces show the target remains -installed and attempts run, but partial estimate gains do not reliably clear -the rung's resume margin. Small constant-rate deterministic QUIC spikes did -reach that margin, so the root cause is not isolated. No controller change or -instrumentation is part of this attempt. - -Finish the proof at its source, pass local checks, and run `/quest-review` -before removing this quest and marking the PR ready. Keep nightly continuation -changes with [moq-dev/moq#7667](https://github.com/moq-dev/moq/pull/7667). - -## Waiting - -- moq-dev/moq: the full local gate must pass after the existing C/C++ pkg-config fix in [moq-dev/moq#7654](https://github.com/moq-dev/moq/pull/7654) and default Gradle discovery fix in [moq-dev/moq#7632](https://github.com/moq-dev/moq/pull/7632), on a Nix builder that permits user namespaces. - -## Related - -- [JS rung disable](/quest/m2/probe/rung-disable-js.md) - the same rule in `js/publish`, probing with padding -- [Ladder controller](/quest/m3/ladder/controller.md) - the same rung disable for generated transcode ladders -- [Viewer up-switch](/quest/m2/probe/viewer-upswitch.md) - the viewer side of moving between renditions as capacity changes diff --git a/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md b/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md index 1bb1d2d1eb3..6a1bd011c31 100644 --- a/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md +++ b/quest/m2/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md @@ -42,9 +42,8 @@ grant; `Options::bandwidth` documents that `300_000 * channels` (`backend/libopus.rs`, `rs/moq-audio/src/opus.rs`). `Policy::min` defaults to a tenth of the ceiling (`rs/moq-mux/src/rate.rs`); for Opus it is the codec floor. A grant below it disables the rendition until bandwidth recovers (decided - 2026-10-04 with the enabled flag), through the disable and re-enable rule - [simulcast rung disable](/quest/m1/simulcast-rung-disable.md) puts beside - `Control` rather than a second one; the encoder never errors. Viewers with + 2026-10-04 with the enabled flag), through the rung disable and re-enable + rule in `moq_mux::rate::Control` rather than a second one; the encoder never errors. Viewers with no enabled alternative hold it ([push-to-talk](/quest/m1/push-to-talk/README.md)), so its demand can stay while its grant is below the floor; recovery evaluates a hypothetical share against the current estimate, as the ladder controller @@ -68,7 +67,3 @@ one policy step, holds it on a `None` grant, and ramps back when the grant returns; a grant below the Opus floor disables the rendition, and recovered bandwidth enables it again even after the last viewer unsubscribed; a PCM Producer ignores every grant. - -## Required - -- [Simulcast rung disable](/quest/m1/simulcast-rung-disable.md) - the shared disable and re-enable rule in `moq_mux::rate` that this quest reuses diff --git a/quest/m2/obs-multitrack-check.md b/quest/m2/obs-multitrack-check.md index 8edade70472..41e231b6cb1 100644 --- a/quest/m2/obs-multitrack-check.md +++ b/quest/m2/obs-multitrack-check.md @@ -34,7 +34,3 @@ OBS, so the manual run never happened. each rendition advertises its own bitrate in the catalog. Public API: none. Wire: none. - -## Related - -- [Simulcast rung disable](/quest/m1/simulcast-rung-disable.md) - the shaped-uplink proof of the rung disable [moq-dev/moq#7029](https://github.com/moq-dev/moq/pull/7029) added, which turns these layers off as the grant falls diff --git a/quest/m2/probe/README.md b/quest/m2/probe/README.md index 434d87c7631..eb92d3c2b75 100644 --- a/quest/m2/probe/README.md +++ b/quest/m2/probe/README.md @@ -50,7 +50,7 @@ publisher-local padding handle: Decided with the maintainer on 2026-10-20: - The send target, native probing, - and the native publishers' [rung disable](/quest/m1/simulcast-rung-disable.md) + and the native publishers' rung disable moved to m1, so Rust, FFI, and OBS publishers recover a stopped rendition in the next breaking release. Their rung disable needs the target and native probing, not padding. @@ -96,7 +96,6 @@ This README's own work, once the children land: ## Required -- [Native rung disable proof](/quest/m1/simulcast-rung-disable.md) - a shaped uplink through moq-ffi proves the top rendition stops and resumes over real BBR3 probing - [Browser send estimate](/quest/m2/probe/browser-send-estimate.md) - shipping Chrome and Safari publishers adapt to the relay's PROBE estimate, since they have no `getStats()` - [One page-wide getStats sampling ticker](/quest/m2/probe/stats-ticker.md) - a page's connections share one sampling ticker - [Padding probes](/quest/m2/probe/padding.md) - a browser probes with a lowest-priority padding stream on an ALR schedule diff --git a/quest/m2/probe/padding.md b/quest/m2/probe/padding.md index f19f7a120d1..64cfcd616b9 100644 --- a/quest/m2/probe/padding.md +++ b/quest/m2/probe/padding.md @@ -23,7 +23,7 @@ Decided with the maintainer on 2026-10-17 (see the implements the sender hook in `js/net`, mirroring Rust's `transport::Probe`. Decided 2026-10-20: browser-only. A native session without native probing (moq-quic under CUBIC) has no sender, so - [native rung disable](/quest/m1/simulcast-rung-disable.md) never stops a + the native rung disable (`moq_mux::rate::Policy::lower`) never stops a rung there. - **The browser's send target.** The `js/net` mirror of the Rust send target (`Allocator::set_target`, `Session::bandwidth`, `transport::Probe`) lands here, beside the sender that diff --git a/quest/m2/probe/rung-disable-js.md b/quest/m2/probe/rung-disable-js.md index a6e5fb5808a..0bcc6d6d215 100644 --- a/quest/m2/probe/rung-disable-js.md +++ b/quest/m2/probe/rung-disable-js.md @@ -9,7 +9,7 @@ lower renditions keep their share instead of every rung degrading together. ## Plan -Split from [native rung disable](/quest/m1/simulcast-rung-disable.md) by the +Split from the native rung disable by the maintainer on 2026-10-20: the native publishers moved to m1 on native probing, while a browser can only probe with [padding](/quest/m2/probe/padding.md), which stays in m2. @@ -34,7 +34,6 @@ Public API: whatever knob `@moq/publish` exposes for the rule. Wire: none. ## Required -- [Native rung disable](/quest/m1/simulcast-rung-disable.md) - the shared rule and constants this mirrors - [Padding probes](/quest/m2/probe/padding.md) - the browser's sender, which raises the estimate so a disabled rung can resume ## Related diff --git a/quest/m2/quic-realtime/README.md b/quest/m2/quic-realtime/README.md index e4b1bae884c..9a05d33019b 100644 --- a/quest/m2/quic-realtime/README.md +++ b/quest/m2/quic-realtime/README.md @@ -56,7 +56,7 @@ spike assumed no consumer; teleoperation uplinks are one): `probe_inflight`, so moq-quic's native probing ([moq-dev/moq#6635](https://github.com/moq-dev/moq/pull/6635)) still raises its estimate, and the bake-off measures probing alongside queuing - delay. Reason: the [native rung disable](/quest/m1/simulcast-rung-disable.md) + delay. Reason: the native rung disable (`moq_mux::rate::Policy::lower`) only stops a rung on a session with a sender, so a `Realtime` that cannot probe would silently leave moq-cli, OBS, and FFI simulcast publishers with no rung disable or recovery. A candidate that cannot probe does not ship. @@ -116,7 +116,7 @@ Remaining work once the candidates finish (this README's own job): estimate is its reference window over the RTT, not a delivery rate, so each keyframe burst grew it past the path: 241 kB/s on a 125 kB/s path in #7547, which let a rung with keyframes recover without a probe. The - [native rung disable](/quest/m1/simulcast-rung-disable.md) could re-enable + native rung disable could re-enable a rung the path can't carry on such an estimate, so the comparison reports each candidate's estimate against the path's capacity under the encoder model. diff --git a/rs/moq-congestion/src/lib.rs b/rs/moq-congestion/src/lib.rs index f41ef308b4e..454a1f71d8f 100644 --- a/rs/moq-congestion/src/lib.rs +++ b/rs/moq-congestion/src/lib.rs @@ -249,8 +249,8 @@ pub struct Metrics { pub pacing_rate: Option, /// Estimated sustainable path bandwidth (bytes/s) pub bandwidth_estimate: Option, - /// Bytes in flight up to which an application-limited sender may resend in-flight data, - /// while the controller probes for bandwidth (bytes) + /// Bytes in flight an application-limited sender may fill by resending in-flight data, the + /// last packet crossing it, while the controller probes for bandwidth (bytes) /// /// `None` when the controller is not probing, or cannot learn from a sender that fills its /// pacing with redundant data. diff --git a/rs/moq-ffi/src/test/rung.rs b/rs/moq-ffi/src/test/rung.rs index be74fffd63f..4d27196e4ed 100644 --- a/rs/moq-ffi/src/test/rung.rs +++ b/rs/moq-ffi/src/test/rung.rs @@ -74,7 +74,10 @@ async fn shaped_uplink_disables_and_resumes() { bitrate: Some(max), ..Default::default() }), - lower: tracks.first().cloned(), + lower: tracks + .first() + .cloned() + .map(|producer| MoqMediaProducer::Track { producer }), ..video_init(MoqVideoFormat::Avc3, h264_init()) }, ) diff --git a/rs/moq-quic/src/connection/bandwidth_probe.rs b/rs/moq-quic/src/connection/bandwidth_probe.rs index 094de2bc47c..f091e43a601 100644 --- a/rs/moq-quic/src/connection/bandwidth_probe.rs +++ b/rs/moq-quic/src/connection/bandwidth_probe.rs @@ -13,7 +13,7 @@ //! An attempt that ends short of the target found the path's ceiling, and another would only //! spend the same duplicates finding it again. So each miss in a row doubles how long the probe //! sits out before retrying the target, up to a bound that finds capacity which returns within a -//! few seconds. A miss shows within two round trips, as a round of duplicates that raised +//! few seconds. A miss shows within two rounds, as a round of duplicates that raised //! nothing, while the controller's probe lasts several rounds more before it sees the plateau //! itself, so an attempt stops there. Each raise restarts that count: an attempt that begins //! below the plateau, as once BBR3 has aged an application-limited estimate, climbs back to it @@ -61,10 +61,32 @@ pub(super) struct BandwidthProbe { /// Probing toward the target struct Attempt { - /// When the attempt began or last raised the estimate - since: Instant, /// The highest bandwidth estimate since it began, in bytes per second estimate: u64, + /// The packet number whose acknowledgment ends the round in progress + round: u64, + /// Rounds acknowledged in full since the attempt began or last raised the estimate + flat: u32, +} + +impl Attempt { + /// An attempt from `estimate`, whose first round ends once packet `next` is acknowledged + fn new(estimate: u64, next: u64) -> Self { + Self { + estimate, + round: next, + flat: 0, + } + } +} + +/// Where the connection's packet numbers stand, to count an attempt's rounds +#[derive(Clone, Copy)] +struct Rounds { + /// The largest packet number acknowledged + acked: Option, + /// The packet number sent next + next: u64, } /// Whether a probe packet may be sent now @@ -116,7 +138,12 @@ impl BandwidthProbe { /// Judge the attempt in progress by `estimate`, returning whether the probe may fill the /// controller's probe now - fn judge(&mut self, probing: bool, estimate: Option, rtt: Duration, now: Instant) -> bool { + /// + /// An attempt stalls once two rounds are acknowledged without a raise: the round in flight + /// at the last raise, then one sent at the inflight it allowed. Counted in packets rather + /// than round trips, so a round the controller's pacing spreads out, or the peer's ACK delay + /// holds back, is still judged whole. + fn judge(&mut self, probing: bool, estimate: Option, rounds: Rounds, now: Instant) -> bool { let estimate = estimate.unwrap_or(0); if estimate >= self.target { self.attempt = None; @@ -126,10 +153,12 @@ impl BandwidthProbe { } if let Some(attempt) = &mut self.attempt { if estimate > attempt.estimate { - attempt.since = now; - attempt.estimate = estimate; + *attempt = Attempt::new(estimate, rounds.next); + } else if rounds.acked.is_some_and(|acked| acked >= attempt.round) { + attempt.round = rounds.next; + attempt.flat += 1; } - let stalled = now > attempt.since + 2 * rtt; + let stalled = attempt.flat >= 2; if !probing || stalled { self.attempt = None; self.misses = self.misses.saturating_add(1); @@ -167,8 +196,9 @@ impl BandwidthProbe { } impl Connection { - /// Whether a packet of `bytes` resending in-flight data may go now - pub(super) fn bandwidth_probe_ready(&mut self, now: Instant, bytes: u64) -> Ready { + /// Whether a packet of up to `segment` bytes resending in-flight data may go now, after the + /// `untracked` bytes already built but not yet in flight + pub(super) fn bandwidth_probe_ready(&mut self, now: Instant, untracked: u64, segment: u64) -> Ready { let target = self.bandwidth_probe.target; if target == 0 || !self.state.is_established() @@ -179,10 +209,14 @@ impl Connection { } let metrics = self.path.congestion.metrics(); let probing = metrics.probe_inflight.is_some(); - let rtt = self.path.rtt.get(); + let space = &self.spaces[SpaceId::Data]; + let rounds = Rounds { + acked: space.largest_acked_packet, + next: space.next_packet_number, + }; if !self .bandwidth_probe - .judge(probing, metrics.bandwidth_estimate, rtt, now) + .judge(probing, metrics.bandwidth_estimate, rounds, now) { return Ready::Idle; } @@ -192,7 +226,11 @@ impl Connection { if !self.find_bandwidth_probe_source() { return Ready::Idle; } - if self.path.in_flight.bytes + bytes > inflight { + // The packet that reaches the bound may cross it. Holding each one under it would leave a + // flight of a few packets, as a small BDP has, most of a packet short of the bound, which + // is the whole of the gain past BBR.bw the probe has to measure. A probe packet is also + // often smaller than the segment, holding only the little a sparse sender has in flight. + if self.path.in_flight.bytes + untracked >= inflight { // Before the first duplicate, media alone past the bound, as when another flow's queue // inflates the round trip over BBR.min_rtt, is all the sender has: taking its samples // as the path's would age BBR3's model down to the media @@ -205,14 +243,14 @@ impl Connection { // target would cap the reading; the cluster's size bounds the duplicates instead let mtu = self.path.current_mtu(); if !self.path.congestion.paces_probe() - && let Some(at) = self.bandwidth_probe.pacing.delay(target, bytes, mtu, now) + && let Some(at) = self.bandwidth_probe.pacing.delay(target, untracked + segment, mtu, now) { return Ready::Blocked(Some(at)); } let estimate = metrics.bandwidth_estimate.unwrap_or(0); self.bandwidth_probe .attempt - .get_or_insert(Attempt { since: now, estimate }); + .get_or_insert(Attempt::new(estimate, rounds.next)); Ready::Send } @@ -331,6 +369,8 @@ mod tests { /// The path's rate, short of the target const PATH: u64 = 375_000; const MS: Duration = Duration::from_millis(1); + /// Packets sent each round + const ROUND: u64 = 10; fn probe(now: Instant) -> BandwidthProbe { let mut probe = BandwidthProbe::new(&transport::Config::default(), now); @@ -338,22 +378,34 @@ mod tests { probe } - /// Begin an attempt at `now`, as the first probe packet does - fn attempt(probe: &mut BandwidthProbe, now: Instant, estimate: u64) { - assert!(probe.judge(true, Some(estimate), RTT, now), "sitting out at {now:?}"); - probe.attempt.get_or_insert(Attempt { since: now, estimate }); + /// Packet numbers `n` rounds on: the first packet of this round acknowledged, the rest in + /// flight + fn round(n: u64) -> Rounds { + Rounds { + acked: (n > 0).then(|| n * ROUND), + next: (n + 1) * ROUND, + } + } + + /// Begin an attempt in round `n` at `now`, as the first probe packet does + fn attempt(probe: &mut BandwidthProbe, n: u64, now: Instant, estimate: u64) { + assert!( + probe.judge(true, Some(estimate), round(n), now), + "sitting out at {now:?}" + ); + probe.attempt.get_or_insert(Attempt::new(estimate, round(n).next)); } - /// Miss at `now` with an attempt whose round of duplicates raised nothing + /// Miss two round trips after `now`, with an attempt whose duplicates raised nothing fn stall(probe: &mut BandwidthProbe, now: Instant) -> Instant { - attempt(probe, now, PATH); + attempt(probe, 0, now, PATH); assert!( - probe.judge(true, Some(PATH), RTT, now + 2 * RTT), - "stopped before a round came back" + probe.judge(true, Some(PATH), round(1), now + RTT), + "stopped before a round of duplicates came back" ); - let missed = now + 2 * RTT + MS; + let missed = now + 2 * RTT; assert!( - !probe.judge(true, Some(PATH), RTT, missed), + !probe.judge(true, Some(PATH), round(2), missed), "kept filling a stalled attempt" ); missed @@ -366,42 +418,57 @@ mod tests { let mut probe = probe(now); for backoff in [BACKOFF, MAX_BACKOFF, MAX_BACKOFF] { now = stall(&mut probe, now); - assert!(!probe.judge(true, Some(PATH), RTT, now + backoff - MS)); + assert!(!probe.judge(true, Some(PATH), round(2), now + backoff - MS)); now += backoff; } } + /// A round that takes many round trips, as the controller's pacing or the peer's ACK delay + /// spreads it out, is still judged whole + #[test] + fn slow_round_is_not_a_stall() { + let now = Instant::now(); + let mut probe = probe(now); + attempt(&mut probe, 0, now, PATH); + let late = now + 10 * RTT; + assert!(probe.judge(true, Some(PATH), round(0), late)); + assert!(probe.judge(true, Some(PATH), round(1), late)); + assert!( + probe.judge(true, Some(PATH + 1), round(1), late + RTT), + "a raise in the second round" + ); + assert!(probe.judge(true, Some(PATH + 1), round(2), late + RTT)); + } + /// An attempt that raises the estimate goes on until the controller's probe ends, which misses /// if it is still short of the target #[test] fn rising_attempt_misses_at_its_end() { let now = Instant::now(); let mut probe = probe(now); - attempt(&mut probe, now, PATH); + attempt(&mut probe, 0, now, PATH); let later = now + 4 * RTT; - assert!(probe.judge(true, Some(PATH + 1), RTT, later)); - assert!(!probe.judge(false, Some(PATH + 1), RTT, later)); - assert!(!probe.judge(true, Some(PATH + 1), RTT, later + BACKOFF - MS)); - assert!(probe.judge(true, Some(PATH + 1), RTT, later + BACKOFF)); + assert!(probe.judge(true, Some(PATH + 1), round(4), later)); + assert!(!probe.judge(false, Some(PATH + 1), round(4), later)); + assert!(!probe.judge(true, Some(PATH + 1), round(5), later + BACKOFF - MS)); + assert!(probe.judge(true, Some(PATH + 1), round(5), later + BACKOFF)); } - /// An attempt that begins below the plateau stalls two round trips after it climbs back, - /// rather than going on until the controller's probe ends; dipping and recovering to the - /// plateau is not a raise + /// An attempt that begins below the plateau stalls two rounds after it climbs back, rather + /// than going on until the controller's probe ends; dipping and recovering to the plateau + /// is not a raise #[test] fn climbing_back_stalls_at_the_plateau() { let now = Instant::now(); let mut probe = probe(now); - attempt(&mut probe, now, PATH / 2); - let back = now + RTT; - assert!(probe.judge(true, Some(PATH), RTT, back)); - assert!(probe.judge(true, Some(PATH * 3 / 4), RTT, back + RTT)); + attempt(&mut probe, 0, now, PATH / 2); + assert!(probe.judge(true, Some(PATH), round(1), now + RTT)); assert!( - probe.judge(true, Some(PATH), RTT, back + 2 * RTT), - "stalled before a round came back" + probe.judge(true, Some(PATH * 3 / 4), round(2), now + 2 * RTT), + "stalled before a round of duplicates came back" ); assert!( - !probe.judge(true, Some(PATH), RTT, back + 2 * RTT + MS), + !probe.judge(true, Some(PATH), round(3), now + 3 * RTT), "kept filling once the estimate stopped rising" ); } @@ -415,20 +482,20 @@ mod tests { now = stall(&mut probe, now + BACKOFF); // The estimate reached the target, then aged below it - assert!(!probe.judge(true, Some(TARGET), RTT, now)); + assert!(!probe.judge(true, Some(TARGET), round(2), now)); now = stall(&mut probe, now); assert!( - probe.judge(true, Some(PATH), RTT, now + BACKOFF), + probe.judge(true, Some(PATH), round(2), now + BACKOFF), "the misses before still count" ); now = stall(&mut probe, now + BACKOFF); probe.set_target(TARGET); assert!( - !probe.judge(true, Some(PATH), RTT, now), + !probe.judge(true, Some(PATH), round(2), now), "the same target retried at once" ); probe.set_target(2 * TARGET); - assert!(probe.judge(true, Some(PATH), RTT, now)); + assert!(probe.judge(true, Some(PATH), round(2), now)); } } diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 0d420a238d4..a7b4f164c9d 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -705,7 +705,7 @@ impl Connection { let bandwidth_probe = space_id == SpaceId::Data && can_send.is_empty() - && !close && match self.bandwidth_probe_ready(now, segment_size as u64 + untracked_bytes) { + && !close && match self.bandwidth_probe_ready(now, untracked_bytes, segment_size as u64) { bandwidth_probe::Ready::Idle => false, bandwidth_probe::Ready::Blocked(until) => { if let Some(until) = until { diff --git a/rs/moq-quic/src/tests/probe.rs b/rs/moq-quic/src/tests/probe.rs index b39d6f562d3..3e968704add 100644 --- a/rs/moq-quic/src/tests/probe.rs +++ b/rs/moq-quic/src/tests/probe.rs @@ -20,7 +20,7 @@ use rand_pcg::Pcg32; use super::*; use crate::congestion::{bbr3, cubic, gcc}; -/// The media's bitrate, 2 Mbit/s, which the probe has to see past +/// The media's usual bitrate, 2 Mbit/s, which the probe has to see past const MEDIA: u64 = 250_000; /// The rate the session wants, 4 Mbit/s const TARGET: u64 = 500_000; @@ -58,12 +58,13 @@ enum Upload { Bbr, } -/// The bytes of frame `index` of a group whose keyframe is `keyframe` bytes -fn frame_size(keyframe: u64, index: u64) -> u64 { - let group = MEDIA * GOP / FPS; +/// The bytes of frame `index` of a group in `scenario`'s media trace +fn frame_size(scenario: &Scenario, index: u64) -> u64 { + let group = scenario.media * GOP / FPS; + let keyframe = scenario.keyframe.max(group / GOP); match index { - 0 => keyframe.max(group / GOP), - _ => (group - keyframe.max(group / GOP)) / (GOP - 1), + 0 => keyframe, + _ => (group - keyframe) / (GOP - 1), } } @@ -90,6 +91,8 @@ struct Scenario { buffer: u32, /// The probe target; zero for the baseline target: u64, + /// The media's bitrate + media: u64, /// Each group's first frame in bytes; at most the average frame for a constant bitrate keyframe: u64, duration: Duration, @@ -115,6 +118,7 @@ impl Default for Scenario { rates: vec![(Duration::ZERO, 375_000)], buffer: 75_000, target: TARGET, + media: MEDIA, keyframe: 50_000, duration: Duration::from_secs(30), idle: None, @@ -344,9 +348,9 @@ fn run(scenario: &Scenario) -> Outcome { let at = frame_at(next_frame) - start; let paused = scenario.idle.is_some_and(|(from, until)| at >= from && at < until); if !paused { - write_frame(&mut pair, client_ch, &mut groups, scenario.keyframe, next_frame, now); + write_frame(&mut pair, client_ch, &mut groups, scenario, next_frame, now); outcome.frames += 1; - outcome.unique += frame_size(scenario.keyframe, next_frame % GOP); + outcome.unique += frame_size(scenario, next_frame % GOP); } next_frame += 1; } @@ -364,7 +368,7 @@ fn run(scenario: &Scenario) -> Outcome { receive( &mut pair, server_ch, - scenario.keyframe, + scenario, &mut readings, &groups, now, @@ -436,7 +440,14 @@ fn run(scenario: &Scenario) -> Outcome { } /// Write frame `n` of the trace, opening its group's stream on a keyframe -fn write_frame(pair: &mut Pair, ch: ConnectionHandle, groups: &mut Vec, keyframe: u64, n: u64, now: Instant) { +fn write_frame( + pair: &mut Pair, + ch: ConnectionHandle, + groups: &mut Vec, + scenario: &Scenario, + n: u64, + now: Instant, +) { let index = n % GOP; if index == 0 || groups.is_empty() { if let Some(group) = groups.last_mut() { @@ -464,7 +475,7 @@ fn write_frame(pair: &mut Pair, ch: ConnectionHandle, groups: &mut Vec, k } let group = groups.last_mut().unwrap(); if group.reset.is_none() { - let data = [(n % 251) as u8].repeat(frame_size(keyframe, index) as usize); + let data = [(n % 251) as u8].repeat(frame_size(scenario, index) as usize); group.backlog.extend(data); group.written.push(now); } @@ -512,7 +523,7 @@ fn expire(pair: &mut Pair, ch: ConnectionHandle, groups: &mut [Group], now: Inst fn receive( pair: &mut Pair, ch: ConnectionHandle, - keyframe: u64, + scenario: &Scenario, readings: &mut HashMap, groups: &[Group], now: Instant, @@ -549,9 +560,9 @@ fn receive( // The sender opens each group's stream in order, after any it opened before the media let frames = &groups[(id.index() - groups[0].id.index()) as usize].written; - let mut end: u64 = (0..reading.frame).map(|i| frame_size(keyframe, i)).sum(); + let mut end: u64 = (0..reading.frame).map(|i| frame_size(scenario, i)).sum(); while (reading.frame as usize) < frames.len() { - end += frame_size(keyframe, reading.frame); + end += frame_size(scenario, reading.frame); if reading.read < end { break; } @@ -773,6 +784,32 @@ fn opening() -> Scenario { } } +/// The lowest rung of a simulcast ladder, 250 kbit/s, probes for the 1.25 Mbit/s its rungs +/// reserve over a 2 Mbit/s path with a 40ms round trip, and reaches it. A frame fits in a packet +/// and the probe's inflight in a few, so packet granularity and the peer's ACK delay are much of +/// every round. Its resent bytes are not held to [`OVERHEAD`], set for a target twice the media: +/// holding an estimate five times the media resends more per media byte. +#[test] +fn sparse_rises_to_target() { + let _guard = subscribe(); + let scenario = Scenario { + media: 31_250, + target: 156_250, + keyframe: 0, + rates: vec![(Duration::ZERO, 250_000)], + buffer: 12_500, + duration: Duration::from_secs(10), + ..Scenario::default() + }; + let outcome = run(&scenario); + outcome.report("sparse"); + let rise = outcome.rise(Duration::ZERO, scenario.target); + assert!( + rise.is_some_and(|rise| rise <= RISE), + "estimate reached the target after {rise:?}" + ); +} + /// A path short of the target, behind a queue a second deep: each attempt that misses sits out /// past the longest BBR3 waits between its own probes, and the path opening up is still found #[test] @@ -857,8 +894,8 @@ fn single_packet_writes() { assert!(pair.client_conn_mut(client_ch).stats().path.sent_bandwidth_probes > 0); } -/// A batch of probe packets stays within the inflight the controller allows a probe, counting the -/// packet still being built +/// A batch of probe packets stops at the inflight the controller allows a probe, crossing it by +/// less than its last packet, counting the packet still being built #[test] fn batch_within_probe_inflight() { let _guard = subscribe(); @@ -881,9 +918,10 @@ fn batch_within_probe_inflight() { .probe_inflight .expect("probing in Startup"); assert!(conn.stats().path.sent_bandwidth_probes > 1); + let mtu = u64::from(conn.path_mtu()); assert!( - conn.bytes_in_flight() <= inflight, - "{} bytes in flight past the probe's {inflight}", + conn.bytes_in_flight() < inflight + mtu, + "{} bytes in flight a packet past the probe's {inflight}", conn.bytes_in_flight() ); }