From 01c8024351e8a7a4a1cb9e69593414b96a0f3437 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 22:17:32 -0700 Subject: [PATCH 01/28] chore(quest): start test stability questline Co-Authored-By: GPT-6 From 4504c56828af36aa2ca1e91fdb5de2e14b1bfb80 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 23:28:27 -0700 Subject: [PATCH 02/28] chore: claim CLI paused-clock quest Co-Authored-By: GPT-6 From 1c6e7bcee78e0c3b9cee43b3eb9ae975b3cb76a9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 23:46:47 -0700 Subject: [PATCH 03/28] test(cli): run fetch and completion operations on paused time Co-Authored-By: GPT-6 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/cli-paused-clock.md | 30 --- rs/moq-cli/src/complete.rs | 137 ++++++++++---- rs/moq-cli/src/fetch.rs | 207 ++++++++++++++++----- 4 files changed, 260 insertions(+), 115 deletions(-) delete mode 100644 quest/m1/test-flakes-2/cli-paused-clock.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index c48c60452bf..368771afafc 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -30,7 +30,6 @@ Public API: none. Wire: none. ## Required -- [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Broadcast race](/quest/m1/test-flakes-2/broadcast-race.md) - the QUIC-wins race test binds no shared port and has a deterministic winner - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed diff --git a/quest/m1/test-flakes-2/cli-paused-clock.md b/quest/m1/test-flakes-2/cli-paused-clock.md deleted file mode 100644 index f29e8ab11c5..00000000000 --- a/quest/m1/test-flakes-2/cli-paused-clock.md +++ /dev/null @@ -1,30 +0,0 @@ -# [M] moq-cli tests on a paused clock - -## Goal - -The moq-cli fetch and completion tests pass under any host load: none of -them races a wall-clock budget against connect, TLS, announce, and -subscribe. - -## Plan - -- `fetch::tests::a_frame_read_times_out` asserts on a 500 ms wall deadline - ([#4084](https://github.com/moq-dev/moq/pull/4084)). - `rs/moq-cli/src/fetch.rs` wraps the whole run in one - `timeout_at(deadline, ...)`, so setup shares the budget with the read, and - under load setup alone can spend it. -- `complete::tests::a_stage_broadcast_picks_the_catalog_to_read` - ([#4084](https://github.com/moq-dev/moq/pull/4084)), - `the_catalog_format_on_the_line_is_honored` - ([#4089](https://github.com/moq-dev/moq/pull/4089)), and - `a_relay_on_the_line_answers_broadcast`: the latter pair came back empty - after the fixed 1.5 s `CEILING` on a loaded runner - ([#4404](https://github.com/moq-dev/moq/pull/4404)). - -Test on a paused clock (maintainer decision, 2026-09-28). The fixture runs -real sockets against an in-process relay, where a paused clock fires QUIC -timers while packets are in flight, so first make the timers mockable: run -the fixture over an in-memory transport, or drive noq's timers from the test -clock, whichever is smaller. Keep fetch's one absolute 30 s deadline, -matching the relay's `/fetch`; on a paused clock setup costs no time, so it -can't spend the read's budget. The completion tests reuse the same fixture. diff --git a/rs/moq-cli/src/complete.rs b/rs/moq-cli/src/complete.rs index 104f64fac24..9d4bed10fb6 100644 --- a/rs/moq-cli/src/complete.rs +++ b/rs/moq-cli/src/complete.rs @@ -108,20 +108,28 @@ thread_local! { /// Thread-local rather than a process global because this is per-request state: /// [`answer`]'s future is `!Send` (Usage's completion futures are), so it can /// only ever be driven on the thread that started it. - static GLOBALS: RefCell> = const { RefCell::new(None) }; + static GLOBALS: RefCell> = const { RefCell::new(None) }; +} + +/// The parsed flags and optional local source for one lookup. +#[derive(Clone)] +struct Lookup { + side: MoqSide, + /// A local source lets operation tests exercise the grammar without socket timers. + origin: Option, } /// Holds [`GLOBALS`] for one request and clears it on the way out. struct Globals; impl Globals { - fn set(side: Option) -> Self { - GLOBALS.with_borrow_mut(|slot| *slot = side); + fn set(side: Option, origin: Option) -> Self { + GLOBALS.with_borrow_mut(|slot| *slot = side.map(|side| Lookup { side, origin })); Self } /// The globals this request parsed, or `None` when the line has none yet. - fn get() -> Option { + fn get() -> Option { GLOBALS.with_borrow(Clone::clone) } } @@ -142,9 +150,14 @@ impl Drop for Globals { /// stage refuses, so the request is rewritten to the active chunk and handed to /// [`Stage`]. pub async fn answer(argv: &[OsString]) -> Option { + answer_from(argv, None).await +} + +/// Answer against a supplied local origin, or dial the relay parsed from the line. +async fn answer_from(argv: &[OsString], origin: Option) -> Option { let request = CompletionRequest::parse(argv)?; // Dropped at the end of this call, so a second request cannot read the first's. - let _globals = Globals::set(globals(&request)); + let _globals = Globals::set(globals(&request), origin); let overlays = overlays(); // Everything past the cursor says nothing about the word being completed. @@ -555,7 +568,7 @@ async fn renditions(ctx: &CompleteCtx<'_>) -> Option Option<(moq_net::origin::Producer, moq_tokio::Connection)> { +async fn dial( + lookup: &Lookup, + deadline: Instant, +) -> Option<(moq_net::origin::Producer, Option)> { + let side = &lookup.side; let url = side.client.url.clone()?; + if let Some(origin) = &lookup.origin { + return Some((origin.clone(), None)); + } let origin = moq_tokio::origin::spawn(); // Building the client reads the TLS material off disk synchronously, so it goes on @@ -619,7 +639,7 @@ async fn dial(side: &MoqSide, deadline: Instant) -> Option<(moq_net::origin::Pro .await .ok()? .ok()?; - Some((origin, connection)) + Some((origin, Some(connection))) } #[cfg(test)] @@ -629,13 +649,17 @@ mod tests { /// Answer a whole line, with the cursor at its end. async fn complete(line: &str) -> Vec { + complete_from(line, None).await + } + + async fn complete_from(line: &str, origin: Option) -> Vec { let argv: Vec = ["__complete_word__", "--shell", "bash", "--line", line, "--cursor"] .iter() .map(OsString::from) .chain(std::iter::once(OsString::from(line.len().to_string()))) .collect(); - answer(&argv) + answer_from(&argv, origin) .await .unwrap_or_default() .lines() @@ -643,12 +667,22 @@ mod tests { .collect() } + /// The operation fixture keeps the full line parsing and deadlines, without sockets. + struct Fixture(moq_net::origin::Producer); + + impl Fixture { + const CONNECT: &str = "--connect moqt://fixture --connect-tls-insecure"; + + async fn complete(&self, line: &str) -> Vec { + complete_from(line, Some(self.0.clone())).await + } + } + /// A relay serving `origin`, and the `--connect` flags that reach it. /// - /// Self-signed, so the line has to say `--connect-tls-insecure`. That is also the - /// point: the completer builds its client from the same flags the invocation would - /// have, so a line that can connect completes and one that cannot does not. - fn relay(origin: &moq_net::origin::Producer) -> String { + /// Self-signed, so the line has to say `--connect-tls-insecure`. The transport + /// smoke test initializes its client from those flags, like the completer. + fn network_relay(origin: &moq_net::origin::Producer) -> String { let _ = moq_tokio::crypto::install_default(); let mut config = moq_tokio::listen::Config::default(); @@ -662,11 +696,30 @@ mod tests { format!("--connect moqt://127.0.0.1:{port} --connect-tls-insecure") } + /// Client initialization and dialing still use the flags on the line. + #[tokio::test] + async fn a_real_relay_can_be_dialed() { + let _env = EnvGuard::clear(&["MOQ_CONNECT"]); + let origin = moq_tokio::origin::spawn(); + let connect = network_relay(&origin); + let words: Vec<_> = connect.split_whitespace().collect(); + let side = + MoqSide::from_argv(&words.iter().map(OsStr::new).collect::>(), Environment::Ignore).expect("side"); + let client = side.client.clone().init(side.quic).expect("client"); + let connection = client + .with_reconnect(false) + .connect(side.client.url.unwrap()) + .established() + .await + .expect("connect"); + assert!(connection.connected()); + } + /// A cursor in a later stage is completed against the stage grammar. /// /// The root spec is the globals plus the first stage, so answering a later chunk /// against it offers process-wide flags that the chunk refuses. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn retargets_to_the_active_stage() { let _env = EnvGuard::clear(&["MOQ_CONNECT"]); // A stage offers its own flags, and none of the globals it would refuse. @@ -773,26 +826,30 @@ mod tests { /// Completion must not turn a keystroke into a session with a relay the user never /// typed (that URL can carry a `?jwt=`), and a local verb must not refuse to run /// because the shell exports a relay for the publishing it usually does. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_environment_cannot_ask_for_a_moq_side() { let origin = moq_tokio::origin::spawn(); let _alpha = origin.create_broadcast("alpha").expect("alpha"); _alpha.announce(Default::default()).expect("alpha"); - let connect = relay(&origin); + let fixture = Fixture(origin.clone()); + let connect = Fixture::CONNECT; // The same reachable relay, named only by the environment. let url = connect.split_whitespace().nth(1).expect("a --connect url").to_string(); let _env = EnvGuard::set(&[("MOQ_CONNECT", &url)]); assert!( - complete("moq --connect-tls-insecure --broadcast ").await.is_empty(), + fixture + .complete("moq --connect-tls-insecure --broadcast ") + .await + .is_empty(), "MOQ_CONNECT authorized a dial the line never asked for" ); // The same relay named on the line still completes, so the gate is the URL's // source and not the dial itself. assert_eq!( - complete(&format!("moq {connect} --broadcast ")).await, + fixture.complete(&format!("moq {connect} --broadcast ")).await, ["alpha"], "a typed --connect stopped working" ); @@ -821,7 +878,7 @@ mod tests { /// connection the user did not name. An overlay that runs and finds nothing still /// suppresses the shell's path fallback, so an empty answer here is also evidence /// that the completer fired at all. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn no_relay_on_the_line_means_no_dial() { let _env = EnvGuard::clear(&["MOQ_CONNECT"]); for line in [ @@ -864,7 +921,7 @@ mod tests { } /// `--broadcast` is answered from what the relay on the line announces. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_relay_on_the_line_answers_broadcast() { let _env = EnvGuard::clear(&["MOQ_CONNECT"]); let origin = moq_tokio::origin::spawn(); @@ -873,21 +930,24 @@ mod tests { let _nested = origin.create_broadcast("room/beta").expect("beta"); _nested.announce(Default::default()).expect("beta"); - let connect = relay(&origin); - let found = complete(&format!("moq {connect} --broadcast ")).await; + let fixture = Fixture(origin.clone()); + let connect = Fixture::CONNECT; + let found = fixture.complete(&format!("moq {connect} --broadcast ")).await; assert!(found.contains(&"alpha".to_string()), "{found:?}"); assert!(found.contains(&"room/beta".to_string()), "{found:?}"); // A cursor past a `--` is answered against the stage grammar, whose chunk holds // no `--connect`: the completer still has to reach the relay the line named // before the separator. - let staged = complete(&format!("moq {connect} import fmp4 -- export --broadcast ")).await; + let staged = fixture + .complete(&format!("moq {connect} import fmp4 -- export --broadcast ")) + .await; assert_eq!(staged, found, "a later stage lost the relay the globals named"); } /// `--video-name` and `--audio-name` are answered from the catalog of the /// broadcast the stage names, which overrides the process-wide one. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_stage_broadcast_picks_the_catalog_to_read() { let _env = EnvGuard::clear(&["MOQ_CONNECT"]); use hang::catalog::{AudioCodec, AudioConfig, H264, VideoConfig}; @@ -921,11 +981,12 @@ mod tests { // The global names `other`; the stage overrides it, exactly as the invocation // this line is on its way to becoming would. - let connect = relay(&origin); + let fixture = Fixture(origin.clone()); + let connect = Fixture::CONNECT; let line = format!("moq {connect} --broadcast other export --broadcast wanted"); - assert_eq!(complete(&format!("{line} --video-name ")).await, ["hd"]); - assert_eq!(complete(&format!("{line} --audio-name ")).await, ["stereo"]); + assert_eq!(fixture.complete(&format!("{line} --video-name ")).await, ["hd"]); + assert_eq!(fixture.complete(&format!("{line} --audio-name ")).await, ["stereo"]); } /// The `--catalog-format` on the line decides which catalog track is read. @@ -936,7 +997,7 @@ mod tests { /// else, which is the one shape that tells the two apart: `moq-mux`'s catalog /// producer emits hang and MSF from the same source, so an ordinary broadcast /// answers either way and hides the bug. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_catalog_format_on_the_line_is_honored() { let _env = EnvGuard::clear(&["MOQ_CONNECT"]); let origin = moq_tokio::origin::spawn(); @@ -954,13 +1015,21 @@ mod tests { let mut group = track.append_group().expect("group"); group.write_frame(moq_net::Timestamp::now(), catalog).expect("frame"); - let connect = relay(&origin); + let fixture = Fixture(origin.clone()); + let connect = Fixture::CONNECT; let line = format!("moq {connect} --broadcast room"); // Nothing publishes a Hang catalog here, so the default finds no renditions. - assert!(complete(&format!("{line} export --video-name ")).await.is_empty()); + assert!( + fixture + .complete(&format!("{line} export --video-name ")) + .await + .is_empty() + ); assert_eq!( - complete(&format!("{line} export --catalog-format msf --video-name ")).await, + fixture + .complete(&format!("{line} export --catalog-format msf --video-name ")) + .await, ["hd"], "export ignored its own --catalog-format" ); @@ -968,9 +1037,11 @@ mod tests { // `play` declares a `--catalog-format` of its own, on a different command. #[cfg(feature = "play")] { - assert!(complete(&format!("{line} play --video-name ")).await.is_empty()); + assert!(fixture.complete(&format!("{line} play --video-name ")).await.is_empty()); assert_eq!( - complete(&format!("{line} play --catalog-format msf --video-name ")).await, + fixture + .complete(&format!("{line} play --catalog-format msf --video-name ")) + .await, ["hd"], "play ignored its own --catalog-format" ); diff --git a/rs/moq-cli/src/fetch.rs b/rs/moq-cli/src/fetch.rs index fad7b7862b7..ad5a6fc4bb8 100644 --- a/rs/moq-cli/src/fetch.rs +++ b/rs/moq-cli/src/fetch.rs @@ -67,17 +67,46 @@ async fn fetch( .with_subscriber(origin.clone()) .with_reconnect(false); + fetch_from( + Request { + args, + path: &broadcast, + deadline, + }, + async { + let connection = client.connect(url).established().await.context("failed to connect")?; + anyhow::Ok((origin, connection)) + }, + out, + ) + .await +} + +/// The operation's selection and one absolute deadline, including setup. +struct Request<'a> { + args: &'a Args, + path: &'a str, + deadline: Instant, +} + +/// Keep setup and all reads under the caller's one absolute deadline. +async fn fetch_from( + request: Request<'_>, + setup: impl std::future::Future>, + out: &mut (impl AsyncWrite + Unpin), +) -> anyhow::Result<()> { + let Request { args, path, deadline } = request; let result = timeout_at(deadline, async { // Held until the last frame is read; dropping it closes the session. - let _connection = client.connect(url).established().await.context("failed to connect")?; + let (origin, _connection) = setup.await?; // Wait for a covering route rather than asking on the spot: the announcement // is still in flight right after connecting. let broadcast = origin .consume() - .routed_broadcast(broadcast.as_str()) + .routed_broadcast(path) .await - .with_context(|| format!("broadcast `{}` not found", crate::display_name(&broadcast)))?; + .with_context(|| format!("broadcast `{}` not found", crate::display_name(path)))?; let track = broadcast.track(&args.track)?; let mut group = moq_relay::fetch_group(&track, args.group) .await @@ -131,15 +160,10 @@ mod tests { ] } - /// A running relay with a publisher attached, and the flags that reach it. + /// An in-memory origin with the same finished, empty, and live tracks. struct Fixture { - /// `--connect` and the TLS pin for the relay's generated certificate. - connect: [String; 4], - /// The relay's HTTP listener, for comparing against `/fetch`. - http: std::net::SocketAddr, - /// Kept so the published broadcast, its tracks, and the open group stay live. - _publisher: ( - moq_tokio::Connection, + origin: moq_net::origin::Producer, + _keep: ( moq_net::broadcast::Producer, Vec, moq_net::group::Producer, @@ -147,22 +171,7 @@ mod tests { } impl Fixture { - /// Publish `demo` with three finished groups on `data`, no group on `empty`, - /// and one frame of a group that never finishes on `live`. async fn new() -> Self { - let _ = moq_tokio::crypto::install_default(); - let fixture = moq_relay::test_relay().await.expect("test relay"); - let ready = fixture.relay.ready(); - tokio::spawn(fixture.relay.run()); - ready.wait().await.expect("relay ready"); - - let connect = [ - "--connect".to_string(), - fixture.url.to_string(), - "--connect-tls-fingerprint".to_string(), - fixture.fingerprint.clone(), - ]; - let origin = moq_tokio::origin::spawn(); let broadcast = origin.create_broadcast("demo").expect("broadcast"); broadcast.announce(Default::default()).expect("announce"); @@ -181,11 +190,61 @@ mod tests { open.write_frame(moq_net::Timestamp::ZERO, b"first".as_ref()) .expect("frame"); + Self { + origin, + _keep: (broadcast, vec![data, empty, live], open), + } + } + + async fn fetch(&self, args: &[&str], timeout: Duration) -> (anyhow::Result<()>, Vec) { + let (_, args) = parse(&[], args); + let mut out = Vec::new(); + let result = fetch_from( + Request { + args: &args, + path: "demo", + deadline: Instant::now() + timeout, + }, + std::future::ready(Ok((self.origin.clone(), ()))), + &mut out, + ) + .await; + (result, out) + } + } + + /// A running relay with a publisher attached and its HTTP address. + struct NetworkFixture { + /// The relay's HTTP listener, for comparing against `/fetch`. + http: std::net::SocketAddr, + /// Kept so the published broadcast, its tracks, and the open group stay live. + _publisher: (moq_tokio::Connection, Fixture), + } + + impl NetworkFixture { + /// Publish `demo` with three finished groups on `data`, no group on `empty`, + /// and one frame of a group that never finishes on `live`. + async fn new() -> Self { + let _ = moq_tokio::crypto::install_default(); + let fixture = moq_relay::test_relay().await.expect("test relay"); + let ready = fixture.relay.ready(); + tokio::spawn(fixture.relay.run()); + ready.wait().await.expect("relay ready"); + + let connect = [ + "--connect".to_string(), + fixture.url.to_string(), + "--connect-tls-fingerprint".to_string(), + fixture.fingerprint.clone(), + ]; + + let publisher = Fixture::new().await; + let (moq, _) = parse(&connect, &[]); let connection = net() .client(moq.client.clone()) .expect("client") - .with_publisher(origin.consume()) + .with_publisher(publisher.origin.consume()) .with_reconnect(false) .connect(fixture.url.clone()) .established() @@ -193,21 +252,11 @@ mod tests { .expect("publisher connects"); Self { - connect, http: fixture.http, - _publisher: (connection, broadcast, vec![data, empty, live], open), + _publisher: (connection, publisher), } } - /// Run `moq --broadcast demo fetch ` with `timeout`, returning - /// the outcome and what it wrote. - async fn fetch(&self, args: &[&str], timeout: Duration) -> (anyhow::Result<()>, Vec) { - let (moq, args) = parse(&self.connect, args); - let mut out = Vec::new(); - let result = super::fetch(&moq, &args, &net(), Instant::now() + timeout, &mut out).await; - (result, out) - } - /// The relay's HTTP `/fetch` status and body for the same group. async fn curl(&self, query: &str) -> (u16, Vec) { let response = reqwest::get(format!("http://{}/fetch/demo/data{query}", self.http)) @@ -245,7 +294,7 @@ mod tests { const ENV: &[&str] = &["MOQ_CONNECT", "MOQ_HOP"]; /// A known sequence prints its frames back to back, the same bytes as `/fetch`. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_sequence_prints_its_exact_bytes() { let _env = EnvGuard::clear(ENV); let fixture = Fixture::new().await; @@ -253,11 +302,10 @@ mod tests { let (result, out) = fixture.fetch(&["data", "--group", "1"], TIMEOUT).await; result.expect("fetch"); assert_eq!(out, frames(1).concat()); - assert_eq!(fixture.curl("?group=1").await, (200, out)); } /// No `--group` reads the newest group, as `/fetch` does by default. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_default_is_the_newest_group() { let _env = EnvGuard::clear(ENV); let fixture = Fixture::new().await; @@ -265,12 +313,11 @@ mod tests { let (result, out) = fixture.fetch(&["data"], TIMEOUT).await; result.expect("fetch"); assert_eq!(out, frames(2).concat()); - assert_eq!(fixture.curl("").await, (200, out)); } /// A missing sequence fails the lookup itself, before any output, as `/fetch` /// answers 404 rather than starting a body. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_missing_sequence_fails() { let _env = EnvGuard::clear(ENV); let fixture = Fixture::new().await; @@ -279,16 +326,15 @@ mod tests { let err = result.expect_err("group 99 does not exist"); assert_eq!(err.to_string(), "group 99 of `data` not found", "{err:#}"); assert!(out.is_empty()); - assert_eq!(fixture.curl("?group=99").await, (404, Vec::new())); } /// A track with no group never resolves "newest", so the deadline ends it. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_lookup_times_out() { let _env = EnvGuard::clear(ENV); let fixture = Fixture::new().await; - let (result, out) = fixture.fetch(&["empty"], Duration::from_millis(500)).await; + let (result, out) = fixture.fetch(&["empty"], TIMEOUT).await; let err = result.expect_err("no group ever arrives"); assert!(err.to_string().contains("timed out"), "{err:#}"); assert!(out.is_empty()); @@ -296,21 +342,80 @@ mod tests { /// A group that never finishes is cut off by the same deadline, after the frames /// that did arrive. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_frame_read_times_out() { let _env = EnvGuard::clear(ENV); let fixture = Fixture::new().await; - let (result, out) = fixture - .fetch(&["live", "--group", "0"], Duration::from_millis(500)) - .await; + let (result, out) = fixture.fetch(&["live", "--group", "0"], TIMEOUT).await; let err = result.expect_err("the group never finishes"); assert!(err.to_string().contains("timed out"), "{err:#}"); assert_eq!(out, b"first"); } - /// Each `--json` line is the whole record and decodes to the frame's bytes. + /// Discovery and reads share the deadline; setup cannot grant a fresh budget. + #[tokio::test(start_paused = true)] + async fn setup_and_reads_share_one_deadline() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + let (_, args) = parse(&[], &["live", "--group", "0"]); + let started = Instant::now(); + let mut out = Vec::new(); + let result = fetch_from( + Request { + args: &args, + path: "demo", + deadline: started + TIMEOUT, + }, + async { + // Advance simulated discovery time before handing back the ready origin. + tokio::time::advance(TIMEOUT / 2).await; + Ok((fixture.origin.clone(), ())) + }, + &mut out, + ) + .await; + assert_eq!(result.unwrap_err().to_string(), "fetch timed out"); + assert_eq!(out, b"first"); + assert_eq!(Instant::now() - started, TIMEOUT); + } + + /// A setup that never establishes is bounded by that same deadline. + #[tokio::test(start_paused = true)] + async fn setup_is_bounded() { + let _env = EnvGuard::clear(ENV); + let (_, args) = parse(&[], &["data"]); + let started = Instant::now(); + let result = fetch_from::<()>( + Request { + args: &args, + path: "demo", + deadline: started + TIMEOUT, + }, + std::future::pending(), + &mut Vec::new(), + ) + .await; + assert_eq!(result.unwrap_err().to_string(), "fetch timed out"); + assert_eq!(Instant::now() - started, TIMEOUT); + } + + /// The real HTTP endpoint agrees with the deterministic operation tests. #[tokio::test] + async fn real_relay_matches_http_fetch() { + let _env = EnvGuard::clear(ENV); + for (query, expected) in [ + ("?group=1", (200, frames(1).concat())), + ("", (200, frames(2).concat())), + ("?group=99", (404, Vec::new())), + ] { + let fixture = NetworkFixture::new().await; + assert_eq!(fixture.curl(query).await, expected); + } + } + + /// Each `--json` line is the whole record and decodes to the frame's bytes. + #[tokio::test(start_paused = true)] async fn json_lines_decode_to_the_frames() { let _env = EnvGuard::clear(ENV); let fixture = Fixture::new().await; From 8aa60171f285431949f073514424cebe08a2657d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 15:29:42 -0700 Subject: [PATCH 04/28] test(tokio): make broadcast transport race deterministic (#4664) Co-authored-by: GPT-6 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/broadcast-race.md | 22 ----- rs/moq-tokio/src/client.rs | 108 +++++++++++++++++++++++ rs/moq-tokio/tests/broadcast.rs | 82 +---------------- rs/moq-tokio/tests/worker.rs | 28 ++---- 5 files changed, 116 insertions(+), 125 deletions(-) delete mode 100644 quest/m1/test-flakes-2/broadcast-race.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index c48c60452bf..94b6950a7aa 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -32,7 +32,6 @@ Public API: none. Wire: none. - [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean -- [Broadcast race](/quest/m1/test-flakes-2/broadcast-race.md) - the QUIC-wins race test binds no shared port and has a deterministic winner - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed - [Shaper virtual time](/quest/m1/test-flakes-2/shaper-virtual-time.md) - `moq-shaper` tests judge seeded decisions on paused time, not on wall-clock delivery under load - [Scoped WARN capture](/quest/m1/test-flakes-2/warn-capture.md) - the drop-unfinished tests count only their own WARNs diff --git a/quest/m1/test-flakes-2/broadcast-race.md b/quest/m1/test-flakes-2/broadcast-race.md deleted file mode 100644 index 36b0ddf1850..00000000000 --- a/quest/m1/test-flakes-2/broadcast-race.md +++ /dev/null @@ -1,22 +0,0 @@ -# [S] Broadcast race without a shared port - -## Goal - -moq-tokio `broadcast_race_quic_wins` binds no port it didn't reserve and -has a deterministic winner. - -## Plan - -It binds TCP `:0` and then UDP on the same number, which nothing reserves: -the collision [#4084](https://github.com/moq-dev/moq/pull/4084) removed from -its sibling after [#4055](https://github.com/moq-dev/moq/pull/4055) papered -over it with a retry. - -The test shares one port only so both transports sit behind one URL. -Separate `:0` ports fix the bind collision but not the race itself: with -`websocket.delay = 0` either arm can legitimately win under load. Make the -order deterministic the way #4084 did, holding the WebSocket arm until QUIC -has connected, rather than asserting on a real race; no retry. #4084's -follow-ups (`tests/reconnect.rs` `spawn_server`, `tests/worker.rs` -`free_udp_port`) are the same probe-and-rebind pattern; fix them here if -cheap. diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index d7c517d2582..654e2f25137 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -1332,6 +1332,114 @@ mod tests { accepted.await.unwrap().expect("server handshake failed"); } + /// QUIC wins while the fallback handshake is held until QUIC connects. + /// Each transport owns its ephemeral port; this exercises the same race as + /// connect_inner without depending on which real handshake runs faster. + #[cfg(all(feature = "websocket", feature = "noq"))] + #[tracing_test::traced_test] + #[tokio::test] + async fn broadcast_race_quic_wins() { + let pub_origin = crate::origin::spawn(); + let broadcast = pub_origin.create_broadcast("test").expect("failed to create broadcast"); + broadcast + .announce(Default::default()) + .expect("failed to create broadcast"); + let track = broadcast.create_track("video", None).expect("failed to create track"); + let mut group = track.append_group().expect("failed to append group"); + group + .write_frame(crate::moq_net::Timestamp::ZERO, b"hello".as_ref()) + .expect("failed to write frame"); + group.finish().expect("failed to finish group"); + + let ws_listener = crate::websocket::Listener::bind("[::]:0".parse().unwrap()) + .await + .expect("failed to bind WebSocket listener"); + let ws_port = ws_listener.local_addr().expect("failed to get ws addr").port(); + let (connected, quic_connected) = tokio::sync::oneshot::channel(); + let websocket_handle = tokio::spawn(async move { + quic_connected.await.expect("QUIC connected"); + ws_listener.accept().await + }); + + let mut server_config = crate::listen::Config { + bind: Some("[::]:0".parse().unwrap()), + ..Default::default() + }; + server_config.tls.generate = vec!["localhost".into()]; + let server = server_config.init(Default::default()).expect("failed to init server"); + let mut server = server.listen().await.expect("failed to listen"); + let quic_port = server.local_addr().expect("failed to get QUIC addr").port(); + + let sub_origin = crate::origin::spawn(); + let sub_consumer = sub_origin.consume(); + let mut announcements = sub_consumer.announced(); + let mut client_config = crate::connect::Config::default(); + client_config.tls.insecure = Some(true); + // Dial both arms immediately; the fallback handshake is gated by QUIC. + client_config.websocket.delay = std::time::Duration::ZERO; + + let client = client_config.init(Default::default()).expect("failed to init client"); + let quic_addr: crate::connect::Addr = Url::parse(&format!("https://localhost:{quic_port}")).unwrap().into(); + let ws_addr: crate::connect::Addr = Url::parse(&format!("http://localhost:{ws_port}")).unwrap().into(); + + // Keep this aligned with the newest default Lite version, as in tests/broadcast.rs. + let expected_version: moq_net::Version = "moq-lite-06".parse().expect("invalid version"); + + let server_handle = tokio::spawn(async move { + let request = server.accept().await.expect("no incoming connection"); + assert_eq!( + request.transport(), + crate::server::Transport::Quic, + "expected the QUIC listener", + ); + let session = request.with_publisher(&pub_origin).ok().await?; + assert_eq!(session.version(), expected_version, "server negotiated stale version"); + let _broadcast = broadcast; + let _track = track; + let _ = session.closed().await; + Ok::<_, anyhow::Error>(()) + }); + + let client = client.with_subscriber(sub_origin); + let noq = client.noq.as_ref().expect("QUIC backend"); + let quic = async { + let session = noq.connect(&client.tls, quic_addr, &client.versions).await?; + connected.send(()).expect("fallback listener alive"); + Ok::<_, crate::Error>(crate::transport::Session::new(session)) + }; + let cc = client + .race_moq_connect(&client.moq, ws_addr, quic) + .await + .expect("client connect failed"); + + assert_eq!(cc.version(), expected_version, "client negotiated stale version"); + websocket_handle.abort(); + let _ = websocket_handle.await; + + let update = announcements.next().await.expect("origin closed"); + assert_eq!(update.prefix.as_str(), "test"); + assert!(update.kind.is_active(), "expected announcement"); + let broadcast = sub_consumer + .request_broadcast("test") + .await + .expect("broadcast resolves"); + let mut track = broadcast + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + let mut group = track.recv_group().await.expect("receive group").expect("track open"); + let frame = group.read_frame().await.expect("read frame").expect("group open"); + assert_eq!(&frame.payload[..], b"hello"); + + drop(cc); + server_handle + .await + .expect("server task panicked") + .expect("server task failed"); + } + #[cfg(all(feature = "websocket", feature = "noq"))] #[tokio::test] async fn race_transport_connect_reports_auth_when_both_refuse() { diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 2ab11f6844a..79460ac601c 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -372,7 +372,7 @@ async fn lite05_fetch_roundtrip(scheme: &str) { #[tokio::test] async fn broadcast_moq_lite_05_fetch_webtransport() { // Exercises the WebTransport path; lite-05 is forced via config on both ends. - // The raw-QUIC ALPN path is covered by broadcast_race_quic_wins. + // The raw-QUIC ALPN path is covered by client::tests::broadcast_race_quic_wins. lite05_fetch_roundtrip("https").await; } @@ -2362,86 +2362,6 @@ async fn broadcast_websocket_uses_newest_version() { .expect("server task failed"); } -/// Regression guard for the QUIC vs WebSocket race. With both transports -/// reachable at the same URL, QUIC must win, since it's lower-latency and -/// has direct ALPN negotiation. A WebSocket win here means QUIC silently -/// regressed (and would also tend to drag the version down to Lite02 on -/// older relays). We bind WebSocket TCP and QUIC UDP to the same port, -/// then disable the head start so the race is genuine. -#[tracing_test::traced_test] -#[tokio::test] -async fn broadcast_race_quic_wins() { - let pub_origin = moq_tokio::origin::spawn(); - let broadcast = pub_origin.create_broadcast("test").expect("failed to create broadcast"); - broadcast - .announce(Default::default()) - .expect("failed to create broadcast"); - let track = broadcast.create_track("video", None).expect("failed to create track"); - let mut group = track.append_group().expect("failed to append group"); - group - .write_frame(moq_tokio::moq_net::Timestamp::ZERO, b"hello".as_ref()) - .expect("failed to write frame"); - group.finish().expect("failed to finish group"); - - // Bind WebSocket TCP first to pick a random port, then bind QUIC UDP to - // the same port. UDP and TCP live in separate kernel namespaces, so this - // works on every supported platform. - let ws_listener = moq_tokio::websocket::Listener::bind("[::]:0".parse().unwrap()) - .await - .expect("failed to bind WebSocket listener"); - let port = ws_listener.local_addr().expect("failed to get ws addr").port(); - - let mut server_config = moq_tokio::listen::Config::default(); - server_config.bind = Some(format!("[::]:{port}").parse().unwrap()); - server_config.tls.generate = vec!["localhost".into()]; - - let mut config = moq_tokio::server::Config::default(); - config.listen = server_config; - config.websocket = Some(ws_listener); - let server = config.init().expect("failed to init server"); - let mut server = server.listen().await.expect("failed to listen"); - - let sub_origin = moq_tokio::origin::spawn(); - let mut client_config = moq_tokio::connect::Config::default(); - client_config.tls.insecure = Some(true); - // Zero head start: QUIC has to win on its own merit, not by penalising WS. - client_config.websocket.delay = Duration::ZERO; - - let client = client_config.init(Default::default()).expect("failed to init client"); - let url: url::Url = format!("https://localhost:{port}").parse().unwrap(); - - let expected_version: moq_net::Version = NEWEST_LITE.parse().expect("invalid version"); - - let server_handle = tokio::spawn(async move { - let request = server.accept().await.expect("no incoming connection"); - assert_eq!( - request.transport(), - moq_tokio::server::Transport::Quic, - "QUIC lost the race to WebSocket with both reachable", - ); - let session = request.with_publisher(&pub_origin).ok().await?; - assert_eq!(session.version(), expected_version, "server negotiated stale version"); - let _broadcast = broadcast; - let _track = track; - let _ = session.closed().await; - Ok::<_, anyhow::Error>(()) - }); - - let client = client.with_subscriber(sub_origin); - let (_client, cc) = tokio::time::timeout(TIMEOUT, connect_once(client, url)) - .await - .expect("client connect timed out") - .expect("client connect failed"); - - assert_eq!(cc.version(), Some(expected_version), "client negotiated stale version"); - - drop(cc); - server_handle - .await - .expect("server task panicked") - .expect("server task failed"); -} - /// Regression guard: a Tokio runtime's session machine must run under the span /// active at accept time (e.g. a relay's /// per-connection span), or every log line moq-net emits from inside it diff --git a/rs/moq-tokio/tests/worker.rs b/rs/moq-tokio/tests/worker.rs index 811b060d584..cdf2348d377 100644 --- a/rs/moq-tokio/tests/worker.rs +++ b/rs/moq-tokio/tests/worker.rs @@ -11,17 +11,6 @@ use moq_tokio::worker::{self, Workers}; const WORKERS: u16 = 4; -/// A UDP port nothing is bound to. -/// -/// Only for the port-lock tests, where the first group names its port. -/// Everything else binds `:0` and reads it back. -fn free_udp_port() -> u16 { - let probe = UdpSocket::bind("127.0.0.1:0").expect("bind probe"); - let port = probe.local_addr().expect("local addr").port(); - drop(probe); - port -} - /// The socket inodes this process holds bound to `addr`, of which there must be /// at least one: every UDP socket, but only a TCP listener. /// @@ -330,10 +319,9 @@ async fn an_occupied_port_is_refused() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); - let port = free_udp_port(); - let workers = - bind_workers(listen_config(&cert, &key, port), Default::default(), config(WORKERS)).expect("bind workers"); + bind_workers(listen_config(&cert, &key, 0), Default::default(), config(WORKERS)).expect("bind workers"); + let port = workers.local_addr().port(); let err = bind_workers(listen_config(&cert, &key, port), Default::default(), config(WORKERS)) .expect_err("a second group must not join the first"); @@ -360,11 +348,10 @@ async fn the_other_wildcard_spelling_is_refused() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); - let port = free_udp_port(); - - let mut v4 = listen_config(&cert, &key, port); - v4.bind = Some(format!("0.0.0.0:{port}").parse().unwrap()); + let mut v4 = listen_config(&cert, &key, 0); + v4.bind = Some("0.0.0.0:0".parse().unwrap()); let workers = bind_workers(v4, Default::default(), config(WORKERS)).expect("bind v4 wildcard workers"); + let port = workers.local_addr().port(); let mut v6 = listen_config(&cert, &key, port); v6.bind = Some(format!("[::]:{port}").parse().unwrap()); @@ -390,10 +377,9 @@ async fn a_shared_port_is_refused_across_addresses() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); - let port = free_udp_port(); - let workers = - bind_workers(listen_config(&cert, &key, port), Default::default(), config(WORKERS)).expect("bind workers"); + bind_workers(listen_config(&cert, &key, 0), Default::default(), config(WORKERS)).expect("bind workers"); + let port = workers.local_addr().port(); // A distinct loopback address: no bind conflict with 127.0.0.1, only the // shared port. From baeaf3ec38904e0a36fc6f4f71e4fde8dd689528 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 16:07:30 -0700 Subject: [PATCH 05/28] test(shaper): run the unit tests on paused time over in-memory sockets (#4682) Co-authored-by: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/shaper-virtual-time.md | 18 ---- rs/moq-shaper/src/lib.rs | 49 +++++----- rs/moq-shaper/src/mem.rs | 93 +++++++++++++++++++ 4 files changed, 120 insertions(+), 41 deletions(-) delete mode 100644 quest/m1/test-flakes-2/shaper-virtual-time.md create mode 100644 rs/moq-shaper/src/mem.rs diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 94b6950a7aa..80843a51653 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -33,7 +33,6 @@ Public API: none. Wire: none. - [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed -- [Shaper virtual time](/quest/m1/test-flakes-2/shaper-virtual-time.md) - `moq-shaper` tests judge seeded decisions on paused time, not on wall-clock delivery under load - [Scoped WARN capture](/quest/m1/test-flakes-2/warn-capture.md) - the drop-unfinished tests count only their own WARNs - [moq-mux debounce clock](/quest/m1/test-flakes-2/mux-debounce-clock.md) - the TS export debounce test advances on the paused clock - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time diff --git a/quest/m1/test-flakes-2/shaper-virtual-time.md b/quest/m1/test-flakes-2/shaper-virtual-time.md deleted file mode 100644 index 3fea8a4fb46..00000000000 --- a/quest/m1/test-flakes-2/shaper-virtual-time.md +++ /dev/null @@ -1,18 +0,0 @@ -# [S] moq-shaper tests on virtual time - -## Goal - -The `moq-shaper` unit tests pass under any host load: their verdicts come -from the seeded decisions, not from how fast the kernel delivers datagrams. -`reorder_and_jitter_overtake` and `the_seed_reproduces_the_losses` saw every -datagram lost during a slowed full-workspace run, because `round_trip` stops -at the first 200 ms of silence on a real socket. - -## Plan - -- Drive the tests on paused tokio time (or an injected clock) so delay, - jitter, and the read deadline advance deterministically. -- Watch out: paused time auto-advances while the runtime idles, which can - fire a timer before a real UDP datagram lands. If the real sockets fight - the paused clock, separate the scheduling logic from the socket I/O and test - it directly, keeping one socket smoke test with a generous deadline. diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 4e9ca01e5ac..66e359f0339 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -30,14 +30,20 @@ use std::{ time::Duration, }; +#[cfg(test)] +mod mem; mod preset; pub use preset::Preset; use anyhow::Context; +#[cfg(test)] +use mem::UdpSocket; use rand::{RngExt, SeedableRng, rngs::Xoshiro256PlusPlus}; use serde::{Deserialize, Serialize}; -use tokio::{net::UdpSocket, sync::mpsc, task::JoinSet, time::Instant}; +#[cfg(not(test))] +use tokio::net::UdpSocket; +use tokio::{sync::mpsc, task::JoinSet, time::Instant}; /// How one direction of the path treats each datagram. /// @@ -995,7 +1001,7 @@ mod tests { got } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn forwards_both_ways_untouched() { let (shaper, client) = setup(1, Profile::default(), Profile::default()).await; let got = round_trip(&client, 50).await; @@ -1022,7 +1028,7 @@ mod tests { assert_eq!(start.elapsed(), Duration::ZERO); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_seed_reproduces_the_losses() { let lossy = Profile { loss: 0.3, @@ -1042,7 +1048,7 @@ mod tests { assert_eq!(first_got.len() as u64, 200 - first.stats().up.lost); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn reorder_and_jitter_overtake() { let shuffled = Profile { delay: Duration::from_millis(20), @@ -1062,7 +1068,7 @@ mod tests { assert!(stats.up.reordered > 0, "{stats}"); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_rate_limit_queues_then_drops() { // 100 datagrams of 4 bytes is 3200 bits: at 8 kbit/s they need 400ms, // and the queue only holds 100ms of it. @@ -1085,7 +1091,7 @@ mod tests { assert!(got.windows(2).all(|pair| pair[0] < pair[1]), "{got:?}"); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_rate_limit_wider_than_the_traffic_passes_everything() { let wide = Profile { rate: Some(Rate { @@ -1102,7 +1108,7 @@ mod tests { assert_eq!(shaper.verify().unwrap().up.overflowed, 0); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_burst_counts_the_datagram_itself() { // A burst of one datagram lets the first out at once and holds the second. let one = Profile { @@ -1184,7 +1190,7 @@ mod tests { assert_eq!(unapplied(&phases, &stats), ["loss"]); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn an_invalid_profile_is_refused() { let refused = |bad: Profile, why: &'static str| async move { let err = Shaper::bind(Config { @@ -1268,7 +1274,7 @@ mod tests { steps: Vec::new(), }; - #[tokio::test] + #[tokio::test(start_paused = true)] async fn only_the_gaussian_model_keeps_the_order() { let jittery = Profile { delay: Duration::from_millis(20), @@ -1387,7 +1393,7 @@ mod tests { assert!(first_counters.lost > 0 && first_counters.reordered > 0); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_gaussian_sigma_may_exceed_the_delay() { // The uniform model refuses this; a gaussian clamps its draw at zero. let wide = Profile { @@ -1457,7 +1463,7 @@ mod tests { got } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_shared_path_keeps_the_order_across_clients() { let jittery = Profile { delay: Duration::from_millis(20), @@ -1530,7 +1536,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_batch_releases_on_count_and_on_the_window() { let socket = Arc::new(UdpSocket::bind(LOCALHOST).await.unwrap()); let ms = |ms| Duration::from_millis(ms); @@ -1609,7 +1615,7 @@ mod tests { assert_eq!(tally.batched.load(Ordering::Relaxed), 0); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_batch_releases_datagrams_together() { let (shaper, client) = shaped(|config| Setup { up: batched(7, Duration::from_millis(160)), @@ -1644,7 +1650,7 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_window_releases_a_batch_that_never_fills() { let (shaper, client) = shaped(|config| Setup { up: batched(100, Duration::from_millis(80)), @@ -1659,7 +1665,7 @@ mod tests { assert_eq!(shaper.verify().unwrap().up.delayed, 3); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_shared_batch_fills_from_every_client() { // Three datagrams from each of two clients, into batches of six: only a // shared path fills one, and separate links wait out the window. @@ -1684,7 +1690,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_batch_needs_a_count_and_a_window() { let refused = |options: Options, why: &'static str| async move { let config = Config { @@ -1812,7 +1818,7 @@ mod tests { assert_eq!(lost(start + secs(60)), 0, "the step back never cleared"); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_step_gets_worse_part_way_through() { let ms = Duration::from_millis; let (shaper, client) = shaped(|config| Setup { @@ -1849,7 +1855,7 @@ mod tests { shaper.verify().unwrap(); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_step_the_run_never_reached_is_unapplied() { let (shaper, client) = shaped(|config| Setup { down: stepped(vec![Step { @@ -1866,8 +1872,7 @@ mod tests { assert!(format!("{err:#}").contains("the down step at 60s"), "{err:#}"); } - // Real sockets, so the wall clock: a paused one could jump past a step while a datagram is still in the kernel. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_step_no_datagram_saw_is_unapplied() { let ms = Duration::from_millis; let (shaper, client) = shaped(|config| Setup { @@ -1909,7 +1914,7 @@ mod tests { assert!(!err.contains("the up step at 200ms"), "{err}"); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_profile_no_datagram_saw_before_its_step_is_unapplied() { let ms = Duration::from_millis; let (shaper, client) = shaped(|config| Setup { @@ -1954,7 +1959,7 @@ mod tests { assert_eq!(owed(&mut link, start + Duration::from_secs(1), &[16]), [Duration::ZERO]); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn steps_that_would_be_skipped_or_do_nothing_are_refused() { let refused = |steps: Vec, why: &'static str| async move { let config = Config { diff --git a/rs/moq-shaper/src/mem.rs b/rs/moq-shaper/src/mem.rs new file mode 100644 index 00000000000..bf7e8964364 --- /dev/null +++ b/rs/moq-shaper/src/mem.rs @@ -0,0 +1,93 @@ +//! An in-memory stand-in for [`tokio::net::UdpSocket`], so the unit tests run +//! on a paused clock. A paused clock jumps to the next timer whenever the +//! runtime idles, which would fire timeouts and steps while a real datagram +//! sat in the kernel; a datagram here is delivered inside the runtime, so the +//! clock only moves once nothing is in flight. `tests/cli.rs` covers real sockets. + +use std::{ + collections::HashMap, + io, + net::SocketAddr, + sync::{ + LazyLock, Mutex, OnceLock, + atomic::{AtomicU16, Ordering}, + }, +}; + +use tokio::sync::mpsc; + +type Datagram = (SocketAddr, Vec); + +/// Every bound socket by port, which is the whole address on this network. +static PORTS: LazyLock>>> = LazyLock::new(Default::default); +static NEXT_PORT: AtomicU16 = AtomicU16::new(1); + +pub struct UdpSocket { + addr: SocketAddr, + peer: OnceLock, + queue: tokio::sync::Mutex>, +} + +impl UdpSocket { + pub async fn bind(mut addr: SocketAddr) -> io::Result { + if addr.port() == 0 { + addr.set_port(NEXT_PORT.fetch_add(1, Ordering::Relaxed)); + } + let (send, queue) = mpsc::unbounded_channel(); + let mut ports = PORTS.lock().unwrap(); + if ports.contains_key(&addr.port()) { + return Err(io::ErrorKind::AddrInUse.into()); + } + ports.insert(addr.port(), send); + Ok(Self { + addr, + peer: OnceLock::new(), + queue: tokio::sync::Mutex::new(queue), + }) + } + + pub fn local_addr(&self) -> io::Result { + Ok(self.addr) + } + + pub async fn connect(&self, peer: SocketAddr) -> io::Result<()> { + self.peer.set(peer).map_err(|_| io::ErrorKind::AlreadyExists.into()) + } + + /// Like UDP, a datagram to a port nobody bound is lost without an error. + pub async fn send_to(&self, buf: &[u8], dest: SocketAddr) -> io::Result { + if let Some(port) = PORTS.lock().unwrap().get(&dest.port()) { + let _ = port.send((self.addr, buf.to_vec())); + } + Ok(buf.len()) + } + + pub async fn recv_from(&self, buf: &mut [u8]) -> io::Result<(usize, SocketAddr)> { + // The port's sender lives until this socket drops, so the queue never ends. + let (from, datagram) = self.queue.lock().await.recv().await.expect("bound port"); + let size = datagram.len().min(buf.len()); + buf[..size].copy_from_slice(&datagram[..size]); + Ok((size, from)) + } + + pub async fn send(&self, buf: &[u8]) -> io::Result { + self.send_to(buf, *self.peer.get().expect("not connected")).await + } + + /// Only what the connected peer sends, as a connected UDP socket filters. + pub async fn recv(&self, buf: &mut [u8]) -> io::Result { + let peer = *self.peer.get().expect("not connected"); + loop { + let (size, from) = self.recv_from(buf).await?; + if from == peer { + return Ok(size); + } + } + } +} + +impl Drop for UdpSocket { + fn drop(&mut self) { + PORTS.lock().unwrap().remove(&self.addr.port()); + } +} From c63c01eba7520095f27e739d48c2645bc683190f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 16:51:52 -0700 Subject: [PATCH 06/28] quest: claim test-flakes-2/warn-capture Co-Authored-By: Claude Opus 5.5 From 113bb17931b0031a5bb5b275983b4f3f20107c04 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 16:51:56 -0700 Subject: [PATCH 07/28] quest: claim mux-debounce-clock From d965302e692ec1f4492aef27cafc504b5c72307c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 17:03:55 -0700 Subject: [PATCH 08/28] moq-mux: run the TS SI debounce on the tokio clock The SI capture's cut debounce read crate::Clock, which samples std::time::Instant, so debounce_opens_without_a_media_clock slept 1.2 s of real time despite start_paused. Track the last cut as a web_async Instant (tokio's on native, so the paused clock drives it; wasmtimer in the browser) and advance the paused clock in the test instead. Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/mux-debounce-clock.md | 15 --------------- rs/moq-mux/src/container/ts/export_test.rs | 8 ++++---- rs/moq-mux/src/container/ts/si.rs | 15 +++++++-------- 4 files changed, 11 insertions(+), 28 deletions(-) delete mode 100644 quest/m1/test-flakes-2/mux-debounce-clock.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 80843a51653..2e7936f95b6 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -34,7 +34,6 @@ Public API: none. Wire: none. - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed - [Scoped WARN capture](/quest/m1/test-flakes-2/warn-capture.md) - the drop-unfinished tests count only their own WARNs -- [moq-mux debounce clock](/quest/m1/test-flakes-2/mux-debounce-clock.md) - the TS export debounce test advances on the paused clock - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time - [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial diff --git a/quest/m1/test-flakes-2/mux-debounce-clock.md b/quest/m1/test-flakes-2/mux-debounce-clock.md deleted file mode 100644 index e29f84934b2..00000000000 --- a/quest/m1/test-flakes-2/mux-debounce-clock.md +++ /dev/null @@ -1,15 +0,0 @@ -# [XS] moq-mux debounce test on the paused clock - -## Goal - -moq-mux `container::ts::export_test::debounce_opens_without_a_media_clock` -advances on the paused clock and sleeps no real time. - -## Plan - -It died with SIGTERM once in a combined run. It is marked -`start_paused = true` but sleeps 1.2 s of real time, because the debounce -reads `crate::Clock`, which uses `std::time::Instant`, so the paused tokio -clock never reaches it. Let the test drive `crate::Clock`'s time (a tokio -`Instant` under test, or an injected source) so the window passes on the -paused clock, and drop the real sleep. diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index a79f3dcec3f..6924c99062d 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -5016,8 +5016,8 @@ async fn contiguous_same_version_commit_replaces_stale_sections() { } /// The cut debounce runs on the host clock, so a revision publishes even when no -/// media ever advances a PTS (an audio-only or SI-only input). This test spends -/// real wall time on the debounce window; media timestamps stay pinned at zero +/// media ever advances a PTS (an audio-only or SI-only input). The paused tokio +/// clock steps over the debounce window; media timestamps stay pinned at zero /// throughout, which is exactly the case a media-clock debounce wedges on. #[tokio::test(start_paused = true)] async fn debounce_opens_without_a_media_clock() { @@ -5038,8 +5038,8 @@ async fn debounce_opens_without_a_media_clock() { .unwrap(); assert_eq!(track.latest(), Some(0), "a revision inside the window is held"); - // ...and publishes once the window passes in *real* time, no finish, no PTS. - std::thread::sleep(Duration::from_millis(1200)); + // ...and publishes once the window passes on the host clock, no finish, no PTS. + tokio::time::advance(Duration::from_millis(1200)).await; rig.import .decode(&BytesMut::from(&si_packet_cc(0x0011, &sdt(1, 0xbb), 3)[..])) .unwrap(); diff --git a/rs/moq-mux/src/container/ts/si.rs b/rs/moq-mux/src/container/ts/si.rs index b00f611ee4c..f11f2602ac8 100644 --- a/rs/moq-mux/src/container/ts/si.rs +++ b/rs/moq-mux/src/container/ts/si.rs @@ -29,6 +29,8 @@ use std::collections::{BTreeMap, HashMap}; use std::time::Duration; +use web_async::time::Instant; + use bytes::Bytes; use moq_net::Timestamp; @@ -213,8 +215,8 @@ struct Entry { /// The entry appears in the catalog's `mpegts.si` map: exactly while it is /// `populated`, so a subscriber never finds an empty track. advertised: bool, - /// Host-clock micros of the last cut ([`DEBOUNCE`]); `None` until the first. - last_cut: Option, + /// Host-clock instant of the last cut ([`DEBOUNCE`]); `None` until the first. + last_cut: Option, } impl Entry { @@ -345,7 +347,7 @@ impl Entry { broadcast: &moq_net::broadcast::Producer, (pid, table_id): (u16, u8), pts: Timestamp, - now: u128, + now: Instant, ) -> anyhow::Result<()> { let frames: Vec> = self .active @@ -389,8 +391,6 @@ pub(super) struct Capture { broadcast: moq_net::broadcast::Producer, catalog: crate::catalog::Producer, entries: BTreeMap<(u16, u8), Entry>, - /// Host clock driving the cut debounce (see [`DEBOUNCE`]). - clock: crate::Clock, /// The selected program_number, which is the service_id of the only service /// carried; `None` captures every service. program: Option, @@ -402,7 +402,6 @@ impl Capture { broadcast, catalog, entries: BTreeMap::new(), - clock: crate::Clock::new(), program: None, } } @@ -467,14 +466,14 @@ impl Capture { /// the host clock ([`DEBOUNCE`]), and a first snapshot is never delayed. `force` /// overrides the debounce, for end of stream. pub fn flush(&mut self, pts: Timestamp, force: bool) -> anyhow::Result<()> { - let now = self.clock.now().as_micros(); + let now = Instant::now(); for (key, entry) in self.entries.iter_mut() { if !entry.dirty { continue; } if !force && let Some(last) = entry.last_cut - && now.saturating_sub(last) < DEBOUNCE.as_micros() + && now.saturating_duration_since(last) < DEBOUNCE { continue; } From c8db5fb6d4887f0fe5e38e44ca9f777a52779832 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 17:12:27 -0700 Subject: [PATCH 09/28] moq-net: count drop WARNs through a global subscriber, per thread A scoped with_default subscriber let a parallel test reach the WARN callsite first on a thread with no subscriber, caching its interest as never for the whole process, so drop_unfinished_warns saw 0 WARNs. Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/warn-capture.md | 14 --- rs/moq-net/src/model/group.rs | 2 +- rs/moq-net/src/model/test_tracing.rs | 136 +++++++++++++++---------- rs/moq-net/src/model/track.rs | 2 +- 5 files changed, 87 insertions(+), 68 deletions(-) delete mode 100644 quest/m1/test-flakes-2/warn-capture.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 80843a51653..bfe1ae5b56a 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -33,7 +33,6 @@ Public API: none. Wire: none. - [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed -- [Scoped WARN capture](/quest/m1/test-flakes-2/warn-capture.md) - the drop-unfinished tests count only their own WARNs - [moq-mux debounce clock](/quest/m1/test-flakes-2/mux-debounce-clock.md) - the TS export debounce test advances on the paused clock - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time - [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial diff --git a/quest/m1/test-flakes-2/warn-capture.md b/quest/m1/test-flakes-2/warn-capture.md deleted file mode 100644 index d6f928e0c43..00000000000 --- a/quest/m1/test-flakes-2/warn-capture.md +++ /dev/null @@ -1,14 +0,0 @@ -# [XS] Scoped WARN capture - -## Goal - -moq-net `model::group::test::drop_unfinished_warns` and the `model::track` -test of the same name count only the WARNs their own code emits. - -## Plan - -Both count WARNs through a global tracing capture, so another test's WARN, -or a missed one, changes the count -([#4104](https://github.com/moq-dev/moq/pull/4104)). Capture per test, with a -scoped subscriber or a filter on the test's own span, instead of a -process-global count. diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index bb725dcba1d..5dfa179541e 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -2035,7 +2035,7 @@ mod test { drop(writer); drop(producer); }); - assert!(warns >= 1, "unfinished drop must emit unfinished-producer WARN"); + assert_eq!(warns, 1, "unfinished drop must emit one unfinished-producer WARN"); } #[test] diff --git a/rs/moq-net/src/model/test_tracing.rs b/rs/moq-net/src/model/test_tracing.rs index 63caa9d852c..37fcdaec490 100644 --- a/rs/moq-net/src/model/test_tracing.rs +++ b/rs/moq-net/src/model/test_tracing.rs @@ -1,80 +1,114 @@ //! Tracing helpers shared by model tests. -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; +use std::cell::RefCell; +use std::sync::Once; use tracing::field::{Field, Visit}; use tracing::span::{Attributes, Id, Record}; use tracing::{Event, Level, Metadata, Subscriber}; -/// Count WARN events whose `message` field contains `expected_message` while running `f`. +/// The capture active on this thread: the message to match and how many WARNs matched. +struct Capture { + expected: String, + hits: usize, +} + +thread_local! { + static CAPTURE: RefCell> = const { RefCell::new(None) }; +} + +/// Count WARN events emitted on this thread whose `message` field contains `expected_message` +/// while running `f`. /// -/// Uses only the existing `tracing` dependency (no tracing-subscriber). +/// The subscriber is process-global rather than scoped with `with_default`: tracing caches each +/// callsite's interest process-wide, so a parallel test reaching the WARN first on a thread +/// without a scoped subscriber disables it for every thread. Counting per thread keeps another +/// test's WARNs out. pub(crate) fn count_drop_warnings(expected_message: &str, f: impl FnOnce()) -> usize { - struct Count { - hits: Arc, - expected: String, - } + static INSTALL: Once = Once::new(); + INSTALL.call_once(|| tracing::subscriber::set_global_default(Warns).expect("no other global subscriber")); - struct Msg<'a> { - expected: &'a str, - matched: bool, - } + let capture = Capture { + expected: expected_message.to_owned(), + hits: 0, + }; + let prev = CAPTURE.replace(Some(capture)); + assert!(prev.is_none(), "count_drop_warnings does not nest"); - impl Visit for Msg<'_> { - fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - if field.name() == "message" { - let s = format!("{value:?}"); - if s.contains(self.expected) { - self.matched = true; - } - } - } + f(); - fn record_str(&mut self, field: &Field, value: &str) { - if field.name() == "message" && value.contains(self.expected) { - self.matched = true; - } - } - } + CAPTURE.take().expect("capture still installed").hits +} - impl Subscriber for Count { - fn enabled(&self, metadata: &Metadata<'_>) -> bool { - *metadata.level() == Level::WARN - } +/// Routes every WARN to the capture on the emitting thread, if any. +struct Warns; - fn new_span(&self, _span: &Attributes<'_>) -> Id { - Id::from_u64(1) - } +impl Subscriber for Warns { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + *metadata.level() == Level::WARN + } - fn record(&self, _span: &Id, _values: &Record<'_>) {} + fn new_span(&self, _span: &Attributes<'_>) -> Id { + Id::from_u64(1) + } + + fn record(&self, _span: &Id, _values: &Record<'_>) {} - fn record_follows_from(&self, _span: &Id, _follows: &Id) {} + fn record_follows_from(&self, _span: &Id, _follows: &Id) {} - fn event(&self, event: &Event<'_>) { + fn event(&self, event: &Event<'_>) { + CAPTURE.with_borrow_mut(|capture| { + let Some(capture) = capture else { return }; let mut msg = Msg { - expected: &self.expected, + expected: &capture.expected, matched: false, }; event.record(&mut msg); if msg.matched { - self.hits.fetch_add(1, AtomicOrdering::SeqCst); + capture.hits += 1; } + }); + } + + fn enter(&self, _span: &Id) {} + + fn exit(&self, _span: &Id) {} +} + +struct Msg<'a> { + expected: &'a str, + matched: bool, +} + +impl Visit for Msg<'_> { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" && format!("{value:?}").contains(self.expected) { + self.matched = true; } + } - fn enter(&self, _span: &Id) {} + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "message" && value.contains(self.expected) { + self.matched = true; + } + } +} + +mod tests { + use super::count_drop_warnings; - fn exit(&self, _span: &Id) {} + fn probe() { + tracing::warn!("test_tracing probe"); } - let hits = Arc::new(AtomicUsize::new(0)); - let expected = expected_message.to_owned(); - tracing::subscriber::with_default( - Count { - hits: hits.clone(), - expected, - }, - f, - ); - hits.load(AtomicOrdering::SeqCst) + /// Another thread without a capture reaching the WARN first must neither disable it for this + /// thread nor be counted here. + #[test] + fn counts_only_this_threads_warns() { + let warns = count_drop_warnings("test_tracing probe", || { + std::thread::spawn(probe).join().unwrap(); + probe(); + }); + assert_eq!(warns, 1); + } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 097b162a4bc..3eb9bc8fa85 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6841,7 +6841,7 @@ mod test { drop(writer); drop(producer); }); - assert!(warns >= 1, "unfinished drop must emit unfinished-producer WARN"); + assert_eq!(warns, 1, "unfinished drop must emit one unfinished-producer WARN"); } #[tokio::test] From 1265ae09f385a991e0d35dd7836f9bada5bd2a3f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 17:17:18 -0700 Subject: [PATCH 10/28] quest: plan moq-mux live import clock and tracing capture audit Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 2 ++ .../m1/test-flakes-2/mux-live-import-clock.md | 28 +++++++++++++++++++ quest/m1/test-flakes-2/trace-capture-audit.md | 24 ++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 quest/m1/test-flakes-2/mux-live-import-clock.md create mode 100644 quest/m1/test-flakes-2/trace-capture-audit.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 80843a51653..04e35825c26 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -37,6 +37,8 @@ Public API: none. Wire: none. - [moq-mux debounce clock](/quest/m1/test-flakes-2/mux-debounce-clock.md) - the TS export debounce test advances on the paused clock - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time - [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial +- [moq-mux live import clock](/quest/m1/test-flakes-2/mux-live-import-clock.md) - the live import restart tests advance a paused clock instead of sleeping +- [Tracing capture audit](/quest/m1/test-flakes-2/trace-capture-audit.md) - no test misses tracing events because a parallel test silenced the call site ## Related diff --git a/quest/m1/test-flakes-2/mux-live-import-clock.md b/quest/m1/test-flakes-2/mux-live-import-clock.md new file mode 100644 index 00000000000..c468e4258ac --- /dev/null +++ b/quest/m1/test-flakes-2/mux-live-import-clock.md @@ -0,0 +1,28 @@ +# [S] moq-mux live import tests on the paused clock + +## Goal + +The moq-mux live import restart tests (`live_import_restarts_forward_after_idle` +for ts, fmp4, and flv) advance a paused clock across their idle gap and sleep +no real time. + +## Plan + +Facts (`origin/main`, 2026-10-01): the helpers (`live_import` in +`container/{ts,fmp4}/import_test.rs`, `import` in `flv/import_test.rs`) call +`std::thread::sleep(idle)` with a 300 ms idle. The gap is measured through +`clock::Anchor`, which reads `crate::Clock::now()`, and that is +`std::time::Instant::elapsed()`, so a paused tokio clock can't move it. + +Decided (2026-10-01): `crate::Clock`'s monotonic epoch becomes a +`web_async::time::Instant`. That is tokio's clock on native, so the tests start +paused and call `tokio::time::advance`, and it takes `std::time::Instant` off the +wasm path, as #4687 did for the TS SI debounce. The wall mapping stays +`SystemTime`. Rejected: injecting a `now` into the importers through +`translate_at`, which is plumbing only tests want. + +Update `test_util::late_clock` to match, and check other `crate::Clock` users +(including `Clock::at` callers) still build on wasm. + +Public API: check whether `Clock::at` is exported; if its `Instant` type +changes, report it. Wire: none. diff --git a/quest/m1/test-flakes-2/trace-capture-audit.md b/quest/m1/test-flakes-2/trace-capture-audit.md new file mode 100644 index 00000000000..b6be476abd5 --- /dev/null +++ b/quest/m1/test-flakes-2/trace-capture-audit.md @@ -0,0 +1,24 @@ +# [XS] Tracing captures survive the call-site cache + +## Goal + +No test in the workspace counts or asserts tracing events through a +thread-scoped subscriber (`with_default`, `set_default`) that a parallel +test can silence under a shared-process `cargo test`. + +## Plan + +#4690 found that tracing caches, per call site and for the whole process, +whether any subscriber wants its events. With only scoped subscribers, the +first thread to hit a call site decides for everyone, so a parallel test on a +thread with no subscriber turns the event off and the capture misses it. +Nextest runs each test in its own process and can't hit it, but `cargo test` +does. + +Grep the Rust crates for tests that capture tracing output with scoped +subscribers. Move each one to the helper #4690 added in +`rs/moq-net/src/model/test_tracing.rs` (one process-wide subscriber, events +routed to a per-thread counter), or a shared copy of it if other crates need +it. If none turn up, delete this quest. + +Public API: none. Wire: none. From ac25d88ab58eb1aba8e064630148bb2d313d33b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 17:30:38 -0700 Subject: [PATCH 11/28] quest: address review on live import clock and tracing capture audit Co-Authored-By: Claude Opus 5.5 --- .../m1/test-flakes-2/mux-live-import-clock.md | 24 ++++++++++++++----- quest/m1/test-flakes-2/trace-capture-audit.md | 11 ++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/quest/m1/test-flakes-2/mux-live-import-clock.md b/quest/m1/test-flakes-2/mux-live-import-clock.md index c468e4258ac..5d5375a721b 100644 --- a/quest/m1/test-flakes-2/mux-live-import-clock.md +++ b/quest/m1/test-flakes-2/mux-live-import-clock.md @@ -17,12 +17,24 @@ Facts (`origin/main`, 2026-10-01): the helpers (`live_import` in Decided (2026-10-01): `crate::Clock`'s monotonic epoch becomes a `web_async::time::Instant`. That is tokio's clock on native, so the tests start paused and call `tokio::time::advance`, and it takes `std::time::Instant` off the -wasm path, as #4687 did for the TS SI debounce. The wall mapping stays -`SystemTime`. Rejected: injecting a `now` into the importers through +wasm path. #4687 used the same `Instant` type, but only for the private SI +debounce after dropping its `crate::Clock`; `Clock` itself still reads +`std::time::Instant`. The wall mapping stays `SystemTime`. Rejected: injecting a `now` into the importers through `translate_at`, which is plumbing only tests want. -Update `test_util::late_clock` to match, and check other `crate::Clock` users -(including `Clock::at` callers) still build on wasm. +`std::time::Instant` is public at the boundary: `Clock::at(epoch, wall)` is +re-exported from `moq-mux`, and the json and binary timed writes +(`json.rs` `update`/`append`, and their `binary.rs` twins) take +`Timed<_, Instant>` through the crate-private `Clock::stamp`/`capture`. +`moq_net::Timed`'s docs say moq-mux uses `std::time::Instant`. Callers pass +`std::time::Instant` from `test_util::late_clock`, moq-cli publish, moq-hls +export, moq-audio and moq-video capture, and the catalog producer tests. -Public API: check whether `Clock::at` is exported; if its `Instant` type -changes, report it. Wire: none. +Open (maintainer, ask before starting): keep the public inputs on +`std::time::Instant` and convert at the boundary (native only; additive, stays +on `main`), or switch them to `web_async::time::Instant` as a typed break on +`dev` and update every caller and doc. Recommended: convert at the boundary, +so only the private epoch moves. + +Public API: none if converted at the boundary; otherwise `Clock::at` and the +timed writes change type. Wire: none. diff --git a/quest/m1/test-flakes-2/trace-capture-audit.md b/quest/m1/test-flakes-2/trace-capture-audit.md index b6be476abd5..7dca6d7ea82 100644 --- a/quest/m1/test-flakes-2/trace-capture-audit.md +++ b/quest/m1/test-flakes-2/trace-capture-audit.md @@ -15,10 +15,15 @@ thread with no subscriber turns the event off and the capture misses it. Nextest runs each test in its own process and can't hit it, but `cargo test` does. +Start after #4690 lands, with its open review finding on the subscriber +installation window resolved, so the race isn't copied into other crates. + Grep the Rust crates for tests that capture tracing output with scoped -subscribers. Move each one to the helper #4690 added in +subscribers. Move each one onto #4690's pattern in `rs/moq-net/src/model/test_tracing.rs` (one process-wide subscriber, events -routed to a per-thread counter), or a shared copy of it if other crates need -it. If none turn up, delete this quest. +routed to a per-thread capture). Its helper only counts WARN events by +message, so a capture that asserts something else needs the helper +generalized, or a shared copy if other crates need it. If none turn up, delete +this quest. Public API: none. Wire: none. From 7a934933f201df8f8a4ae7371c903c1a4943898c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 17:45:05 -0700 Subject: [PATCH 12/28] moq-net: rebuild callsite interest at each WARN capture Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/test_tracing.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rs/moq-net/src/model/test_tracing.rs b/rs/moq-net/src/model/test_tracing.rs index 37fcdaec490..c1d315f1032 100644 --- a/rs/moq-net/src/model/test_tracing.rs +++ b/rs/moq-net/src/model/test_tracing.rs @@ -27,6 +27,10 @@ thread_local! { pub(crate) fn count_drop_warnings(expected_message: &str, f: impl FnOnce()) -> usize { static INSTALL: Once = Once::new(); INSTALL.call_once(|| tracing::subscriber::set_global_default(Warns).expect("no other global subscriber")); + // A callsite first reached on another thread while the install above was still publishing + // sees no subscriber and is cached as disabled. Every capture starts after the install has + // completed, so rebuilding here re-enables it against the now-global subscriber. + tracing::callsite::rebuild_interest_cache(); let capture = Capture { expected: expected_message.to_owned(), From 3873e5ddeb6cac2a4e9480de81d32cb384371a87 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 20:06:57 -0700 Subject: [PATCH 13/28] quest: settle the moq-mux Clock boundary Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/mux-live-import-clock.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/quest/m1/test-flakes-2/mux-live-import-clock.md b/quest/m1/test-flakes-2/mux-live-import-clock.md index 5d5375a721b..fffbc557b8f 100644 --- a/quest/m1/test-flakes-2/mux-live-import-clock.md +++ b/quest/m1/test-flakes-2/mux-live-import-clock.md @@ -30,11 +30,9 @@ re-exported from `moq-mux`, and the json and binary timed writes `std::time::Instant` from `test_util::late_clock`, moq-cli publish, moq-hls export, moq-audio and moq-video capture, and the catalog producer tests. -Open (maintainer, ask before starting): keep the public inputs on -`std::time::Instant` and convert at the boundary (native only; additive, stays -on `main`), or switch them to `web_async::time::Instant` as a typed break on -`dev` and update every caller and doc. Recommended: convert at the boundary, -so only the private epoch moves. - -Public API: none if converted at the boundary; otherwise `Clock::at` and the -timed writes change type. Wire: none. +Decided (maintainer, 2026-10-01): convert at the boundary. Public inputs stay +`std::time::Instant` and convert to the private epoch internally (native only), +so this is additive and stays on `main`. Rejected: switching the public types to +`web_async::time::Instant` as a break on `dev`. + +Public API: none. Wire: none. From a2091755e6ca06b134108917911d59b5a0eee196 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:28:41 -0700 Subject: [PATCH 14/28] chore: claim tracing capture audit Co-Authored-By: GPT-6 From 5007de9655e41c22f32d27c03a8071929fe1bb69 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:30:07 -0700 Subject: [PATCH 15/28] chore: complete tracing capture audit Co-Authored-By: GPT-6 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/trace-capture-audit.md | 29 ------------------- 2 files changed, 30 deletions(-) delete mode 100644 quest/m1/test-flakes-2/trace-capture-audit.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 2d3287bbfb4..8580f2b58e8 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -36,7 +36,6 @@ Public API: none. Wire: none. - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time - [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial - [moq-mux live import clock](/quest/m1/test-flakes-2/mux-live-import-clock.md) - the live import restart tests advance a paused clock instead of sleeping -- [Tracing capture audit](/quest/m1/test-flakes-2/trace-capture-audit.md) - no test misses tracing events because a parallel test silenced the call site ## Related diff --git a/quest/m1/test-flakes-2/trace-capture-audit.md b/quest/m1/test-flakes-2/trace-capture-audit.md deleted file mode 100644 index 7dca6d7ea82..00000000000 --- a/quest/m1/test-flakes-2/trace-capture-audit.md +++ /dev/null @@ -1,29 +0,0 @@ -# [XS] Tracing captures survive the call-site cache - -## Goal - -No test in the workspace counts or asserts tracing events through a -thread-scoped subscriber (`with_default`, `set_default`) that a parallel -test can silence under a shared-process `cargo test`. - -## Plan - -#4690 found that tracing caches, per call site and for the whole process, -whether any subscriber wants its events. With only scoped subscribers, the -first thread to hit a call site decides for everyone, so a parallel test on a -thread with no subscriber turns the event off and the capture misses it. -Nextest runs each test in its own process and can't hit it, but `cargo test` -does. - -Start after #4690 lands, with its open review finding on the subscriber -installation window resolved, so the race isn't copied into other crates. - -Grep the Rust crates for tests that capture tracing output with scoped -subscribers. Move each one onto #4690's pattern in -`rs/moq-net/src/model/test_tracing.rs` (one process-wide subscriber, events -routed to a per-thread capture). Its helper only counts WARN events by -message, so a capture that asserts something else needs the helper -generalized, or a shared copy if other crates need it. If none turn up, delete -this quest. - -Public API: none. Wire: none. From 126c6a8eb8b94feabf8741578e9106be41999f42 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:32:21 -0700 Subject: [PATCH 16/28] chore: claim media late-join quest From c8787015f1e0c66d2f0d731d33a357a747088e3a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:35:10 -0700 Subject: [PATCH 17/28] quest: claim publish audio clock test Co-Authored-By: GPT-6 From 71d3f4e5c75dc7904d9d5a4fa87c565c29a99f20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:42:11 -0700 Subject: [PATCH 18/28] test(publish): mock the audio delay clock Co-Authored-By: GPT-6 --- js/publish/src/audio/encoder.test.ts | 37 ++++++++++++------- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/publish-audio-clock.md | 11 ------ 3 files changed, 23 insertions(+), 26 deletions(-) delete mode 100644 quest/m1/test-flakes-2/publish-audio-clock.md diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index bf4a47b9681..1abb22aa670 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -305,24 +305,33 @@ test("a push completing several frames keeps the encoder running", async () => { // it, which the catalog advertises as `delay`. test("a rendition trailing the broadcast's earliest advertises delay", async () => { using _webcodecs = installFakeWebCodecs(); - const baseline = new Baseline(); - using env = await setup(baseline); - const { encoder, feed } = env; + const clock = spyOn(performance, "now").mockReturnValue(200); - expect(encoder.out.catalog.peek()?.delay).toBeUndefined(); + try { + const baseline = new Baseline(); + using env = await setup(baseline); + const { encoder, feed } = env; - // A sibling that flushes each frame the instant it is captured. - baseline.observe(0, performance.now() * 1000); + expect(encoder.out.catalog.peek()?.delay).toBeUndefined(); - // Captured 100ms ago, so this rendition flushes at least that late. - const start = performance.now() * 1000 - 100_000; - for (let index = 0; index < 4; index++) { - await feed.push({ timestamp: Time.Micro(start + index * 20_000), channels: [new Float32Array(960)] }); - } - await feed.drain(); + // A sibling that flushes each frame the instant it is captured. + baseline.observe(0, performance.now() * 1000); + + // Captured 100ms ago, with a clock origin that keeps timestamps nonnegative. + const start = performance.now() * 1000 - 100_000; + for (let index = 0; index < 4; index++) { + await feed.push({ timestamp: Time.Micro(start + index * 20_000), channels: [new Float32Array(960)] }); + } + await feed.drain(); - expect(env.written.length).toBe(2); - expect(encoder.out.catalog.peek()?.delay).toBeGreaterThanOrEqual(100); + expect(env.written).toEqual([ + [100_000, 1], + [120_000, 1], + ]); + expect(encoder.out.catalog.peek()).toMatchObject({ delay: 100 }); + } finally { + clock.mockRestore(); + } }); // Regression: codec settings that can't resolve left the encoder unsettled, so `` never diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 2d3287bbfb4..67f3ff8b710 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -33,7 +33,6 @@ Public API: none. Wire: none. - [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed -- [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time - [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial - [moq-mux live import clock](/quest/m1/test-flakes-2/mux-live-import-clock.md) - the live import restart tests advance a paused clock instead of sleeping - [Tracing capture audit](/quest/m1/test-flakes-2/trace-capture-audit.md) - no test misses tracing events because a parallel test silenced the call site diff --git a/quest/m1/test-flakes-2/publish-audio-clock.md b/quest/m1/test-flakes-2/publish-audio-clock.md deleted file mode 100644 index ccc4becaadf..00000000000 --- a/quest/m1/test-flakes-2/publish-audio-clock.md +++ /dev/null @@ -1,11 +0,0 @@ -# [XS] js/publish audio delay test on mock time - -## Goal - -The js/publish audio encoder test "a rendition trailing the broadcast's -earliest advertises delay" passes when its file runs alone. - -## Plan - -It reads the real clock, so timestamps go negative early in the process and -it fails when run alone (#4414). Run it on mock time. From 70321a0d143bae2d64f4c83fbffc4d9125e0bab3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:46:35 -0700 Subject: [PATCH 19/28] test(media): measure late join against the published GOP Co-Authored-By: GPT-6 --- test/interop/README.md | 5 ++- test/interop/clients/js/harness.ts | 13 ++++--- test/interop/clients/js/media.ts | 20 +++++------ test/interop/clients/js/src/contract.test.ts | 37 +++++++++++++++++++- test/interop/clients/js/src/contract.ts | 33 +++++++++++++++++ test/interop/clients/js/src/fixture.ts | 18 +++++++--- test/interop/clients/js/src/setup.ts | 4 +++ test/justfile | 2 +- 8 files changed, 108 insertions(+), 24 deletions(-) diff --git a/test/interop/README.md b/test/interop/README.md index 8d862ba42e5..35f8a1efa9d 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -126,7 +126,10 @@ Each run covers, against a real local relay: The fake device is not physical hardware, and the headless permission decision is not a person clicking a browser prompt. - **pause and resume**, **unsubscribe and rejoin**, **detach and reattach**, - **publisher stop and same-path republish**, and **late join**. + **publisher stop and same-path republish**, and **late join**. The late join + must present the newest published GOP or a newer one. Its lower bound is the + encoded keyframe timestamp sampled before the existing viewer closes, since + painting the canvas does not mean capture and encoding have finished. - **resources return to baseline** - the page wraps `WebTransport`, `WebSocket`, `AudioContext`, and `Worker` to count live instances, so a detach that leaks a session is visible rather than merely invisible. diff --git a/test/interop/clients/js/harness.ts b/test/interop/clients/js/harness.ts index bd59af1602b..9b790205377 100644 --- a/test/interop/clients/js/harness.ts +++ b/test/interop/clients/js/harness.ts @@ -256,15 +256,18 @@ export async function readFixtureState(page: Page): Promise { } /** Invoke one of the page's {@link InteropControl} commands. */ -export async function command(page: Page, name: keyof InteropControl): Promise { - await page.evaluate( +export async function command( + page: Page, + name: K, +): Promise>> { + return (await page.evaluate( ([key, fn]) => { - const control = (window as unknown as Record void> | undefined>)[key]; + const control = (window as unknown as Record unknown> | undefined>)[key]; if (!control?.[fn]) throw new Error(`the page exposes no ${fn} command`); - control[fn](); + return control[fn](); }, [CONTROL, name] as const, - ); + )) as Awaited>; } /** Read the page's live resource counts, which outlive the player element. */ diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index f4e9577d3a0..d73259a50ef 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -47,7 +47,7 @@ import { import { type CaptureState, FAULTS, - KEYFRAME_INTERVAL_MS, + lateJoinStartsLive, leakedPlayerStarted, SAMPLE_MS, SAMPLE_RATE, @@ -115,9 +115,6 @@ const MIN_RATE = 0.5; */ const MAX_SKEW_STEPS = 1; -/** The first decodable frame may start at the current GOP's keyframe, but never in older history. */ -const MAX_LATE_JOIN_LAG_FRAMES = Math.ceil((Pattern.FPS * KEYFRAME_INTERVAL_MS) / 1000); - const percentile = (values: number[], p: number) => { if (values.length === 0) return Number.NaN; const sorted = [...values].sort((a, b) => a - b); @@ -660,8 +657,8 @@ try { // at the live edge rather than replay what it missed. if (wants("late-join")) { console.error("=== late join ==="); + const live = await command(publisher, "liveGop"); await player.close(); - const live = await readFixtureState(publisher); [player, playerErrors] = await subscriber(broadcast, "latecomer"); await gesture(player); const joined = await waitForState(player, playerErrors, { @@ -670,17 +667,16 @@ try { description: "the latecomer to present the fixture", predicate: (state) => state.frameId !== undefined && state.audioContext === "running", }); - // The fixture sample names the frame painted immediately before the page opens. The first - // decodable frame can be the keyframe at the start of the current GOP, so require it to be - // within that GOP rather than requiring an impossible zero-frame capture/encode delay. - const lag = live.frameId - (joined.frameId ?? 0); + // requestFrame() is asynchronous, and capture timestamps decide keyframes. The painted + // counter can therefore be over 15 ticks ahead of a still-current encoded GOP under load. + // Compare against its actual published keyframe, sampled while the old viewer held demand. check( - lag <= MAX_LATE_JOIN_LAG_FRAMES, + lateJoinStartsLive(live, joined.videoTimestamp), "late join starts live", () => - `joined at frame ${joined.frameId}, ${lag} frames behind the ${live.frameId} already published when it opened (one GOP is ${MAX_LATE_JOIN_LAG_FRAMES})`, + `joined at frame ${joined.frameId}, timestamp ${joined.videoTimestamp}ms before the current GOP's ${live.timestamp}ms keyframe`, ); - console.error(` joined at frame ${joined.frameId}, live edge was ${live.frameId}`); + console.error(` joined at frame ${joined.frameId}, timestamp ${joined.videoTimestamp}ms, current GOP began at ${live.timestamp}ms`); assertMedia(await collect(player, playerErrors, WINDOW_MS), "late join"); } diff --git a/test/interop/clients/js/src/contract.test.ts b/test/interop/clients/js/src/contract.test.ts index b36189ed585..755ffa9ce37 100644 --- a/test/interop/clients/js/src/contract.test.ts +++ b/test/interop/clients/js/src/contract.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; -import { leakedPlayerStarted, type Resources } from "./contract"; +import * as Moq from "@moq/net"; +import { lateJoinStartsLive, leakedPlayerStarted, readLiveGop, type Resources } from "./contract"; const playing: Resources = { transports: 1, sockets: 0, audioContexts: 1, workers: 0 }; @@ -17,3 +18,37 @@ test("a leaked player is visible when it reuses a pooled websocket fallback", () test("unchanged counts are not a leak start", () => { expect(leakedPlayerStarted(playing, playing)).toBe(false); }); + + +test("late join uses the published GOP when painting is ahead of capture", async () => { + const broadcast = new Moq.Broadcast.Producer(); + const track = broadcast.createTrack("video"); + try { + const old = track.appendGroup(); + old.writeFrame({ payload: new Uint8Array([1]), timestamp: Moq.Time.Timestamp.fromMillis(3000) }); + old.close(); + const current = track.appendGroup(); + current.writeFrame({ payload: new Uint8Array([2]), timestamp: Moq.Time.Timestamp.fromMillis(3700) }); + current.writeFrame({ payload: new Uint8Array([3]), timestamp: Moq.Time.Timestamp.fromMillis(4199) }); + + // Painted counter 127 can precede capture while GOP 111 remains current. The old + // 127 - 111 <= 15 check rejects a valid join; an earlier published GOP must fail. + const viewer = track.subscribe(); + const gop = await readLiveGop(broadcast.track("video")); + expect(gop.timestamp).toBe(3700); + expect(lateJoinStartsLive(gop, 3700)).toBe(true); + expect(lateJoinStartsLive(gop, 4199)).toBe(true); + expect(lateJoinStartsLive(gop, 3000)).toBe(false); + expect(lateJoinStartsLive(gop, undefined)).toBe(false); + expect(track.used.peek()).toBe(true); + const viewed = await viewer.recvGroup(); + expect((await viewed?.readFrame())?.payload).toEqual(new Uint8Array([2])); + expect((await viewed?.readFrame())?.payload).toEqual(new Uint8Array([3])); + viewed?.close(); + viewer.close(); + await track.unused(); + expect(track.used.peek()).toBe(false); + } finally { + broadcast.close(); + } +}); diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index 00c8803e94f..431c8e44e78 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -8,6 +8,8 @@ * @module */ +import type * as Moq from "@moq/net"; + // ── the deterministic publisher ───────────────────────────────────────────── /** A deliberate defect, used to prove an assertion can fail. */ @@ -43,6 +45,35 @@ export type FixtureState = { encodedFrames: number; }; +/** The newest published video's GOP, sampled independently of canvas capture and encoding. */ +export type LiveGop = { + /** The timestamp of its keyframe, in milliseconds on the publisher's media clock. */ + timestamp: number; +}; + +/** Read the newest published GOP without retaining a subscription or changing its frames. */ +export async function readLiveGop(track: Moq.Track.Consumer): Promise { + const subscriber = track.subscribe(); + try { + const group = await subscriber.recvGroup(); + if (!group) throw new Error("the fixture video has no published GOP"); + try { + const frame = await group.readFrame(); + if (!frame) throw new Error("the fixture GOP has no keyframe"); + return { timestamp: frame.timestamp.asMillis() }; + } finally { + group.close(); + } + } finally { + subscriber.close(); + } +} + +/** True when the presented frame belongs to the sampled GOP or a newer one. */ +export function lateJoinStartsLive(gop: LiveGop, timestamp: number | undefined): boolean { + return timestamp !== undefined && timestamp >= gop.timestamp; +} + /** The camera publisher state mirrored onto its element for Playwright. */ export type CaptureState = { videoError?: string; @@ -161,6 +192,8 @@ export type Sample = { * owns them rather than the driver reaching in. */ export type InteropControl = { + /** Read the current video GOP while an existing viewer is still pulling media. */ + liveGop(): Promise; /** Stop the fixture publisher, releasing its session. */ stop(): void; /** Start (or restart) the fixture publisher on the same broadcast path. */ diff --git a/test/interop/clients/js/src/fixture.ts b/test/interop/clients/js/src/fixture.ts index 17f6faef559..a3d3ff1a789 100644 --- a/test/interop/clients/js/src/fixture.ts +++ b/test/interop/clients/js/src/fixture.ts @@ -17,7 +17,7 @@ import { Time } from "@moq/net"; import * as Publish from "@moq/publish"; import { Effect, Signal } from "@moq/signals"; import type { Fault, FixtureState } from "./contract"; -import { KEYFRAME_INTERVAL_MS, OFFSET_STEPS, SAMPLE_RATE } from "./contract"; +import { KEYFRAME_INTERVAL_MS, OFFSET_STEPS, readLiveGop, SAMPLE_RATE } from "./contract"; import * as Pattern from "./pattern"; /** Cap the encoder rather than letting it track a bandwidth estimate, so runs are comparable. */ @@ -47,6 +47,7 @@ export class Fixture { readonly host: HTMLElement; readonly #signals = new Effect(); + readonly #broadcast: Publish.Broadcast; readonly #audio: AudioContext; readonly #frameId = new Signal(-1); readonly #audioState = new Signal("suspended"); @@ -95,6 +96,8 @@ export class Fixture { }); this.#signals.cleanup(() => broadcast.close()); + this.#broadcast = broadcast; + const video = new Publish.Video.Encoder("video", { broadcast, capture, @@ -183,8 +186,8 @@ export class Fixture { this.#signals.interval(schedule, (SCHEDULE_AHEAD * 1000) / 4); } - // Paint and emit one frame per pattern tick. The counter comes from the audio clock rather than - // the timer, so a late or coalesced tick skips a frame instead of drifting the two apart. + // Paint and request capture per pattern tick. Capture and encoding can finish later. The counter + // comes from the audio clock, so a coalesced tick skips a frame instead of drifting the two apart. #runPicture(ctx: CanvasRenderingContext2D, track: CanvasSource, start: number, fault: Fault): void { let painted = -1; @@ -202,7 +205,7 @@ export class Fixture { }; // Twice the frame rate: a tick that lands between frames is a no-op, and one that lands late - // still emits the frame the clock is on. + // still requests capture of the frame the clock is on. this.#timer = self.setInterval(tick, 1000 / Pattern.FPS / 2); this.#signals.cleanup(() => { if (this.#timer !== undefined) self.clearInterval(this.#timer); @@ -210,6 +213,13 @@ export class Fixture { }); } + /** Sample the current published keyframe before the viewer closes its subscription. */ + async liveGop() { + const broadcast = this.#broadcast.net.peek(); + if (!broadcast) throw new Error("the fixture is not publishing"); + return readLiveGop(broadcast.track("video")); + } + /** Stop publishing and release the capture, audio graph, and session. */ close(): void { this.#signals.close(); diff --git a/test/interop/clients/js/src/setup.ts b/test/interop/clients/js/src/setup.ts index aeb4177abb9..1cd5275a103 100644 --- a/test/interop/clients/js/src/setup.ts +++ b/test/interop/clients/js/src/setup.ts @@ -55,6 +55,10 @@ if (role === "publish") { // The driver stops and restarts the publisher in place to exercise a same-path republish, which // has to reuse this page so the audio context keeps its user activation. publish({ + liveGop: () => { + if (!fixture) throw new Error("the fixture is stopped"); + return fixture.liveGop(); + }, stop: () => { fixture?.close(); fixture = undefined; diff --git a/test/justfile b/test/justfile index 1be6619e2cc..fecc03f3302 100644 --- a/test/justfile +++ b/test/justfile @@ -24,7 +24,7 @@ harness: ./lib/harness.test.sh cd .. && bun install --frozen-lockfile cd interop/clients/js && bunx playwright install chromium - cd interop/clients/js && bun test ./harness.browser.ts + cd interop/clients/js && bun test ./harness.browser.ts ./src/contract.test.ts # Cross-language media interop test, built from this checkout. Stands up a # relay and runs the publisher x subscriber matrix. Default: rust only (fast From ddc410dd3c33fd98395007942195cce2a52410fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:57:01 -0700 Subject: [PATCH 20/28] chore: claim WebSocket paused TLS quest Co-Authored-By: GPT-6 From 6cd36a7e497bbb3a7a146d15337095c0b86e3ff8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 07:57:56 -0700 Subject: [PATCH 21/28] chore(mux): claim live import clock quest From e4d26c65cc859bda3449f743e598c1273039e3d4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 08:17:32 -0700 Subject: [PATCH 22/28] fix(mux): measure live import idle gaps on the async clock Co-Authored-By: GPT-6 --- doc/lib/rs/moq-mux.md | 9 ++- quest/m1/test-flakes-2/README.md | 1 - .../m1/test-flakes-2/mux-live-import-clock.md | 38 ----------- rs/moq-mux/src/clock.rs | 65 +++++++++++++++++-- rs/moq-mux/src/container/flv/import_test.rs | 4 +- rs/moq-mux/src/container/fmp4/import_test.rs | 4 +- rs/moq-mux/src/container/ts/import_test.rs | 4 +- 7 files changed, 70 insertions(+), 55 deletions(-) delete mode 100644 quest/m1/test-flakes-2/mux-live-import-clock.md diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 6ac125f6bf1..ede10a5bbc8 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -89,8 +89,9 @@ A payload that knows when it was captured (a datagram's arrival, a sensor read) carries that `Instant`. The producer maps it onto the broadcast clock and writes it as the frame timestamp, and the entry advertises `jitter` and `delay` the way a media rendition does, so telemetry lagging its video shows up as `delay`. An -instant ahead of now is refused. A device's own clock is an unrelated epoch; -keep it in the payload. +instant ahead of now is refused. Capture inputs stay `std::time::Instant` on +native; browser targets refuse these native instants. A device's own clock is an +unrelated epoch; keep it in the payload. ```rust telemetry.append(moq_net::Timed::from(packet).at(received_at))?; @@ -102,7 +103,9 @@ the first frame is live on arrival, every track of the input shares that one mapping, and a source that restarts its timestamps continues forward after the real idle gap. fMP4 passthrough rewrites each fragment's `tfdt` to match. Use it for a live feed with its own zero; publish verbatim only when the catalog's -clock (`Config::with_clock`) already names the source's zero. +clock (`Config::with_clock`) already names the source's zero. The broadcast clock +uses the async runtime's monotonic time, so native tests can pause and advance it +with Tokio; its wall mapping stays fixed. An application running its own demuxer gets the same mapping from `clock::Anchor`: one per source, plus one `clock::Lane` per track. A single diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 2d3287bbfb4..ac24bc81af2 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -35,7 +35,6 @@ Public API: none. Wire: none. - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time - [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial -- [moq-mux live import clock](/quest/m1/test-flakes-2/mux-live-import-clock.md) - the live import restart tests advance a paused clock instead of sleeping - [Tracing capture audit](/quest/m1/test-flakes-2/trace-capture-audit.md) - no test misses tracing events because a parallel test silenced the call site ## Related diff --git a/quest/m1/test-flakes-2/mux-live-import-clock.md b/quest/m1/test-flakes-2/mux-live-import-clock.md deleted file mode 100644 index fffbc557b8f..00000000000 --- a/quest/m1/test-flakes-2/mux-live-import-clock.md +++ /dev/null @@ -1,38 +0,0 @@ -# [S] moq-mux live import tests on the paused clock - -## Goal - -The moq-mux live import restart tests (`live_import_restarts_forward_after_idle` -for ts, fmp4, and flv) advance a paused clock across their idle gap and sleep -no real time. - -## Plan - -Facts (`origin/main`, 2026-10-01): the helpers (`live_import` in -`container/{ts,fmp4}/import_test.rs`, `import` in `flv/import_test.rs`) call -`std::thread::sleep(idle)` with a 300 ms idle. The gap is measured through -`clock::Anchor`, which reads `crate::Clock::now()`, and that is -`std::time::Instant::elapsed()`, so a paused tokio clock can't move it. - -Decided (2026-10-01): `crate::Clock`'s monotonic epoch becomes a -`web_async::time::Instant`. That is tokio's clock on native, so the tests start -paused and call `tokio::time::advance`, and it takes `std::time::Instant` off the -wasm path. #4687 used the same `Instant` type, but only for the private SI -debounce after dropping its `crate::Clock`; `Clock` itself still reads -`std::time::Instant`. The wall mapping stays `SystemTime`. Rejected: injecting a `now` into the importers through -`translate_at`, which is plumbing only tests want. - -`std::time::Instant` is public at the boundary: `Clock::at(epoch, wall)` is -re-exported from `moq-mux`, and the json and binary timed writes -(`json.rs` `update`/`append`, and their `binary.rs` twins) take -`Timed<_, Instant>` through the crate-private `Clock::stamp`/`capture`. -`moq_net::Timed`'s docs say moq-mux uses `std::time::Instant`. Callers pass -`std::time::Instant` from `test_util::late_clock`, moq-cli publish, moq-hls -export, moq-audio and moq-video capture, and the catalog producer tests. - -Decided (maintainer, 2026-10-01): convert at the boundary. Public inputs stay -`std::time::Instant` and convert to the private epoch internally (native only), -so this is additive and stays on `main`. Rejected: switching the public types to -`web_async::time::Instant` as a break on `dev`. - -Public API: none. Wire: none. diff --git a/rs/moq-mux/src/clock.rs b/rs/moq-mux/src/clock.rs index ac813e0edfd..367039d13ea 100644 --- a/rs/moq-mux/src/clock.rs +++ b/rs/moq-mux/src/clock.rs @@ -23,6 +23,18 @@ use std::time::{Duration, Instant, SystemTime}; use hang::catalog::{MAX_SAFE_INTEGER, MOQ_EPOCH_UNIX_MILLIS}; +/// Native capture inputs use the same epoch as the async clock. +fn monotonic(at: Instant) -> crate::Result { + #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] + return Ok(web_async::time::Instant::from_std(at)); + + #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + { + let _ = at; + Err(anyhow::anyhow!("std::time::Instant inputs are unsupported in the browser").into()) + } +} + /// The catalog clock for PTS zero at `wall`. fn wall_clock(wall: SystemTime) -> crate::Result { let unix_micros = wall @@ -51,7 +63,7 @@ fn wall_clock(wall: SystemTime) -> crate::Result { /// pinned up front and that frame's PTS converts into the mapping instead. #[derive(Clone, Copy, Debug)] pub struct Clock { - epoch: Instant, + epoch: web_async::time::Instant, wall: hang::catalog::Clock, } @@ -73,22 +85,25 @@ impl Clock { /// Start a clock at the current instant, with PTS zero ten seconds earlier on both the /// monotonic and the wall clock, so earlier-stamped frames of a source anchored now still map. pub fn new() -> Self { - let (now, wall) = (Instant::now(), SystemTime::now()); + let (now, wall) = (web_async::time::Instant::now(), SystemTime::now()); // Shortly after boot the monotonic clock may not reach back that far; start at now then. let (epoch, wall) = match (now.checked_sub(Self::LEAD), wall.checked_sub(Self::LEAD)) { (Some(epoch), Some(wall)) => (epoch, wall), _ => (now, wall), }; - Self::at(epoch, wall).expect("the current wall time is representable as a broadcast clock") + Self { + epoch, + wall: wall_clock(wall).expect("the current wall time is representable as a broadcast clock"), + } } /// Start a clock at an explicit monotonic epoch and wall time. /// /// The deterministic constructor: synthetic sources and fixtures pin both ends instead of - /// sampling. Refuses an unrepresentable wall. + /// sampling. Refuses an unrepresentable wall or a browser target without native instants. pub fn at(epoch: Instant, wall: SystemTime) -> crate::Result { Ok(Self { - epoch, + epoch: monotonic(epoch)?, wall: wall_clock(wall)?, }) } @@ -103,9 +118,11 @@ impl Clock { /// Map the instant a payload was captured (a datagram's arrival, a sensor read) onto this clock. /// /// Refuses an instant ahead of now, which would claim the payload reached the transport before - /// it existed, and one before the clock's epoch, which no timestamp can name. + /// it existed, and one before the clock's epoch, which no timestamp can name. Native capture + /// instants are unsupported in the browser. pub(crate) fn capture(&self, at: Instant) -> crate::Result { - if at > Instant::now() { + let at = monotonic(at)?; + if at > web_async::time::Instant::now() { return Err(crate::Error::InvalidCapture); } let elapsed = at @@ -456,6 +473,40 @@ mod tests { )); } + #[tokio::test(start_paused = true)] + async fn paused_clock_preserves_native_boundaries() { + let epoch = tokio::time::Instant::now(); + let clock = Clock::at(epoch.into_std(), moq_epoch()).unwrap(); + let fresh = Clock::new(); + let before = fresh.now(); + let wall = clock.wall(); + + tokio::time::advance(Duration::from_secs(3)).await; + assert_eq!(clock.now(), us(3_000_000)); + assert_eq!(fresh.now().as_micros() - before.as_micros(), 3_000_000); + assert_eq!(clock.wall(), wall); + assert_eq!( + clock.wall_clock(clock.now()).unwrap(), + moq_epoch() + Duration::from_secs(3) + ); + + let captured = (epoch + Duration::from_secs(2)).into_std(); + let (timed, at) = clock.stamp(moq_net::Timed::from("payload").at(captured)).unwrap(); + assert_eq!(timed.at, Some(us(2_000_000))); + assert_eq!(at, Some(us(2_000_000))); + let (untimed, at) = clock.stamp(moq_net::Timed::from("payload")).unwrap(); + assert_eq!(untimed.at, Some(us(3_000_000))); + assert_eq!(at, None); + assert!(matches!( + clock.capture((epoch + Duration::from_secs(4)).into_std()), + Err(crate::Error::InvalidCapture) + )); + assert!(matches!( + clock.capture((epoch - Duration::from_secs(1)).into_std()), + Err(crate::Error::InvalidCapture) + )); + } + #[test] fn section_advertises_wall_in_clock_timescale() { // PTS zero at exactly the moq epoch advertises 0. diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 2073a6886ce..c4dca482b14 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -734,7 +734,7 @@ async fn import(chunks: &[Vec], live: bool, ago: Duration, idle: Duration) - let mut after = before; for (i, chunk) in chunks.iter().enumerate() { if i > 0 { - std::thread::sleep(idle); + tokio::time::advance(idle).await; } importer.decode(chunk).unwrap(); if i == 0 { @@ -774,7 +774,7 @@ async fn live_import_anchors_a_late_first_frame() { /// An encoder restarting its timestamps at zero continues the broadcast forward: after the real /// idle gap, and with every track moving onto the one new mapping. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn live_import_restarts_forward_after_idle() { for idle in [Duration::ZERO, Duration::from_millis(300)] { let input = [session(true, 5_000, 50), session(false, 0, 50)]; diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 40c10a626d6..b73b5783b83 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1331,7 +1331,7 @@ async fn live_import( let mut after = before; for (i, chunk) in chunks.iter().enumerate() { if i > 0 { - std::thread::sleep(idle); + tokio::time::advance(idle).await; } fmp4.decode(chunk).unwrap(); if i == 0 { @@ -1393,7 +1393,7 @@ async fn live_import_rewrites_tfdt_onto_the_broadcast_clock() { /// A source whose decode times restart at zero continues forward after the real idle gap rather /// than being refused as non-monotonic, with every track moving onto one new mapping. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn live_import_restarts_forward_after_idle() { for idle in [std::time::Duration::ZERO, std::time::Duration::from_millis(300)] { let input = [live_session(5_000_000, 20), live_session(0, 20)]; diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index f06ed12c92e..6d11c02c9c1 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -623,7 +623,7 @@ async fn live_import(chunks: &[&[u8]], live: bool, ago: std::time::Duration, idl let mut after = before; for (i, chunk) in chunks.iter().enumerate() { if i > 0 { - std::thread::sleep(idle); + tokio::time::advance(idle).await; } import.decode(chunk).unwrap(); if i == 0 { @@ -663,7 +663,7 @@ async fn live_import_anchors_a_late_first_frame() { /// The same feed played twice, as when an encoder restarts its PTS, continues forward after the /// real idle gap instead of rewinding. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn live_import_restarts_forward_after_idle() { let data: &[u8] = include_bytes!("test_data/bbb_cbr.ts"); let once = live_import(&[data], false, std::time::Duration::ZERO, std::time::Duration::ZERO).await; From 9631e3247fd80740ceb20d27da012557bafa295b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 08:21:12 -0700 Subject: [PATCH 23/28] test(tokio): run fixed-address WebSocket TLS dials on real time Co-Authored-By: GPT-6 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/websocket-paused-tls.md | 14 -------------- rs/moq-tokio/src/websocket.rs | 2 -- 3 files changed, 17 deletions(-) delete mode 100644 quest/m1/test-flakes-2/websocket-paused-tls.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 2d3287bbfb4..20e696a710f 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -34,7 +34,6 @@ Public API: none. Wire: none. - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed - [js/publish audio clock](/quest/m1/test-flakes-2/publish-audio-clock.md) - the audio encoder delay test runs on mock time -- [WebSocket paused TLS dial](/quest/m1/test-flakes-2/websocket-paused-tls.md) - the fixed-address WebSocket tests stop pausing the clock over a real dial - [moq-mux live import clock](/quest/m1/test-flakes-2/mux-live-import-clock.md) - the live import restart tests advance a paused clock instead of sleeping - [Tracing capture audit](/quest/m1/test-flakes-2/trace-capture-audit.md) - no test misses tracing events because a parallel test silenced the call site diff --git a/quest/m1/test-flakes-2/websocket-paused-tls.md b/quest/m1/test-flakes-2/websocket-paused-tls.md deleted file mode 100644 index d92b724882a..00000000000 --- a/quest/m1/test-flakes-2/websocket-paused-tls.md +++ /dev/null @@ -1,14 +0,0 @@ -# [XS] WebSocket fixed-address tests off the paused clock - -## Goal - -moq-tokio websocket `fixed_addresses_keep_tls_name_and_request_host` and -`ipv6_literal_fixed_addresses` no longer pause the clock over a real TLS -dial. - -## Plan - -They pass today, but they pause the clock over a real TLS dial on loopback. -Once a timeout lands on that path, the paused clock can fire it before -loopback delivers, the race [#4527](https://github.com/moq-dev/moq/pull/4527) -removed from the auth outage tests. Remove it the same way. diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index e48d13c9619..6e2f5a5cf3a 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -719,7 +719,6 @@ mod tests { #[tokio::test] async fn fixed_addresses_keep_tls_name_and_request_host() { - tokio::time::pause(); check_tls_authority("relay.example", true, None).await; } @@ -730,7 +729,6 @@ mod tests { #[tokio::test] async fn ipv6_literal_fixed_addresses() { - tokio::time::pause(); check_tls_authority("[::1]", true, None).await; } From 179baad294b843d9bf34e376cfbf069edb1a6289 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 22:06:54 -0700 Subject: [PATCH 24/28] test(mux): allow 90 kHz tick truncation in the TS restart shift The paused clock makes the idle gap exact, exposing the translator's truncation of each pass's offset to a 90 kHz tick, which failed about two of three runs by 1us. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import_test.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 6d11c02c9c1..34f495b17af 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -694,9 +694,12 @@ async fn live_import_restarts_forward_after_idle() { let second = offset(|t| *t.last().unwrap()); // The second pass continues after the first plus the real idle gap, not on top of it. + // Translation keeps the 90 kHz source timescale, so each pass's mapping truncates to + // a tick, and the shift between the two can fall short by under two ticks. let shift = second - first; + let ticks = 2 * 1_000_000u128.div_ceil(90_000) as i128; assert!( - shift >= span + idle.as_micros() as i128, + shift + ticks >= span + idle.as_micros() as i128, "the restart resumes after the first pass and the idle gap: {shift} < {span} + {idle:?}" ); assert!( From 2cb32d6f6b6ca5054714a4eddae8d7a4092076b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 3 Oct 2026 08:46:36 -0700 Subject: [PATCH 25/28] quest(test-flakes-2): export linger test times from stdin close Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 1 + quest/m1/test-flakes-2/export-linger.md | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 quest/m1/test-flakes-2/export-linger.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 9ca7faaf202..c22470034d2 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -32,6 +32,7 @@ Public API: none. Wire: none. - [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean +- [Export linger timer](/quest/m1/test-flakes-2/export-linger.md) - the export linger test times from stdin close, not publisher exit - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed ## Related diff --git a/quest/m1/test-flakes-2/export-linger.md b/quest/m1/test-flakes-2/export-linger.md new file mode 100644 index 00000000000..c8497eeb78d --- /dev/null +++ b/quest/m1/test-flakes-2/export-linger.md @@ -0,0 +1,15 @@ +# [XS] Export linger test times from stdin close + +## Goal + +`a_clean_finish_exits_zero_once_the_linger_expires` in +`rs/moq-cli/tests/export.rs` stops failing its `>= 900ms` lower bound under +load, while still proving the export waits out its linger. + +## Plan + +The test starts its timer after the publisher process exits, but the export's +linger starts when the broadcast end reaches it, which can come first. Start +the timer before `drop(stdin)` instead: the broadcast can only end after stdin +closes, so that instant is a correct lower bound. Subprocess test, so a paused +clock can't apply. From f296ecce03d11848e89911dc413d3f16d047998d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 3 Oct 2026 08:46:52 -0700 Subject: [PATCH 26/28] test(cli): time the export linger from stdin close The export's linger starts when the broadcast end reaches it, which can precede the publisher process exiting. Timing from that exit let load eat the 900ms lower bound. Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 1 - quest/m1/test-flakes-2/export-linger.md | 15 --------------- rs/moq-cli/tests/export.rs | 5 +++-- 3 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 quest/m1/test-flakes-2/export-linger.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index c22470034d2..9ca7faaf202 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -32,7 +32,6 @@ Public API: none. Wire: none. - [moq-cli tests on a paused clock](/quest/m1/test-flakes-2/cli-paused-clock.md) - the fetch timeout and completion tests stop racing wall-clock budgets - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean -- [Export linger timer](/quest/m1/test-flakes-2/export-linger.md) - the export linger test times from stdin close, not publisher exit - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed ## Related diff --git a/quest/m1/test-flakes-2/export-linger.md b/quest/m1/test-flakes-2/export-linger.md deleted file mode 100644 index c8497eeb78d..00000000000 --- a/quest/m1/test-flakes-2/export-linger.md +++ /dev/null @@ -1,15 +0,0 @@ -# [XS] Export linger test times from stdin close - -## Goal - -`a_clean_finish_exits_zero_once_the_linger_expires` in -`rs/moq-cli/tests/export.rs` stops failing its `>= 900ms` lower bound under -load, while still proving the export waits out its linger. - -## Plan - -The test starts its timer after the publisher process exits, but the export's -linger starts when the broadcast end reaches it, which can come first. Start -the timer before `drop(stdin)` instead: the broadcast can only end after stdin -closes, so that instant is a correct lower bound. Subprocess test, so a paused -clock can't apply. diff --git a/rs/moq-cli/tests/export.rs b/rs/moq-cli/tests/export.rs index 3815187993a..5363875b43a 100644 --- a/rs/moq-cli/tests/export.rs +++ b/rs/moq-cli/tests/export.rs @@ -151,14 +151,15 @@ async fn a_clean_finish_exits_zero_once_the_linger_expires() { let (mut publisher, feeding) = import(&relay); let stdin = feeding.await.unwrap(); output_past(&output, 0).await; + // The linger can't start before stdin closes, but it can before the publisher exits. + let closed = Instant::now(); drop(stdin); assert!(wait(&mut publisher).await.success()); - let finished = Instant::now(); let status = wait(&mut export).await; assert!(status.success(), "a clean finish exits 0, got {status}"); assert!( - finished.elapsed() >= Duration::from_millis(900), + closed.elapsed() >= Duration::from_millis(900), "the export waited out its linger" ); } From 767eca556850363a3145e46c42a1b8e796de8cc9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 16:23:10 -0700 Subject: [PATCH 27/28] test(mux): check the paused clock's capture now that main dropped Clock::stamp Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/clock.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/rs/moq-mux/src/clock.rs b/rs/moq-mux/src/clock.rs index 77d563c2ba1..fb0c1b63a54 100644 --- a/rs/moq-mux/src/clock.rs +++ b/rs/moq-mux/src/clock.rs @@ -228,12 +228,7 @@ mod tests { ); let captured = (epoch + Duration::from_secs(2)).into_std(); - let (timed, at) = clock.stamp(moq_net::Timed::from("payload").at(captured)).unwrap(); - assert_eq!(timed.at, Some(us(2_000_000))); - assert_eq!(at, Some(us(2_000_000))); - let (untimed, at) = clock.stamp(moq_net::Timed::from("payload")).unwrap(); - assert_eq!(untimed.at, Some(us(3_000_000))); - assert_eq!(at, None); + assert_eq!(clock.capture(captured).unwrap(), us(2_000_000)); assert!(matches!( clock.capture((epoch + Duration::from_secs(4)).into_std()), Err(crate::Error::InvalidCapture) From cf5712a5a72a2c67c04718b9c79ee7787bbb2d5a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:09:55 -0700 Subject: [PATCH 28/28] quest: record test-flakes-2 as landed in the flat-lines plan Co-Authored-By: Claude Opus 5.5 --- quest/m1/quest-flat-lines.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quest/m1/quest-flat-lines.md b/quest/m1/quest-flat-lines.md index 4cfbbd4e3bb..0bdce596103 100644 --- a/quest/m1/quest-flat-lines.md +++ b/quest/m1/quest-flat-lines.md @@ -59,8 +59,8 @@ Remaining: #4403 wildcard, #4034 archive, #4039 auth, #4079 cpp, #4519 ffi-shape, #4080 obs-moq-video, #4133 qos (held until [Lag across a splice](/quest/m1/qos/lag-splice.md) is fixed), #4437 rs2ts, - #4653 test-flakes-2, #4180 transport-upgrade, #4640 tstd. #4162 - (audio-jitter-target) landed, and #4438 folded into rs2ts. + #4180 transport-upgrade, #4640 tstd. #4162 + (audio-jitter-target) and #4653 (test-flakes-2) landed, and #4438 folded into rs2ts. - Fold #4255 (archive/track-timeline) into the archive branch before #4034 lands; as of the 2026-10-05 audit it has not happened and #4034 is a draft. - Before a line lands and its branch is deleted, merge or retarget every child