diff --git a/.config/hakari.toml b/.config/hakari.toml index 14f875f62ef..5e07beeed9b 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -37,4 +37,4 @@ workspace-members = [ workspace-members = ["kio-macros"] # Keep native media dependencies out of unrelated test builds. Final excludes # preserve feature unification for their other dependencies. -third-party = [{ name = "pipewire" }, { name = "libspa" }, { name = "clang-sys" }] +third-party = [{ name = "pipewire" }, { name = "libspa" }, { name = "clang-sys" }, { name = "gstreamer" }, { name = "gstreamer-sys" }] diff --git a/Cargo.lock b/Cargo.lock index d7390ea87be..c828f88eea8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2784,6 +2784,30 @@ dependencies = [ "system-deps 9.0.0", ] +[[package]] +name = "gstreamer-check" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca00e318412c20405c6e0817cb330b706d128c779c9ec613fa2b53e4daf55c7" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-check-sys", +] + +[[package]] +name = "gstreamer-check-sys" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7beea104cb86688e15047b9712419e7d9194c0ca8ce47848fcc8482c0207be9c" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps 9.0.0", +] + [[package]] name = "gstreamer-sys" version = "0.25.4" @@ -4488,6 +4512,7 @@ dependencies = [ "gst-plugin-version-helper", "gstreamer", "gstreamer-audio", + "gstreamer-check", "hang", "kio", "moq-audio", @@ -10630,6 +10655,7 @@ dependencies = [ "num-bigint", "num-complex", "num-integer", + "num-rational", "num-traits", "objc2 0.6.5", "objc2-core-audio", diff --git a/nix/Cargo.nix b/nix/Cargo.nix index c65defdac34..35df0c49d89 100644 --- a/nix/Cargo.nix +++ b/nix/Cargo.nix @@ -12047,7 +12047,10 @@ rec { "v1_28" ]; }; - resolvedDefaultFeatures = [ "default" ]; + resolvedDefaultFeatures = [ + "default" + "v1_16" + ]; }; "gstreamer-audio" = rec { crateName = "gstreamer-audio"; @@ -12318,6 +12321,120 @@ rec { "v1_30" = [ "v1_28" ]; }; }; + "gstreamer-check" = rec { + crateName = "gstreamer-check"; + version = "0.25.0"; + edition = "2024"; + sha256 = "1ismmx6kxdd27xhyrjbrqwld21mp62rpr0g0qq2h9hhjhhqhx85w"; + libName = "gstreamer_check"; + authors = [ + "Sebastian Dröge " + ]; + dependencies = [ + { + name = "glib"; + packageId = "glib"; + } + { + name = "gstreamer"; + packageId = "gstreamer"; + rename = "gst"; + } + { + name = "gstreamer-check-sys"; + packageId = "gstreamer-check-sys"; + } + ]; + features = { + "v1_16" = [ + "gst/v1_16" + "gstreamer-check-sys/v1_16" + ]; + "v1_18" = [ + "gst/v1_18" + "gstreamer-check-sys/v1_18" + "v1_16" + ]; + "v1_20" = [ + "gst/v1_20" + "gstreamer-check-sys/v1_20" + "v1_18" + ]; + "v1_22" = [ + "gst/v1_22" + "gstreamer-check-sys/v1_22" + "v1_20" + ]; + "v1_24" = [ + "gst/v1_24" + "gstreamer-check-sys/v1_24" + "v1_22" + ]; + "v1_26" = [ + "gst/v1_26" + "gstreamer-check-sys/v1_26" + "v1_24" + ]; + "v1_28" = [ + "gst/v1_28" + "gstreamer-check-sys/v1_28" + "v1_26" + ]; + "v1_30" = [ + "gst/v1_30" + "gstreamer-check-sys/v1_30" + "v1_28" + ]; + }; + resolvedDefaultFeatures = [ + "default" + "v1_16" + ]; + }; + "gstreamer-check-sys" = rec { + crateName = "gstreamer-check-sys"; + version = "0.25.4"; + edition = "2024"; + sha256 = "175y0w12qj68zi47ir4crb0994bxkr0i55vv0haqws46rc2a3vkv"; + libName = "gstreamer_check_sys"; + authors = [ + "Sebastian Dröge " + ]; + dependencies = [ + { + name = "glib-sys"; + packageId = "glib-sys"; + } + { + name = "gobject-sys"; + packageId = "gobject-sys"; + } + { + name = "gstreamer-sys"; + packageId = "gstreamer-sys"; + } + { + name = "libc"; + packageId = "libc"; + } + ]; + buildDependencies = [ + { + name = "system-deps"; + packageId = "system-deps 9.0.0"; + } + ]; + features = { + "v1_18" = [ "v1_16" ]; + "v1_20" = [ "v1_18" ]; + "v1_22" = [ "v1_20" ]; + "v1_24" = [ "v1_22" ]; + "v1_26" = [ "v1_24" ]; + "v1_28" = [ "v1_26" ]; + "v1_30" = [ "v1_28" ]; + }; + resolvedDefaultFeatures = [ "v1_16" ]; + }; "gstreamer-sys" = rec { crateName = "gstreamer-sys"; version = "0.25.4"; @@ -12362,6 +12479,7 @@ rec { "v1_28" = [ "v1_26" ]; "v1_30" = [ "v1_28" ]; }; + resolvedDefaultFeatures = [ "v1_16" ]; }; "h2" = rec { crateName = "h2"; @@ -19389,6 +19507,12 @@ rec { name = "futures"; packageId = "futures"; } + { + name = "gstreamer-check"; + packageId = "gstreamer-check"; + rename = "gst-check"; + features = [ "v1_16" ]; + } { name = "moq-audio"; packageId = "moq-audio"; @@ -59742,6 +59866,15 @@ rec { packageId = "num-integer"; features = [ "i128" ]; } + { + name = "num-rational"; + packageId = "num-rational"; + usesDefaultFeatures = false; + features = [ + "num-bigint-std" + "std" + ]; + } { name = "num-traits"; packageId = "num-traits"; @@ -61194,6 +61327,15 @@ rec { packageId = "num-integer"; features = [ "i128" ]; } + { + name = "num-rational"; + packageId = "num-rational"; + usesDefaultFeatures = false; + features = [ + "num-bigint-std" + "std" + ]; + } { name = "num-traits"; packageId = "num-traits"; diff --git a/quest/m1/gst-resume-late-flake.md b/quest/m1/gst-resume-late-flake.md deleted file mode 100644 index a5674b0965b..00000000000 --- a/quest/m1/gst-resume-late-flake.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] moq-gst's resume test holds under load - -## Goal - -`source::imp::session_tests::a_closed_publisher_session_resumes_on_the_next_start` -in `rs/moq-gst/src/source/imp.rs` passes however loaded the machine is. - -## Plan - -It failed once under a full `just check` on 2026-11-17 with "the sink dropped -1 as late: 7 of 10 rendered", and passed three runs alone, on a branch that -touched no moq-gst code ([moq-dev/moq#9045](https://github.com/moq-dev/moq/pull/9045)). -A sink judging lateness by the wall clock fails when the machine is busy. -Reproduce it under load first, then fix the root cause, such as a test clock. -Keep the lateness check: `rendered` asserts every frame arrives on time, so a -sink that stops dropping late buffers must keep a separate lateness assertion. -No retry or looser assertion. diff --git a/quest/m3/moqsrc-redial-replay.md b/quest/m3/moqsrc-redial-replay.md new file mode 100644 index 00000000000..96aade126e1 --- /dev/null +++ b/quest/m3/moqsrc-redial-replay.md @@ -0,0 +1,25 @@ +# [S] moqsrc resumes after a redial without replaying what it rendered + +## Goal + +After `moqsrc`'s session redials and the same broadcast instance returns, its +pads continue at the live edge: no group already rendered plays again, and +playback runs no later than before the redial. + +## Plan + +Found by [moq-dev/moq#9168](https://github.com/moq-dev/moq/pull/9168), whose +test clock made it visible in +`source::imp::session_tests::a_protocol_violation_redials_after_playback` +(`rs/moq-gst/src/source/imp.rs`): after the relay closes the session, the new +session replays every cached group of the same instance from pts 0 under a +fresh timeline based at the redial, so all later frames play 500ms behind, +up to `max-delay` (1s by default), and each redial can add that again. +A wall clock hid it because the synced sink just waited. + +Recommendation: keep the instance's `Timeline` across the redial (the +follower already keeps one per instance in `resumable`) and resume each +rendition after the last group it pushed, so replayed groups are skipped as +stale. Then make `rendered` in that test fail on a frame whose clock wait lies +past the time its stamp places it (the 50ms step), which today it lets the +clock catch up to; a wait at its due time stays valid. diff --git a/rs/moq-gst/Cargo.toml b/rs/moq-gst/Cargo.toml index 18c057c218d..b7ea90b971c 100644 --- a/rs/moq-gst/Cargo.toml +++ b/rs/moq-gst/Cargo.toml @@ -38,6 +38,8 @@ gst-plugin-version-helper = "0.8" [dev-dependencies] criterion = { package = "bench-harness", path = "../bench-harness" } futures = { workspace = true } +# A test clock for synced sinks, so lateness checks hold however loaded the machine is. +gst-check = { package = "gstreamer-check", version = "0.25", features = ["v1_16"] } # Decodes the OpusHead the sink publishes, so the surround regression checks playback. moq-audio = { path = "../moq-audio", default-features = false } # Runs the tests that need a variable set or unset in a child process given it. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7b823b22965..ba6ae1dde74 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -3871,7 +3871,10 @@ mod session_tests { video: Option>, /// The video track once [`Self::finish_video`] finished it. finished: Option>, - start: kio::clock::Instant, + /// What frames are stamped from: the system clock, or the test clock [`rendered`] moves it to. + clock: gst::Clock, + /// The `clock` time of timestamp zero. + start: gst::ClockTime, } impl Publisher { @@ -3905,6 +3908,8 @@ mod session_tests { } fn build(mut broadcast: moq_net::broadcast::Producer, config: VideoConfig) -> Self { + gst::init().unwrap(); + let clock = gst::SystemClock::obtain(); let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); let video = broadcast @@ -3924,15 +3929,26 @@ mod session_tests { catalog, video: Some(video), finished: None, - start: kio::clock::now(), + start: clock.time(), + clock, } } /// Write a keyframe whose every byte is `tag`, stamped with the time since publishing so a - /// synced sink renders it on time. - fn write(&mut self, tag: u8) { + /// synced sink renders it on time, and return that timestamp. + fn write(&mut self, tag: u8) -> gst::ClockTime { let frame = self.frame(tag); + let pts = gst::ClockTime::from_useconds(frame.timestamp.as_micros().unwrap() as u64); self.video.as_mut().unwrap().write(frame).unwrap(); + pts + } + + /// Stamp later frames from `clock`, starting over at zero, unless they already are. + fn follow(&mut self, clock: &gst::Clock) { + if self.clock != *clock { + self.clock = clock.clone(); + self.start = clock.time(); + } } /// Finish the video track, keeping it served. @@ -3942,7 +3958,7 @@ mod session_tests { /// The keyframe [`Self::write`] writes for `tag`. fn frame(&self, tag: u8) -> moq_mux::container::Frame { - let elapsed = self.start.elapsed().as_micros() as u64; + let elapsed = (self.clock.time() - self.start).useconds(); moq_mux::container::Frame { timestamp: moq_net::Timestamp::from_micros(elapsed).unwrap(), payload: bytes::Bytes::from(vec![tag; 64]), @@ -3953,7 +3969,7 @@ mod session_tests { /// Move the timestamps of later frames `by` ahead. fn skip(&mut self, by: Duration) { - self.start = self.start.saturating_sub(by); + self.start = self.start.saturating_sub(by.try_into().unwrap()); } } @@ -5878,7 +5894,8 @@ mod session_tests { } /// `moqsrc` playing `room` from `url`, its first video pad linked by name to a synced sink - /// that drops anything more than 100ms late. + /// that drops anything more than 100ms late, on a test clock that moves only when the test + /// moves it. fn pipeline(url: &url::Url) -> (gst::Pipeline, gst::Element) { register(); let first = NEXT_VIDEO_PAD_ID.load(Ordering::Relaxed); @@ -5889,28 +5906,55 @@ mod session_tests { .unwrap() .downcast::() .unwrap(); + pipeline.use_clock(Some(&gst_check::TestClock::new())); let sink = pipeline.by_name("sink").unwrap(); pipeline.set_state(gst::State::Playing).unwrap(); (pipeline, sink) } + /// The test clock [`pipeline`] runs `sink` on. + fn test_clock(sink: &gst::Element) -> gst_check::TestClock { + sink.parent() + .and_downcast::() + .unwrap() + .pipeline_clock() + .downcast() + .unwrap() + } + /// Keep writing `tag` until the sink renders it, then check it renders every frame after. /// A sink renders a late frame when nothing rendered for a second, so one frame proves /// nothing about lateness. + /// + /// The publisher stamps frames from the pipeline's test clock, which stands still while a + /// frame travels, so a frame is late only when moqsrc placed it wrong, however long it took + /// to arrive. A frame placed later than it was stamped waits on the clock, which then moves + /// to it as real time would. fn rendered(sink: &gst::Element, publisher: &mut Publisher, tag: u8) { const FRAMES: usize = 10; + let clock = test_clock(sink); + publisher.follow(clock.upcast_ref()); + // Release the clock wait the next buffer starts, then report whether the sample it + // rendered carries `tag`, and its pts. A test clock releases no wait on its own. let pull = |timeout: Duration| { - let tagged = sink - .emit_by_name::>("try-pull-sample", &[&(timeout.as_nanos() as u64)])? - .buffer() - .is_some_and(|buffer| buffer.map_readable().unwrap()[0] == tag); - Some(tagged) + let (waiting, _) = clock.timed_wait_for_multiple_pending_ids(1, timeout.as_millis() as u32); + if !waiting { + return None; + } + if let Some(due) = clock.next_entry_time().filter(|due| *due > clock.time()) { + clock.set_time(due); + } + clock.process_next_clock_id().unwrap(); + let sample = + sink.emit_by_name::>("try-pull-sample", &[&(timeout.as_nanos() as u64)])?; + let buffer = sample.buffer()?; + Some((buffer.map_readable().unwrap()[0] == tag, buffer.pts())) }; let mut reached = false; for _ in 0..200 { publisher.write(tag); - while let Some(tagged) = pull(Duration::from_millis(50)) { + while let Some((tagged, _)) = pull(Duration::from_millis(50)) { reached |= tagged; } if reached { @@ -5919,18 +5963,15 @@ mod session_tests { } assert!(reached, "{tag} never rendered"); - for _ in 0..FRAMES { - publisher.write(tag); - block(Duration::from_millis(50)); + for frame in 0..FRAMES { + clock.set_time(clock.time() + gst::ClockTime::from_mseconds(50)); + let pts = publisher.write(tag); + // Earlier frames still arriving render before it. The bound is reached only when the sink + // dropped the frame as late. + let on_time = + std::iter::from_fn(|| pull(Duration::from_secs(10))).any(|(tagged, at)| tagged && at == Some(pts)); + assert!(on_time, "the sink dropped {tag} as late: {frame} of {FRAMES} rendered"); } - let mut frames = 0; - while let Some(tagged) = pull(Duration::from_millis(500)) { - frames += usize::from(tagged); - } - assert!( - frames >= FRAMES, - "the sink dropped {tag} as late: {frames} of {FRAMES} rendered" - ); } /// The pipeline played on one video pad with nothing on the bus. @@ -5956,6 +5997,9 @@ mod session_tests { let relay = Relay::new(); let url = relay.url.clone(); let (pipeline, output) = pipeline(&url); + // This checks reconnecting, not timing, and moqsink stamps from the wall clock, which the + // pipeline's test clock never reaches. + output.set_property("sync", false); let sink = glib::Object::builder::() .property("url", url.as_str()) .property("broadcast", "room") @@ -6075,7 +6119,8 @@ mod session_tests { rendered(&sink, &mut old, 1); // Long enough that a restarted run placed at the old segment would be a second late. - block(Duration::from_secs(1)); + let clock = test_clock(&sink); + clock.set_time(clock.time() + gst::ClockTime::SECOND); let mut new = Publisher::new(&relay.origin, Default::default(), video_rendition()); rendered(&sink, &mut new, 2); assert_played_cleanly(pipeline); diff --git a/rs/workspace-hack/Cargo.toml b/rs/workspace-hack/Cargo.toml index 37055b87dfa..7dae87bb81a 100644 --- a/rs/workspace-hack/Cargo.toml +++ b/rs/workspace-hack/Cargo.toml @@ -65,6 +65,7 @@ nom-181b0d20215bb231 = { package = "nom", version = "8" } num-bigint = { version = "0.4" } num-complex = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } +num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] } num-traits = { version = "0.2", features = ["i128"] } object_store = { version = "0.14", default-features = false, features = ["aws-base", "azure-base", "fs", "gcp-base", "reqwest"] } once_cell = { version = "1" } @@ -153,6 +154,7 @@ nom-181b0d20215bb231 = { package = "nom", version = "8" } num-bigint = { version = "0.4" } num-complex = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } +num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] } num-traits = { version = "0.2", features = ["i128"] } object_store = { version = "0.14", default-features = false, features = ["aws-base", "azure-base", "fs", "gcp-base", "reqwest"] } once_cell = { version = "1" }