diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 80fd4003b6b..fd305241664 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -340,6 +340,7 @@ jobs: # It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases. # `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback. # `rs cli-export` runs moq-cli's export process tests, which take tens of seconds of real time. + # `rs hls-conformance` probes every segment moq-hls serves with ffprobe, against its playlists and manifest. # `gate` runs the queue's Check over every package, catching what a # diff-scoped queue run let into main. # `rs send-depth` proves moq-net's futures Send at the default recursion @@ -351,7 +352,7 @@ jobs: # `rs v4l2` streams moq-video's V4L2 capture from the kernel's vivid # virtual camera, a module load no pull request job should make. # `js worklet-matrix` loads `?worklet` output through the bundlers pinned in test/worklet. - recipe: ["gate", "rs doctest --workspace", "rs send-depth", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] + recipe: ["gate", "rs doctest --workspace", "rs send-depth", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] include: # Miri interprets kio's and moq-net's in-memory unsafe; too slow for the merge gate. - recipe: rs miri diff --git a/Cargo.lock b/Cargo.lock index 80ce7dfedf5..b91e96c5c3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5237,6 +5237,7 @@ dependencies = [ "moq-mux", "moq-net", "moq-tokio", + "mp4-atom", "percent-encoding", "rand 0.10.3", "reqwest", @@ -5244,6 +5245,7 @@ dependencies = [ "rind-rustls", "serde", "serde_json", + "tempfile", "thiserror 2.0.21", "tokio", "tower", diff --git a/Cargo.toml b/Cargo.toml index 02f352e0e19..e84f0b10d73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,6 +176,7 @@ moq-vaapi = "0.1.0" # that argument, so moq-ffi and moq-c opt NVIDIA and VAAPI back in (OpenH264 # through their own `openh264` feature, which release builds enable). moq-video = { version = "0.1.10", path = "rs/moq-video", default-features = false } +mp4-atom = "0.17.0" nix = { version = "0.31.3", features = ["net", "socket", "uio"] } # Upstream noq-proto, only for iroh's controller factory types. noq-proto = { version = "1.2", default-features = false } diff --git a/quest/m0/hls-media-grid.md b/quest/m0/hls-media-grid.md index 6d08c9afce2..08c6ea5882f 100644 --- a/quest/m0/hls-media-grid.md +++ b/quest/m0/hls-media-grid.md @@ -66,6 +66,12 @@ start, consumers.md swaps the first-frame pts for the group start. `moq import capture`, whose audio has been on demand since [moq-dev/moq#7238](https://github.com/moq-dev/moq/pull/7238)) starts capture and gets a playlist with both renditions. +- Convert the nightly segment conformance test + (`rs/moq-hls/tests/integration/conformance.rs`, `just rs hls-conformance`), + which declares its segment duration with `with_hls`. Extend it to check that + every rendition of its unaligned ladder shares the grid's segment numbers + while its segment boundaries follow its own keyframes (moved here from the + conformance quest when that test landed, 2026-11-02). - Convert the HLS recorder (`rs/moq-cli/src/hls/record.rs`). Update the hang draft and the HLS, CLI, and archive docs this makes stale, including `doc/concept/archives.md`, whose HLS paragraph says a recording needs its @@ -79,5 +85,4 @@ tracks, adds `media.join.interval`, and moves `clock` to `media.clock`. ## Related - [Consumers by start](/quest/m1/time-addressing/consumers.md) - later swaps the first-frame pts for group starts -- [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - the nightly probes validate this segmentation - [HLS export follows a resolution change](/quest/m1/hls-reinit-same-codec.md) - its hand-over race is reconsidered without records diff --git a/quest/m1/README.md b/quest/m1/README.md index 8046cb14f65..f14d87c7abd 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -172,7 +172,6 @@ per-frame CPU growth since #5926. - [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut - [Interop coverage](/quest/m1/interop-coverage/README.md) - the riskiest moq-transport features moq-net exposes get cross-implementation tests over drafts 18, 20, and 22 - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - A/A and deliberately slowed Nightly runs set the paired comparison's threshold, so its regression alerts can be trusted -- [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - nightly ffprobe checks that advertised timing, sync flags, and bytes match the media served - [Unbounded groups](/quest/m1/unbounded-groups/README.md) - groups and frames lose their size caps; a group keeps a 32 MB tail, and readers behind it replay through paced flights or refresh - [A follower is told when the serving instance's route leaves](/quest/m1/instance-leave-restart.md) - wire-level tests pin the restart a follower gets when the serving instance's route is withdrawn or dies while another instance's remains - [Poll-first async](/quest/m1/poll-first/README.md) - library crates compile no coroutines by default: async wrappers are generated from `poll_*` and gated behind an `async` feature diff --git a/quest/m1/hls-conformance.md b/quest/m1/hls-conformance.md deleted file mode 100644 index 2a5dfdb296e..00000000000 --- a/quest/m1/hls-conformance.md +++ /dev/null @@ -1,51 +0,0 @@ -# [M] HLS and DASH segment conformance, nightly - -## Goal - -A nightly test serves a multi-rendition broadcast (video renditions and -audio) through `moq-hls`, probes every listed segment with ffprobe, and -fails if what the manifests advertise disagrees with the media served: -`EXTINF` and DASH `t`/`d` against each segment's decoded start and duration, -`EXT-X-TARGETDURATION` held for the run, sync flags against `startWithSAP` and -`EXT-X-INDEPENDENT-SEGMENTS`, and the same URL returning the same bytes across -reloads and a mid-run catalog estimate update. - -## Plan - -From the 2026-10-24 moq-hls audit, which found no player-level HLS or DASH -coverage. Decided with the maintainer on 2026-10-24: - -- ffprobe conformance lands in m1, so it validates the grid segmentation of - [HLS export on a declared grid](/quest/m0/hls-media-grid.md) (which - replaced the Catalog HLS field on 2026-11-02) and the fixed target duration - before the cut. A - browser player test follows in m2 - ([HLS and DASH browser playback](/quest/m2/hls-player-test.md)). -- Cover a sliding window that evicts segments, a GOP that overruns the - declared duration, a mid-run reconfiguration (new rendition instance), and a - recording through the segment cursors. -- Cover an unaligned ladder, renditions whose keyframes differ: every - segment is listed and fetched (no `EXT-X-GAP`), and the DASH manifest - omits `segmentAlignment`. Since the grid replaced records (2026-11-02), - every rendition shares the grid's segment numbers while its segment - boundaries follow its own keyframes; the test checks both. -- Runs nightly; move any part fast enough into the merge gate. - -Public API: none. Wire: none. - -Decided 2026-10-30 (maintainer): fix first. The two bugs the test found -are their own quests, required below, and the test lands once it passes -with both checks intact. - -Added 2026-10-31 (maintainer, the /quest-complete requeue session): once -the fixture is repaired, the test's TS import hits a third bug, a -reconfigure failing `JitterDecreased` after a mid-run jitter rise, also its -own quest below. - -## Required - -- [HLS export lists a same-codec reconfigure's first group under the new init](/quest/m1/hls-reinit-same-codec.md) - the conformance test's same-codec switch fails until it lands - -## Related - -- [Export sync flags](/quest/m2/intra-refresh/export-sync-flags.md) - the sync flags a refresh rendition should advertise diff --git a/quest/m1/hls-reinit-same-codec.md b/quest/m1/hls-reinit-same-codec.md index 782af348e4d..6cf688ff9f8 100644 --- a/quest/m1/hls-reinit-same-codec.md +++ b/quest/m1/hls-reinit-same-codec.md @@ -4,7 +4,7 @@ When a rendition changes resolution (or any other decoder-affecting field) while keeping its codec string, the first group after the change is listed under the new init and segment tag, whichever of the catalog and the HLS records reaches the export first. -The conformance test in [moq-dev/moq#7387](https://github.com/moq-dev/moq/pull/7387) is the end-to-end regression test: its `video1` same-codec switch must pass. +A regression test delivers the new config's first group to the export ahead of the catalog update and finds it under the new init. ## Plan @@ -14,12 +14,10 @@ What remains is the hand-over race. The catalog and the records are separate tra The maintainer decided on 2026-11-02 to remove the catalog's `hls` field (`rs/hang/src/catalog/hls.rs` and the JS zod schema); that removal is now [HLS export on a declared grid](/quest/m0/hls-media-grid.md). It changes how the export finds a rendition's records, so the hand-over pick is designed after it lands, not with a catalog `start` field, a record track per config, or a marker in each record (all three dropped the same day). +The nightly conformance test (`rs/moq-hls/tests/integration/conformance.rs`, from [moq-dev/moq#7387](https://github.com/moq-dev/moq/pull/7387)) passes its `video1` same-codec switch, but it neither forces nor observes which of the catalog update and the new config's first group reaches the export first. It is not a test of the race, so the fix needs its own test that forces that order. + Public API: none expected. Wire: decided by the removal plan. ## Required - [HLS export on a declared grid](/quest/m0/hls-media-grid.md) - the catalog's `hls` field and record tracks go away first - -## Related - -- [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - the test that found it, which waits on this diff --git a/quest/m2/dash-reconfigure.md b/quest/m2/dash-reconfigure.md index 757651ccdec..d590ec81104 100644 --- a/quest/m2/dash-reconfigure.md +++ b/quest/m2/dash-reconfigure.md @@ -29,9 +29,10 @@ Choose during implementation by which dash.js and the conformance test accept without a reload, and record the choice in the PR. Reuse the predecessor window #7436 adds rather than tracking a second one, and update `doc/bin/hls.md`. Test that a reconfigure keeps the old segments listed in -the manifest, and extend the DASH checks of -[HLS and DASH segment conformance](/quest/m1/hls-conformance.md) if it has -landed. +the manifest. The segment conformance test +(`rs/moq-hls/tests/integration/conformance.rs`) compares each DASH timeline +with the recorded segments but exempts a retired instance's last segment, +which the manifest never lists today; drop that exemption. Public API: none. Wire: none. diff --git a/quest/m2/fmp4-declared-clock.md b/quest/m2/fmp4-declared-clock.md index a08653a5477..01da2890d9f 100644 --- a/quest/m2/fmp4-declared-clock.md +++ b/quest/m2/fmp4-declared-clock.md @@ -15,7 +15,3 @@ Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-a - Keep #7491's tests (`tfdt` never steps back across sub-GOP fragments, ffprobe reads every fragment) and add a variable-rate B-frame case with a stall longer than the cadence, whose decode times follow the presentation slots across the stall. `DeclaredClock` derives DTS from PTS and depth alone, so a gap that only the encoder's own decode times would show is out of reach and not a goal. Public API: none. Wire: none. - -## Related - -- [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - its nightly ffprobe checks would catch a regression here diff --git a/quest/m2/hls-player-test.md b/quest/m2/hls-player-test.md index 80967d02351..caed20c70a8 100644 --- a/quest/m2/hls-player-test.md +++ b/quest/m2/hls-player-test.md @@ -10,13 +10,9 @@ advancing at real time. ## Plan From the 2026-10-24 moq-hls audit. Decided with the maintainer on 2026-10-24: -it follows [segment conformance](/quest/m1/hls-conformance.md), reusing its -broadcast fixture, and stays nightly since a browser run is slow. Impair the +it follows segment conformance (`rs/moq-hls/tests/integration/conformance.rs`), +reusing its broadcast fixture, and stays nightly since a browser run is slow. Impair the path with bufferbloat (a rate limit below the media bitrate behind a deep, finite queue) to drive a down-switch. Public API: none. Wire: none. - -## Required - -- [Segment conformance](/quest/m1/hls-conformance.md) - the fixture and manifest-to-media agreement this builds on diff --git a/quest/m2/hls-trailing-gap.md b/quest/m2/hls-trailing-gap.md index 19058e514f5..cef932ff0f6 100644 --- a/quest/m2/hls-trailing-gap.md +++ b/quest/m2/hls-trailing-gap.md @@ -9,7 +9,3 @@ A media playlist that ends with `EXT-X-ENDLIST` no longer lists a zero-duration Found by the conformance test in [moq-dev/moq#7387](https://github.com/moq-dev/moq/pull/7387), which allows the gap because the export writes it by design: a track's end writes a zero-duration stop record. Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). Keep the stop record; omit it from the rendered playlist when `EXT-X-ENDLIST` follows, and drop the conformance test's allowance. Public API: none. Wire: none. - -## Related - -- [HLS and DASH segment conformance](/quest/m1/hls-conformance.md) - the test that allows the gap today diff --git a/rs/justfile b/rs/justfile index b1a6fa00faa..0c0cf620a67 100644 --- a/rs/justfile +++ b/rs/justfile @@ -353,6 +353,11 @@ send-depth: cli-export *args: just rs test -p moq-cli --test integration --run-ignored only -E 'test(/^export::/)' "$@" +# Probe every segment moq-hls serves with ffprobe against its HLS playlists and DASH manifest. +[positional-arguments] +hls-conformance *args: + just rs test -p moq-hls --test integration --run-ignored only -E 'test(/^conformance::/)' "$@" + # Exercise small connection and stream receive windows over real QUIC sockets. quic-stress: just rs test -p moq-tokio --lib --run-ignored only -E 'test(/^quic::driver::tests::stress_/)' diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index c62c3427dd4..3d2c94a8bbf 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -45,6 +45,8 @@ futures = { workspace = true } moq-archive = { path = "../moq-archive" } moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "quic", "aws-lc-rs"] } +mp4-atom = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } tracing-test = { workspace = true } diff --git a/rs/moq-hls/tests/integration/conformance.rs b/rs/moq-hls/tests/integration/conformance.rs new file mode 100644 index 00000000000..a7652ff7f1b --- /dev/null +++ b/rs/moq-hls/tests/integration/conformance.rs @@ -0,0 +1,920 @@ +//! Segment conformance: what the HLS playlists and the DASH manifest advertise, against the media +//! served as ffprobe decodes it. +//! +//! ffmpeg encodes a ladder whose renditions disagree on keyframes: `video0` has a 2s GOP with +//! B-frames, `video1` a 3s one that overruns the declared 2s duration and changes resolution +//! halfway (a reconfigure, so a new rendition instance), and AAC audio packs into the grid. The +//! transport stream is imported and served across a moq-net session while every playlist and the +//! manifest are polled. Each segment is fetched while listed, until a short window evicts it, and +//! refetched on later reloads, one right after a catalog estimate update; the segment cursors +//! record it too. Once the broadcast finishes, ffprobe decodes every segment against its init. +//! +//! Ignored by default since it runs ffmpeg and ffprobe; `just rs hls-conformance` runs it nightly. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::http::StatusCode; +use bytes::Bytes; +use moq_hls::Server; +use moq_hls::export::renditions; + +use crate::support::*; + +/// The media's length, and where `video1` changes resolution. +const SECONDS: u32 = 24; +const RECONFIGURE: u32 = 12; + +/// The ladder's renditions: two video, one audio. +const RENDITIONS: usize = 3; + +/// The declared segment duration, which `video1`'s 3s GOPs overrun. +const DURATION: Duration = Duration::from_secs(2); + +/// The playlist window, short enough to evict most of a run's segments. +const WINDOW: Duration = Duration::from_secs(6); + +/// How far a decoded start or duration may sit from what a playlist or manifest advertises: the +/// records carry milliseconds, so one for each end of a span, plus ffprobe's microsecond rounding. +const TOLERANCE: f64 = 0.0025; + +/// How many pieces the transport stream is imported in, polling after each. +const CHUNKS: usize = 96; + +/// How long a poll waits on a held playlist or manifest request before moving on. +const HOLD: Duration = Duration::from_secs(1); + +/// Every how many polls each listed segment is refetched, besides right after the catalog +/// estimate update and once the broadcast ends. +const RELOAD: usize = 8; + +#[tokio::test(start_paused = true)] +#[ignore = "runs ffmpeg and ffprobe; `just rs hls-conformance` runs it"] +async fn segments_match_their_manifests() { + let dir = tempfile::tempdir().expect("tempdir"); + let ts = encode(dir.path()).await; + + let pair = Pair::connect().await; + let publication = pair.publisher.create_broadcast("live").expect("publish"); + publication.announce(Default::default()).expect("announce"); + let mut broadcast = publication.broadcast().clone(); + let hls = moq_mux::hls::Config::default() + .with_duration(DURATION) + .with_duration_hint(DURATION); + let mut catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default().with_hls(hls)) + .expect("catalog"); + let init = moq_mux::import::ContainerInit::new(moq_mux::import::ContainerFormat::Ts, Bytes::new()); + let mut import = moq_mux::import::Container::new(broadcast, catalog.reserve(), &init).expect("import"); + + let mut config = moq_hls::export::Config::default(); + config.window = WINDOW; + let server = Server::new(pair.subscriber.consume(), config).expect("server"); + let app = server.router(); + let mut seen = Seen::default(); + + // The first chunk publishes the catalog, so the recorder follows every rendition from its + // first segment. + let chunk = ts.len().div_ceil(CHUNKS).next_multiple_of(188); + let mut chunks = ts.chunks(chunk); + import.decode(chunks.next().expect("media")).expect("decode"); + let recorder = Recorder::start(&server).await; + + for (i, chunk) in chunks.enumerate() { + import.decode(chunk).expect("decode"); + if i == CHUNKS / 3 { + // What a publisher's estimator writes back: the rendition keeps its instance and its URLs. + // A published jitter only ever rises, so the edit raises it and never lowers it. + let mut catalog = catalog.modify().expect("catalog open"); + for config in catalog.video.renditions.values_mut() { + config.bitrate = Some(1_234_567); + config.jitter = config.jitter.max(Some(Duration::from_millis(40))); + } + } + settle().await; + // Every listed segment is fetched as soon as it is seen, and refetched now and then. + seen.reload = i % RELOAD == 0 || i == CHUNKS / 3 + 1; + seen.poll(&app).await; + // Imported faster than real time, the window would evict segments before the recording + // reads them, so nothing more is imported until it has read what the playlists list. + recorder.caught_up(&seen.newest()).await; + } + seen.reload = true; + import.finish().expect("finish import"); + catalog.finish().expect("finish catalog"); + + let deadline = kio::clock::now() + TIMEOUT; + while seen.poll(&app).await != Polled::Ended { + assert!(kio::clock::now() < deadline, "the playlists never ended"); + kio::clock::sleep(Duration::from_millis(50)).await; + } + let recorded = recorder.finish().await; + + seen.check(&recorded).await; +} + +/// Let everything a chunk produced cross the session: on the paused clock, a sleep ends only once +/// every task is idle. +async fn settle() { + kio::clock::sleep(Duration::from_millis(10)).await; +} + +/// Encode the ladder as one transport stream in `dir`. +async fn encode(dir: &Path) -> Vec { + // video1 in two halves, each opening with its own parameter sets, so the stream reconfigures. + // One level for both keeps the codec string, so the rendition keeps its name and the export + // replaces its instance rather than adding a rendition. + let mut halves = Vec::new(); + for (start, size) in [(0, "320x180"), (RECONFIGURE, "480x270")] { + let path = dir.join(format!("video1.{start}.h264")); + ffmpeg(&[ + "-f", + "lavfi", + "-i", + &format!("testsrc2=size={size}:rate=30"), + "-t", + &(SECONDS - RECONFIGURE).to_string(), + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-x264-params", + "keyint=90:min-keyint=90:scenecut=0:bframes=0:level=3.0", + "-f", + "h264", + path.to_str().expect("utf-8"), + ]) + .await; + halves.extend(std::fs::read(&path).expect("read half")); + } + let video1 = dir.join("video1.h264"); + std::fs::write(&video1, halves).expect("write video1"); + + let ts = dir.join("ladder.ts"); + ffmpeg(&[ + "-f", + "lavfi", + "-i", + "testsrc2=size=640x360:rate=30", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000", + "-framerate", + "30", + "-i", + video1.to_str().expect("utf-8"), + "-t", + &SECONDS.to_string(), + "-map", + "0:v", + "-map", + "2:v", + "-map", + "1:a", + "-c:v:0", + "libx264", + "-preset:v:0", + "veryfast", + "-x264-params:v:0", + "keyint=60:min-keyint=60:scenecut=0:bframes=2", + "-c:v:1", + "copy", + "-c:a", + "aac", + "-b:a", + "96k", + "-max_interleave_delta", + "0", + "-f", + "mpegts", + ts.to_str().expect("utf-8"), + ]) + .await; + std::fs::read(&ts).expect("read ladder") +} + +async fn ffmpeg(args: &[&str]) { + let output = tokio::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(args) + .output() + .await + .expect("run ffmpeg"); + assert!( + output.status.success(), + "ffmpeg {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// Segment cursors over every rendition instance, recording each segment's bytes by rendition +/// name and number. +struct Recorder { + recorded: Arc>, + task: tokio::task::JoinHandle<()>, +} + +impl Recorder { + async fn start(server: &Server) -> Self { + let broadcaster = kio::clock::timeout(TIMEOUT, server.broadcaster("live")) + .await + .expect("resolve timed out") + .expect("announced"); + let recorded = Arc::new(Mutex::new(Recorded::default())); + let task = tokio::spawn({ + let recorded = recorded.clone(); + async move { + let mut renditions = broadcaster.renditions(); + let mut cursors = Vec::new(); + while let Some(event) = renditions.next().await.expect("renditions") { + let renditions::Event::Added(rendition) = event else { + continue; + }; + let recorded = recorded.clone(); + cursors.push(tokio::spawn(async move { + let mut cursor = rendition.segments(); + let name = &rendition.name; + loop { + let segment = match cursor.next().await { + Ok(Some(segment)) => segment, + Ok(None) => break, + Err(err) => { + let mut recorded = recorded.lock().unwrap(); + recorded.failures.push(format!("recording {name}: {err}")); + recorded.failed.insert(name.clone()); + break; + } + }; + let mut recorded = recorded.lock().unwrap(); + // Where content stops, a zero-duration gap ends the segment before it. + let Some(media) = segment.media else { + if !segment.duration.is_zero() { + recorded + .failures + .push(format!("{name} recorded {} as a gap", segment.segment)); + } + continue; + }; + let segments = recorded.segments.entry(name.clone()).or_default(); + if segments.insert(segment.segment, media).is_some() { + recorded + .failures + .push(format!("{name} recorded {} twice", segment.segment)); + } + } + })); + } + for cursor in cursors { + cursor.await.expect("cursor"); + } + } + }); + Self { recorded, task } + } + + /// Wait until each rendition's cursor has recorded the `newest` segment its playlist listed, + /// or failed. + async fn caught_up(&self, newest: &BTreeMap<&str, u64>) { + let deadline = kio::clock::now() + TIMEOUT; + loop { + let behind = { + let recorded = self.recorded.lock().unwrap(); + newest + .iter() + .find(|(name, number)| { + let failed = recorded.failed.contains(**name); + let read = recorded + .segments + .get(**name) + .is_some_and(|read| read.contains_key(number)); + !failed && !read + }) + .map(|(name, number)| format!("{name} {number}")) + }; + let Some(behind) = behind else { + return; + }; + assert!(kio::clock::now() < deadline, "the recording never reached {behind}"); + kio::clock::sleep(Duration::from_millis(10)).await; + } + } + + async fn finish(self) -> Recorded { + kio::clock::timeout(TIMEOUT, self.task) + .await + .expect("the recording never ended") + .expect("recorder"); + std::mem::take(&mut *self.recorded.lock().unwrap()) + } +} + +/// What the segment cursors read. +#[derive(Default)] +struct Recorded { + /// Each rendition's segments by number, across its instances. + segments: BTreeMap>, + /// Cursors that failed or read something unexpected. + failures: Vec, + /// The renditions whose cursor failed. + failed: BTreeSet, +} + +/// One segment as a media playlist lists it. +#[derive(Clone, Debug, PartialEq)] +struct Listed { + /// The segment's URL, under the broadcast. + uri: String, + /// Its init's URL, from the `EXT-X-MAP` before it. + init: String, + /// Its `EXTINF`, in seconds. + duration: f64, + /// Whether an `EXT-X-DISCONTINUITY` precedes it. + discontinuity: bool, + /// Whether it is an `EXT-X-GAP`: nothing to fetch. + gap: bool, +} + +/// One segment as the DASH manifest lists it. +#[derive(Clone, Debug, PartialEq)] +struct Timed { + uri: String, + init: String, + /// `S@t` and `S@d`, in seconds. + start: f64, + duration: f64, +} + +/// How far one poll got, least first. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Polled { + /// A playlist or the manifest was held waiting for media. + Held, + /// Everything rendered, and the broadcast is live. + Live, + /// Everything rendered, and every playlist and the manifest ended. + Ended, +} + +/// Everything the polls saw. +#[derive(Default)] +struct Seen { + /// Each media playlist's segments by number, as first listed. A segment's listing never + /// changes while it stays listed. + playlists: BTreeMap>, + /// Each media playlist's `EXT-X-TARGETDURATION`, which holds for the run. + targets: BTreeMap, + /// The rendition name each media playlist serves. + names: BTreeMap, + /// Each DASH representation's segments by `S@t`, as first listed. + timelines: BTreeMap>, + /// Every segment and init URL fetched, with its bytes from the first fetch. + bodies: BTreeMap, + /// Whether any playlist evicted a segment. + evicted: bool, + /// Whether this poll refetches what earlier ones already fetched. + reload: bool, + /// Fetches that failed while their segment was listed. + failures: Vec, + /// Whether each DASH adaptation set advertised `startWithSAP="1"`, by representation id. + sap: BTreeMap, +} + +impl Seen { + /// Reload the master playlist, every media playlist, and the manifest, fetching every listed + /// segment: new ones for the first time, the rest again on a [`reload`](Self::reload). + async fn poll(&mut self, app: &axum::Router) -> Polled { + // The master holds a request until a rendition is playable, which needs more media. + let Some(master) = render(app, "/live/master.m3u8").await else { + return Polled::Held; + }; + let mut uris: Vec = master + .lines() + .filter(|line| !line.starts_with('#') && line.ends_with(".m3u8")) + .map(str::to_string) + .collect(); + uris.extend( + master + .lines() + .filter_map(|line| Some(attribute(line, "URI")?.to_string())), + ); + + let mut polled = Polled::Ended; + for uri in uris { + polled = polled.min(self.media(app, &uri).await); + } + polled.min(self.manifest(app).await) + } + + async fn media(&mut self, app: &axum::Router, uri: &str) -> Polled { + let base = format!("/live/{}", uri.trim_end_matches("media.m3u8")); + let Some(body) = render(app, &format!("/live/{uri}")).await else { + return Polled::Held; + }; + let name = uri.rsplit('/').nth(1).expect("a rendition"); + let name = percent_encoding::percent_decode_str(name) + .decode_utf8() + .expect("utf-8") + .into_owned(); + self.names.insert(uri.to_string(), name); + + let tag = |prefix: &str| body.lines().find_map(|line| line.strip_prefix(prefix)); + let target: u64 = tag("#EXT-X-TARGETDURATION:") + .expect("a target") + .parse() + .expect("integer"); + let held = *self.targets.entry(uri.to_string()).or_insert(target); + assert_eq!(held, target, "{uri} changed its EXT-X-TARGETDURATION"); + let mut number: u64 = tag("#EXT-X-MEDIA-SEQUENCE:") + .expect("a sequence") + .parse() + .expect("integer"); + + let mut init = None; + let mut duration = None; + let mut discontinuity = false; + let mut gap = false; + let mut listed = Vec::new(); + for line in body.lines() { + if let Some(map) = line.strip_prefix("#EXT-X-MAP:") { + init = Some(format!("{base}{}", attribute(map, "URI").expect("a map URI"))); + } else if let Some(extinf) = line.strip_prefix("#EXTINF:") { + duration = Some(extinf.trim_end_matches(',').parse::().expect("a duration")); + } else if line == "#EXT-X-DISCONTINUITY" { + discontinuity = true; + } else if line == "#EXT-X-GAP" { + gap = true; + } else if !line.starts_with('#') && !line.is_empty() { + let suffix = format!(".{number}.m4s"); + assert!(line.ends_with(&suffix), "{uri}: segment {number} listed as {line}"); + let segment = Listed { + uri: format!("{base}{line}"), + init: init.clone().expect("an EXT-X-MAP before the segment"), + duration: duration.take().expect("an EXTINF before the segment"), + discontinuity: std::mem::take(&mut discontinuity), + gap: std::mem::take(&mut gap), + }; + // Every segment of an unaligned ladder has media. Only where content stops does a + // zero-duration gap end the segment before it. + assert!( + !segment.gap || segment.duration == 0.0, + "{uri} lists segment {number} as a gap:\n{body}" + ); + listed.push((number, segment)); + number += 1; + } + } + + let playlist = self.playlists.entry(uri.to_string()).or_default(); + if let (Some((first, _)), Some((&last, _))) = (listed.first(), playlist.last_key_value()) { + assert!( + *first <= last + 1, + "{uri} evicted segments {} to {first} unseen", + last + 1 + ); + if *first > *playlist.first_key_value().unwrap().0 { + self.evicted = true; + } + } + for (number, segment) in listed { + let first = playlist.entry(number).or_insert_with(|| segment.clone()); + // The discontinuity tag moves to the first listed segment's sequence header once + // the segment before it is evicted, so only the rest of the listing must hold. + assert_eq!( + (&first.uri, &first.init, first.duration, first.gap), + (&segment.uri, &segment.init, segment.duration, segment.gap), + "{uri} relisted segment {number} differently" + ); + if !segment.gap { + for resource in [&segment.init, &segment.uri] { + if let Err(err) = fetch(app, &mut self.bodies, resource, self.reload).await { + self.failures + .push(format!("{uri} segment {number}: {err} while listed:\n{body}")); + } + } + } + } + match body.contains("#EXT-X-ENDLIST") { + true => Polled::Ended, + false => Polled::Live, + } + } + + async fn manifest(&mut self, app: &axum::Router) -> Polled { + let Some(body) = render(app, "/live/manifest.mpd").await else { + return Polled::Held; + }; + assert!( + !body.contains("segmentAlignment"), + "the manifest claims alignment for an unaligned ladder:\n{body}" + ); + + let mut sap = None; + let mut id = None; + let mut template = None; + let mut entries = Vec::new(); + for element in body.split('<').map(|element| element.trim_end()) { + if element.starts_with("AdaptationSet ") { + sap = Some(attribute(element, "startWithSAP") == Some("1")); + } else if element.starts_with("Representation ") { + let representation = attribute(element, "id").expect("an id").to_string(); + self.sap + .insert(representation.clone(), sap.expect("inside an adaptation set")); + id = Some(representation); + } else if element.starts_with("SegmentTemplate ") { + let timescale: f64 = attribute(element, "timescale").expect("a timescale").parse().unwrap(); + let init = unescape(attribute(element, "initialization").expect("an init")); + let media = unescape(attribute(element, "media").expect("a media template")); + template = Some((timescale, format!("/live/{init}"), format!("/live/{media}"))); + } else if element.starts_with("S ") { + let (timescale, init, media) = template.clone().expect("inside a template"); + let t: u64 = attribute(element, "t").expect("an explicit t").parse().unwrap(); + let d: u64 = attribute(element, "d").expect("a d").parse().unwrap(); + assert!(attribute(element, "r").is_none(), "an S repeats: {element}"); + let segment = Timed { + uri: media.replace("$Time$", &t.to_string()), + init, + start: t as f64 / timescale, + duration: d as f64 / timescale, + }; + entries.push((id.clone().expect("inside a representation"), t, segment)); + } + } + + for (id, t, segment) in entries { + let first = self + .timelines + .entry(id.clone()) + .or_default() + .entry(t) + .or_insert_with(|| segment.clone()); + assert_eq!(*first, segment, "{id} relisted t={t} differently"); + for resource in [&segment.init, &segment.uri] { + if let Err(err) = fetch(app, &mut self.bodies, resource, self.reload).await { + self.failures.push(format!("{id} t={t}: {err} while listed")); + } + } + } + match body.contains(r#"type="static""#) { + true => Polled::Ended, + false => Polled::Live, + } + } + + /// The newest segment each rendition's playlist listed with media, by rendition name. + fn newest(&self) -> BTreeMap<&str, u64> { + let mut newest = BTreeMap::new(); + for (uri, playlist) in &self.playlists { + if let Some((number, _)) = playlist.iter().rev().find(|(_, segment)| !segment.gap) { + let entry = newest.entry(self.names[uri].as_str()).or_insert(*number); + *entry = (*entry).max(*number); + } + } + newest + } + + /// The segments of `name` that end a rendition instance a reconfigure retired: the last + /// listed under each init but the newest. + fn retired(&self, name: &str) -> BTreeSet { + let playlists = self.playlists.iter().filter(|(uri, _)| self.names[*uri] == name); + playlists + .flat_map(|(_, playlist)| playlist.iter().zip(playlist.values().skip(1))) + .filter(|((_, segment), next)| segment.init != next.init) + .map(|((number, _), _)| *number) + .collect() + } + + /// Probe everything fetched, and compare it with what was advertised and recorded. + async fn check(&self, recorded: &Recorded) { + assert!(self.evicted, "no playlist evicted a segment"); + assert_eq!(self.playlists.len(), RENDITIONS, "{:?}", self.playlists.keys()); + assert_eq!(self.timelines.len(), RENDITIONS, "{:?}", self.timelines.keys()); + + // Every disagreement, so one run reports them all. + let mut failures = [self.failures.clone(), recorded.failures.clone()].concat(); + let mut fail = |failure: String| failures.push(failure); + let mut probes = Probes::default(); + let mut overran = false; + let mut reconfigured = false; + for (uri, playlist) in &self.playlists { + let name = &self.names[uri]; + let mut previous: Option<(f64, &Listed)> = None; + let numbers: Vec = playlist.keys().copied().collect(); + if numbers.windows(2).any(|pair| pair[1] != pair[0] + 1) { + fail(format!("{uri} skipped a number: {numbers:?}")); + } + for (number, segment) in playlist.iter().filter(|(_, segment)| !segment.gap) { + let Some(probe) = probes.get(&self.bodies, &segment.init, &segment.uri, &mut fail).await else { + continue; + }; + let at = format!("{uri} segment {number}"); + // Every segment opens on a keyframe, advertised or not: the export doesn't emit + // `EXT-X-INDEPENDENT-SEGMENTS`, but players switch renditions at any boundary. + probe.check_sync(&at, &mut fail); + if (probe.duration() - segment.duration).abs() > TOLERANCE { + fail(format!( + "{at}: EXTINF {} but decodes {:.6}", + segment.duration, + probe.duration() + )); + } + // The scenario, not the playlist's EXT-X-TARGETDURATION: a GOP longer than declared. + overran |= segment.duration > DURATION.as_secs_f64() + TOLERANCE; + if let Some((start, previous)) = previous + && !segment.discontinuity + { + let expected = start + previous.duration; + if (probe.start - expected).abs() > TOLERANCE { + fail(format!( + "{at}: decodes from {:.6} but the segment before it ends at {expected:.6}", + probe.start + )); + } + } + reconfigured |= previous.is_some_and(|(_, previous)| previous.init != segment.init); + previous = Some((probe.start, segment)); + + let recording = recorded.segments.get(name).and_then(|segments| segments.get(number)); + if recording != self.bodies.get(&segment.uri) { + fail(format!("{at}: the recording differs from what was served")); + } + } + let listed: Vec = playlist.iter().filter(|(_, s)| !s.gap).map(|(n, _)| *n).collect(); + let recording: Vec = recorded + .segments + .get(name) + .into_iter() + .flat_map(|r| r.keys().copied()) + .collect(); + if recording != listed { + fail(format!("{name}: recorded {recording:?} but listed {listed:?}")); + } + } + if !overran { + fail("no segment overran the declared duration".to_string()); + } + if !reconfigured { + fail("no rendition was reconfigured".to_string()); + } + + for (id, timeline) in &self.timelines { + // The ladder has no intra refresh, so every adaptation set advertises a SAP. + if !self.sap[id] { + fail(format!("{id} doesn't advertise startWithSAP=\"1\"")); + } + // The two forms name the same bytes, so the timeline lists what the cursors recorded. + let (_, name) = id.split_once('/').expect("a {kind}/{name} id"); + let mut timed: Vec<&Bytes> = timeline + .values() + .filter_map(|segment| self.bodies.get(&segment.uri)) + .collect(); + let recording = recorded.segments.get(name).cloned().unwrap_or_default(); + // Except that the manifest lists only the current instance, so the last segment of one + // a reconfigure retires may never be listed (/quest/m2/dash-reconfigure.md). + let retired = self.retired(name); + let missing: Vec = recording + .iter() + .filter(|(number, bytes)| !timed.contains(bytes) && !retired.contains(number)) + .map(|(number, _)| *number) + .collect(); + timed.retain(|bytes| !recording.values().any(|recorded| recorded == *bytes)); + if !missing.is_empty() || !timed.is_empty() { + fail(format!( + "{id}: the timeline misses recorded segments {missing:?} and lists {} unrecorded", + timed.len() + )); + } + for (t, segment) in timeline { + let Some(probe) = probes.get(&self.bodies, &segment.init, &segment.uri, &mut fail).await else { + continue; + }; + let at = format!("{id} t={t}"); + probe.check_sync(&at, &mut fail); + if (probe.start - segment.start).abs() > TOLERANCE { + fail(format!( + "{at}: S@t is {} but decodes from {:.6}", + segment.start, probe.start + )); + } + if (probe.duration() - segment.duration).abs() > TOLERANCE { + fail(format!( + "{at}: S@d is {} but decodes {:.6}", + segment.duration, + probe.duration() + )); + } + } + } + assert!( + failures.is_empty(), + "{} failures:\n{}", + failures.len(), + failures.join("\n") + ); + } +} + +/// GET the playlist or manifest at `uri`, or `None` while the export holds the request waiting for +/// media that only later chunks bring. +async fn render(app: &axum::Router, uri: &str) -> Option { + let response = kio::clock::timeout(HOLD, get(app, uri)).await.ok()?; + (response.status() == StatusCode::OK).then(|| String::from_utf8(response.body().to_vec()).expect("utf-8")) +} + +/// GET `uri` and keep its bytes, or on a `reload`, check they match the first fetch's. +async fn fetch( + app: &axum::Router, + bodies: &mut BTreeMap, + uri: &str, + reload: bool, +) -> Result<(), String> { + if !reload && bodies.contains_key(uri) { + return Ok(()); + } + let response = get(app, uri).await; + if response.status() != StatusCode::OK { + return Err(format!("{uri} answered {}", response.status())); + } + let body = response.into_body(); + if body.is_empty() { + return Err(format!("{uri} is empty")); + } + let first = bodies.entry(uri.to_string()).or_insert_with(|| body.clone()); + if *first != body { + return Err(format!("{uri} changed bytes on a reload")); + } + Ok(()) +} + +/// The value of `name="..."` in an XML element or an HLS attribute list. +fn attribute<'a>(element: &'a str, name: &str) -> Option<&'a str> { + let needle = format!("{name}=\""); + let start = element + .match_indices(&needle) + .map(|(i, _)| i) + .find(|&i| i == 0 || matches!(element.as_bytes()[i - 1], b' ' | b',' | b':'))?; + element[start + needle.len()..].split('"').next() +} + +fn unescape(value: &str) -> String { + value + .replace(""", "\"") + .replace("'", "'") + .replace("<", "<") + .replace(">", ">") + .replace("&", "&") +} + +/// What ffprobe decoded from one init and segment. +#[derive(Clone, Debug)] +struct Probe { + /// The earliest presentation time, in seconds. + start: f64, + /// The latest presentation end, in seconds. + end: f64, + /// Whether the first packet in decode order is a sync sample. + sync: bool, + /// Whether the first frame presented is a keyframe. + leads: bool, +} + +impl Probe { + fn duration(&self) -> f64 { + self.end - self.start + } + + /// A segment opens on a keyframe, decoded and presented first. + fn check_sync(&self, at: &str, fail: &mut impl FnMut(String)) { + if !self.sync { + fail(format!("{at}: opens on a non-sync frame")); + } else if !self.leads { + fail(format!("{at}: opens on a keyframe that isn't presented first")); + } + } +} + +/// ffprobe results by the init and segment they decoded. +#[derive(Default)] +struct Probes(HashMap<(String, String), Option>); + +impl Probes { + /// Probe `segment` against `init`, unless either failed to fetch or ffprobe failed on it. + async fn get( + &mut self, + bodies: &BTreeMap, + init: &str, + segment: &str, + fail: &mut impl FnMut(String), + ) -> Option { + let key = (init.to_string(), segment.to_string()); + if let Some(probe) = self.0.get(&key) { + return probe.clone(); + } + let (init, segment) = (bodies.get(init)?, bodies.get(segment)?); + let mut input = init.to_vec(); + input.extend_from_slice(segment); + let probe = match probe(&input).await { + Ok(mut probe) => { + let shift = reorder(init, segment); + probe.start -= shift; + probe.end -= shift; + Some(probe) + } + Err(err) => { + fail(format!( + "{} against {}: {}", + key.1, + key.0, + err.lines().next().unwrap_or_default() + )); + None + } + }; + self.0.insert(key, probe.clone()); + probe + } +} + +/// How far ffmpeg delays a fragment's presentation: with no edit list, its mov demuxer shifts +/// every presentation time by the most negative composition offset, so none precedes its +/// decode time. A player presents at `tfdt` plus the offset (ISO/IEC 14496-12 8.8.8), so this is +/// taken back off what ffprobe reports. +fn reorder(init: &[u8], segment: &[u8]) -> f64 { + use mp4_atom::{Any, DecodeMaybe}; + + let mut timescale = None; + let mut offset = 0; + let mut cursor = std::io::Cursor::new([init, segment].concat()); + while let Some(atom) = Any::decode_maybe(&mut cursor).expect("an atom") { + match atom { + Any::Moov(moov) => timescale = Some(moov.trak[0].mdia.mdhd.timescale), + Any::Moof(moof) => { + let offsets = moof + .traf + .iter() + .flat_map(|traf| &traf.trun) + .flat_map(|trun| trun.samples.iter()); + offset = offsets.filter_map(|entry| entry.cts).fold(offset, i64::min); + } + _ => {} + } + } + -(offset as f64) / f64::from(timescale.expect("an init")) +} + +/// Decode a whole fMP4 with ffprobe. +async fn probe(input: &[u8]) -> Result { + use tokio::io::AsyncWriteExt; + + let mut child = tokio::process::Command::new("ffprobe") + .args([ + "-v", + "error", + "-show_entries", + "packet=flags:frame=key_frame,pts_time,duration_time", + ]) + .args(["-of", "csv", "-i", "pipe:0"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("run ffprobe"); + let mut stdin = child.stdin.take().expect("stdin"); + let input = input.to_vec(); + let writer = tokio::spawn(async move { stdin.write_all(&input).await }); + let output = child.wait_with_output().await.expect("ffprobe"); + writer.await.expect("writer").map_err(|err| err.to_string())?; + let stderr = String::from_utf8_lossy(&output.stderr); + if !output.status.success() || !stderr.trim().is_empty() { + return Err(format!("ffprobe failed: {stderr}")); + } + + let mut sync = None; + let mut frames = Vec::new(); + for line in String::from_utf8_lossy(&output.stdout).lines() { + let fields: Vec<&str> = line.trim_end_matches(',').split(',').collect(); + match fields.as_slice() { + ["packet", flags] => { + sync.get_or_insert(flags.starts_with('K')); + } + ["frame", key, pts, duration] => { + let pts: f64 = pts.parse().map_err(|_| format!("no pts in {line:?}"))?; + let duration: f64 = duration.parse().map_err(|_| format!("no duration in {line:?}"))?; + frames.push((*key == "1", pts, duration)); + } + _ => return Err(format!("unexpected ffprobe line {line:?}")), + } + } + // Frames come out in presentation order, so the first is the earliest presented. + let (key, first, _) = *frames.first().ok_or("no frames decoded")?; + let start = frames.iter().map(|f| f.1).fold(f64::INFINITY, f64::min); + let end = frames.iter().map(|f| f.1 + f.2).fold(f64::NEG_INFINITY, f64::max); + Ok(Probe { + start, + end, + sync: sync.unwrap_or(false), + leads: key && first == start, + }) +} diff --git a/rs/moq-hls/tests/integration/main.rs b/rs/moq-hls/tests/integration/main.rs index a0783574568..2731ad47381 100644 --- a/rs/moq-hls/tests/integration/main.rs +++ b/rs/moq-hls/tests/integration/main.rs @@ -3,6 +3,7 @@ #![cfg(feature = "server")] +mod conformance; mod disconnect; mod finish; mod restart; diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 918cf69f69f..2babfea57f5 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -27,7 +27,7 @@ moq-json = { workspace = true } moq-loc = { workspace = true } moq-msf = { workspace = true } moq-net = { workspace = true } -mp4-atom = { version = "0.17.0", features = ["tokio", "bytes", "serde"] } +mp4-atom = { workspace = true, features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" num_enum = { workspace = true } roxmltree = "0.21.1"