diff --git a/quest/m1/README.md b/quest/m1/README.md index b584ce62754..547b06d2fa5 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -165,7 +165,6 @@ audit's receive-memory, priority, and lease bugs rank ahead of that reshape. - [Interop audit](/quest/m1/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down - [Lint override and deprecation audit](/quest/m1/audit-lints.md) - repeat the suppression and deprecation audit, starting with the conditional allows and the lanes clippy never runs - [Refuse unknown MOQ_* environment variables](/quest/m1/env-unknown.md) - every binary exits at startup naming a `MOQ_*` variable it does not read -- [HLS cursors error on every incomplete recording](/quest/m1/hls-teardown.md) - the recorder cursors return `Ok(None)` only for a complete recording - [HLS recorder in the CLI](/quest/m1/hls-record-cli.md) - `export hls --output ` records playlists and segments to a directory - [Group storage benchmark](/quest/m1/group-storage-bench.md) - numbers pick the append-only log, a ring buffer, or sorted runs for `GroupState` before either lands - [Unbounded groups](/quest/m1/unbounded-groups/README.md) - groups and frames lose their size caps; the cache sheds parts of a group under pressure and refills the holes in place diff --git a/quest/m1/hls-record-cli.md b/quest/m1/hls-record-cli.md index 1612c397422..f384be06862 100644 --- a/quest/m1/hls-record-cli.md +++ b/quest/m1/hls-record-cli.md @@ -18,7 +18,3 @@ Decided 2026-10-23 (maintainer interview): - This is the cursors' first consumer outside moq-hls's tests. - Update `doc/bin/cli.md` and `doc/bin/hls.md`, and every example that uses `export hls`. - -## Required - -- [HLS cursors error on every incomplete recording](/quest/m1/hls-teardown.md) - an abrupt end or a missed segment must surface as an error before the recorder relies on it diff --git a/quest/m1/hls-teardown.md b/quest/m1/hls-teardown.md deleted file mode 100644 index 7a16bc35de5..00000000000 --- a/quest/m1/hls-teardown.md +++ /dev/null @@ -1,38 +0,0 @@ -# [M] HLS cursors error on every incomplete recording - -## Goal - -moq-hls's recorder cursors (`export::renditions::Consumer` and -`export::segments::Consumer`) return `Ok(None)` only for a complete -recording. Every other way a window closes, such as a broadcaster closed -without finishing, ends the cursor with an error, as does a segment the -cursor misses because it fell behind. A rendition the catalog removes or -reconfigures is a supported transition, not an error: its cursor drains and -ends cleanly, and the renditions cursor reports `Removed`. Each path is -covered by a test. - -## Plan - -Decided 2026-10-23 (maintainer interview), following -[moq-dev/moq#6468](https://github.com/moq-dev/moq/pull/6468), which made a -failed timeline an error but left a window that closes without ending or -failing returning `Ok(None)`: - -- Audit every path that closes a window and decide each as either a clean - finish or an error; a recorder must never mistake a truncated recording - for a complete one. -- Raised in review, scope accepted by the maintainer: a slow cursor misses - media silently today. `segments::State::next_after` resumes at the oldest - row still in the window, and `Consumer::next` skips a segment whose groups - left the relay cache. Both become errors, with a slow-consumer test; a - segment that is a gap for the rendition stays a skip. -- Kept for static VOD export, 2026-10-23 (maintainer): a directory that plays - from a plain CDN with no moq server. Recording-then-serving is already - covered by moq-archive. -- The cursors get a real consumer in - [the CLI recorder](/quest/m1/hls-record-cli.md). -- The Impact bullet says how moq-hls users migrate to the changed end semantics. - -## Related - -- [HLS recorder in the CLI](/quest/m1/hls-record-cli.md) - the cursors' first consumer diff --git a/quest/m2/hls-restart-drain-cutoff.md b/quest/m2/hls-restart-drain-cutoff.md index 503666b03a0..67fc2483128 100644 --- a/quest/m2/hls-restart-drain-cutoff.md +++ b/quest/m2/hls-restart-drain-cutoff.md @@ -34,8 +34,9 @@ Decided 2026-10-24 (iteration, on the merits): - Rejected: an unbounded drain of the old run; an immediate cutoff at `Restart` that discards listed rows; treating a catalog error as media EOF. - A cutoff recording is complete: its cursor returns `Ok(None)` after the last - listed row, as after `End`. Keep this consistent with the end semantics the - [cursor audit](/quest/m1/hls-teardown.md) settles. + listed row, as after `End`. The cursors return `Ok(None)` only for a + confirmed end and `Error::IncompleteRecording` for any other close, so the + cutoff must end the old windows, not just close them. - No new public API: the boundary shows up as the cursor's end. No wire change. Tests: an old publisher that keeps sending after `Restart` still finalizes @@ -45,5 +46,4 @@ catalog EOF and catalog-error behavior are unchanged. ## Related -- [HLS cursors error on every incomplete recording](/quest/m1/hls-teardown.md) - decides which window closes are clean finishes - [HLS recorder in the CLI](/quest/m1/hls-record-cli.md) - the recorder that would see a replaced run diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index 4aa69209aec..6a9f57ed66e 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -149,6 +149,20 @@ pub enum Error { /// The rewind the importer refused. source: moq_mux::Error, }, + /// A recording source closed without confirming its complete end. + #[error("recording closed before finishing")] + IncompleteRecording, + + /// A recording cursor missed rows that left its window or were reset. + #[error("recording cursor missed segments")] + RecordingLagged, + + /// A listed recording segment no longer has all its media available. + #[error("recording segment {segment} is unavailable")] + SegmentUnavailable { + /// The segment whose media cannot be recorded. + segment: u64, + }, } impl Error { diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index ae934c6d07c..c29a06c04cc 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -191,8 +191,8 @@ impl Broadcaster { /// A cursor over rendition add/remove events, for a consumer that mirrors or records the /// whole broadcast. It replays the current renditions, then yields changes, and returns - /// `None` once the source closes. A rendition is added once the catalog advertises its own - /// timeline, which an unhinted publisher declares only after its first record. + /// `Ok(None)` only once the source finishes cleanly. A rendition is added once the catalog + /// advertises its own timeline, which an unhinted publisher declares after its first record. pub fn renditions(&self) -> renditions::Consumer { self.renditions.subscribe() } @@ -443,7 +443,7 @@ async fn watch_catalog( let mut unbound = false; let mut reference: Option = None; - loop { + let result = loop { match kio::wait(|waiter| consumer.poll_next(waiter)).await { Ok(Some(catalog)) => { // Decide the playlists' retention before any rendition starts following its timeline. @@ -483,20 +483,20 @@ async fn watch_catalog( reference = next; } } - Ok(None) => break, + Ok(None) => break Ok(()), Err(err) => { tracing::warn!(%err, "broadcast catalog stream ended with error"); - break; + break Err(err.into()); } } - } + }; - // The source is done (or errored): let recording cursors finish. With every rendition - // disabled, no reference watcher is left to end the windows. - if reference.is_none() { + // With every rendition disabled, no reference watcher is left to end the windows. + // Only a clean catalog finish confirms completion; a catalog failure fails the windows. + if reference.is_none() && result.is_ok() { renditions.fanout().finish(None); } - renditions.close(); + renditions.finish(result); } /// The rendition every segment boundary comes from, with the track its timeline indexes. @@ -584,7 +584,7 @@ async fn watch_timeline( track: reference.2.clone(), source: Box::new(err), }; - if renditions.fail(&reference, err.clone()) { + if renditions.fail(Some(&reference), err.clone()) { tracing::error!(%err, "reference timeline failed; failing every playlist"); } } @@ -915,7 +915,7 @@ mod tests { tokio::time::timeout(Duration::from_millis(50), cursor.next()) .await .ok() - .flatten() + .and_then(Result::unwrap) } /// The average bitrate encoded in a synthesized AAC init segment. @@ -1914,6 +1914,20 @@ mod tests { video0.push(&gop(sequence)).unwrap(); video1.push(&gop(sequence)).unwrap(); } + let media = broadcast.create_track("video0", timed()).unwrap(); + for sequence in 0..4 { + let mut group = media.append_group().unwrap(); + let frame = vp8_frame(sequence * 2_000_000, true); + hang::container::Frame { + timestamp: frame.timestamp, + payload: frame.payload, + } + .write_to(&mut group) + .unwrap(); + group.finish().unwrap(); + } + media.finish().unwrap(); + let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let rendition = broadcaster.rendition(Kind::Video, "video0").unwrap(); @@ -4050,7 +4064,7 @@ mod tests { let added = tokio::time::timeout(Duration::from_secs(5), renditions.next()) .await .expect("a rendition is discovered"); - let Some(renditions::Event::Added(_rendition)) = added else { + let Ok(Some(renditions::Event::Added(_rendition))) = added else { panic!("expected an Added event"); }; @@ -4061,7 +4075,7 @@ mod tests { .await .expect("the cursor resolves after the broadcaster drops"); assert!( - matches!(next, Some(renditions::Event::Removed { .. })), + matches!(next, Ok(Some(renditions::Event::Removed { .. }))), "dropping the broadcaster releases its renditions to a live cursor" ); @@ -4070,14 +4084,14 @@ mod tests { let ended = tokio::time::timeout(Duration::from_secs(5), renditions.next()) .await .expect("the cursor terminates after the broadcaster drops"); - assert!(ended.is_none(), "the cursor ends once the broadcaster is gone"); + assert!(matches!(ended, Err(crate::Error::IncompleteRecording))); drop((catalog, media, registration, broadcast)); } // The record path: the renditions cursor yields the video rendition, its segment cursor - // yields each finalized segment WITH its media, and closing the publisher drains the final - // segment and then ends both cursors. + // yields each finalized segment with its media, and losing the publisher mid-group fails + // both cursors instead of completing a truncated recording. #[tokio::test] async fn record_cursors_yield_renditions_and_segments() { let origin = produce_origin(); @@ -4110,6 +4124,7 @@ mod tests { let rendition = match tokio::time::timeout(Duration::from_secs(5), renditions.next()) .await .expect("a rendition is discovered") + .unwrap() { Some(renditions::Event::Added(rendition)) => rendition, other => panic!("expected an Added event, got {:?}", other.is_some()), @@ -4149,11 +4164,150 @@ mod tests { end.map(|segment| segment.map(|segment| segment.segment)) ); - // The renditions cursor also ends once the source closes. + // The renditions cursor also fails when the source closes abruptly. let ended = tokio::time::timeout(Duration::from_secs(5), renditions.next()) .await .expect("renditions cursor resolves"); - assert!(ended.is_none(), "renditions cursor ends when the broadcast closes"); + assert!(ended.is_err(), "an abrupt broadcast close fails the renditions cursor"); + } + + #[tokio::test(start_paused = true)] + async fn a_segment_cursor_rejects_an_unfinished_window() { + let test = Timelines::new(); + let rendition = test.rendition("video0"); + let mut cursor = rendition.segments(); + rendition.close(); + assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); + assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); + } + + #[tokio::test(start_paused = true)] + async fn a_failed_catalog_fails_unfinished_segment_windows() { + let test = Timelines::new(); + let rendition = test.rendition("video0"); + let mut segments = rendition.segments(); + test.renditions.finish(Err(crate::Error::NoVariants)); + assert!(matches!(segments.next().await, Err(crate::Error::NoVariants))); + } + + #[tokio::test(start_paused = true)] + async fn a_slow_cursor_refuses_evicted_rows_before_its_first_read() { + let test = Timelines::new(); + let rendition = test.rendition("video0"); + let mut slow = rendition.segments(); + let reference = reference(&test.catalog, None).unwrap(); + for sequence in 0..20 { + test.renditions.fanout().push( + sequence, + moq_mux::timeline::Entry::decode(gop(sequence), moq_net::Timescale::MILLI).unwrap(), + &reference, + ); + } + assert!(rendition.snapshot().media_sequence > 0); + assert!(matches!(slow.next().await, Err(crate::Error::RecordingLagged))); + assert!(matches!(slow.next().await, Err(crate::Error::RecordingLagged))); + } + + #[tokio::test(start_paused = true)] + async fn a_recording_refuses_missing_media_but_skips_rendition_gaps() { + let test = Timelines::new(); + let (_, mut video0) = test.publish("video0"); + let (_, mut video1) = test.publish("video1"); + let media = test.media("video0", 2); + let sibling = test.media("video1", 2); + media.finish().unwrap(); + sibling.finish().unwrap(); + for sequence in 0..3 { + video0.push(&gop(sequence)).unwrap(); + } + // The sibling timeline explicitly has a gap for segment 0, then covers segment 1. + video1 + .push(&hang::timeline::Record::new( + 0, + 2_000, + 2_000, + hang::timeline::Position::group(1), + hang::timeline::Position::group(2), + )) + .unwrap(); + video1.finish().unwrap(); + video0.finish().unwrap(); + test.finished("video0").await; + let rendition = test.rendition("video0"); + let mut cursor = rendition.segments(); + assert_eq!(cursor.next().await.unwrap().unwrap().segment, 0); + assert_eq!(cursor.next().await.unwrap().unwrap().segment, 1); + assert!(matches!( + cursor.next().await, + Err(crate::Error::SegmentUnavailable { segment: 2 }) + )); + assert!(matches!( + cursor.next().await, + Err(crate::Error::SegmentUnavailable { segment: 2 }) + )); + + let rendition = test.rendition("video1"); + let mut cursor = rendition.segments(); + assert_eq!(cursor.next().await.unwrap().unwrap().segment, 1); + assert!(cursor.next().await.unwrap().is_none()); + } + + #[tokio::test(start_paused = true)] + async fn catalog_changes_report_removal_and_drain_old_cursors() { + for reconfigure in [false, true] { + let mut test = Timelines::new(); + let (_, mut timeline) = test.publish("video0"); + let _media = test.media("video0", 2); + timeline.push(&gop(0)).unwrap(); + test.listed("video0", 0).await; + let mut changes = test.renditions.subscribe(); + let mut old = None; + for _ in 0..2 { + if let Some(renditions::Event::Added(rendition)) = changes.next().await.unwrap() + && rendition.name == "video0" + { + old = Some(rendition.segments()); + } + } + let mut old = old.unwrap(); + if reconfigure { + test.catalog.video.renditions.get_mut("video0").unwrap().coded_width = Some(1280); + } else { + test.catalog.video.renditions.remove("video0"); + } + let source = moq_mux::Source::new(produce_origin().consume(), test.broadcast.consume()); + test.renditions.sync(&source, &test.catalog); + assert!( + matches!(changes.next().await.unwrap(), Some(renditions::Event::Removed { name, .. }) if name == "video0") + ); + assert_eq!(old.next().await.unwrap().unwrap().segment, 0); + assert!(old.next().await.unwrap().is_none()); + if reconfigure { + assert!( + matches!(changes.next().await.unwrap(), Some(renditions::Event::Added(rendition)) if rendition.name == "video0") + ); + } + } + } + + #[tokio::test(start_paused = true)] + async fn a_removed_rendition_keeps_draining_its_delayed_timeline() { + let mut test = Timelines::new(); + let (_, mut reference) = test.publish("video0"); + let (_, mut sibling) = test.publish("video1"); + let _media = test.media("video1", 1); + reference.push(&gop(0)).unwrap(); + test.listed("video0", 0).await; + let rendition = test.rendition("video1"); + let mut cursor = rendition.segments(); + test.catalog.video.renditions.remove("video1"); + let source = moq_mux::Source::new(produce_origin().consume(), test.broadcast.consume()); + test.renditions.sync(&source, &test.catalog); + // Removal closes the window, while its existing own-timeline watcher stays alive. + sibling.push(&gop(0)).unwrap(); + sibling.finish().unwrap(); + assert_eq!(cursor.next().await.unwrap().unwrap().segment, 0); + assert!(cursor.next().await.unwrap().is_none()); } /// A two-second GOP record: record `n` covers group `n` from `2n` seconds. diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index e4ac4912411..2d2e6e00447 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -466,6 +466,11 @@ impl Rendition { .resolve(self.kind == Kind::Video, Duration::from(row.pts)..row.end) } + /// Whether the rendition declares no media for this row. + pub(crate) fn is_gap(&self, row: &segments::Row) -> bool { + self.resolve(row) == Content::Gap + } + /// Poll until `row` resolves on this rendition. pub(crate) fn poll_resolved(&self, waiter: &kio::Waiter, row: &segments::Row) -> Poll<()> { if self.is(&row.reference) { @@ -543,11 +548,8 @@ impl Rendition { self.live.subscribe(self.clone()) } - /// Retire this rendition: close its window so every [`segments::Consumer`] over it drains - /// what's left and then ends. - /// - /// Called when the catalog drops or replaces the rendition; a cursor's `Arc` would - /// otherwise park forever on a window nothing feeds anymore. + /// Close the window after recording its end or failure, so segment cursors drain it. + /// An unmarked close fails the recording instead of reporting completion. pub(crate) fn close(&self) { self.live.close(); } diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index 3b61cbb6531..f75a3f825d8 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -8,10 +8,10 @@ //! A [`Consumer`] is a cursor for a recorder that mirrors the *whole* broadcast: [`next`](Consumer::next) //! yields one [`Event`] at a time as renditions are added or removed, replaying the current //! set as [`Added`](Event::Added) when first consumed, and returning `None` once the source -//! closes. +//! finishes cleanly, or an error when the recording is incomplete. use std::collections::{BTreeMap, VecDeque}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Weak}; use std::task::Poll; use std::time::{Duration, SystemTime}; @@ -107,6 +107,8 @@ pub(crate) struct Producer { #[derive(Default)] struct State { renditions: BTreeMap>, + /// The catalog's confirmed end, distinct from dropping its watcher. + finished: Option>, // Cache catalog quality order without making any_held_externally count it as a consumer. audio: Vec>, } @@ -118,7 +120,7 @@ struct State { /// channel open (so `renditions()` cursors would never terminate). #[derive(Clone)] pub(crate) struct Fanout { - feed: Arc>, + feed: kio::Producer, } impl Producer { @@ -126,7 +128,7 @@ impl Producer { Self { state: kio::Producer::new(State::default()), fanout: Fanout { - feed: Arc::new(Mutex::new(Feed { + feed: kio::Producer::new(Feed { targets: Vec::new(), history: VecDeque::new(), last: None, @@ -139,7 +141,7 @@ impl Producer { anchor: None, generation: None, window: Some(window), - })), + }), }, } } @@ -152,24 +154,25 @@ impl Producer { /// The playlist window duration every rendition's window is trimmed to, or `None` when /// only the source timeline trims them. pub fn window(&self) -> Option { - self.fanout.feed.lock().unwrap().window + self.fanout.feed.read().window } /// The estimated wall-clock time of timeline `pts` 0 (see [`Feed::anchor`]); `None` until /// the first record arrives. pub fn anchor(&self) -> Option { - self.fanout.feed.lock().unwrap().anchor + self.fanout.feed.read().anchor } /// Why a reference timeline the playlists list failed, if one did. pub fn failure(&self) -> Option { - self.fanout.feed.lock().unwrap().failed.clone() + self.fanout.feed.read().failed.clone() } /// A cursor over rendition changes, replaying the current set as [`Event::Added`]. pub fn subscribe(&self) -> Consumer { Consumer { state: self.state.consume(), + feed: self.fanout.feed.consume(), seen: BTreeMap::new(), } } @@ -229,6 +232,17 @@ impl Producer { } } + /// Record the catalog's end before closing its change stream. + pub fn finish(&self, result: crate::Result<()>) { + if let Err(err) = &result { + self.fanout.fail(None, err.clone()); + } + if let Ok(mut state) = self.state.write() { + state.finished = Some(result); + } + self.close(); + } + /// Close the channel, signalling consumers that no more renditions will appear. /// /// Deliberately does NOT cascade into the renditions' own segment windows: those are ended @@ -243,19 +257,22 @@ impl Fanout { /// List every record the source timeline retains, trimming only on its pops, instead of /// a window. Call before the first record. pub fn unbound(&self) { - self.feed.lock().unwrap().window = None; + self.feed + .write() + .unwrap_or_else(|_| unreachable!("fanout is open")) + .window = None; } /// Select the next reference, whose records list only past the last advertised one. pub fn select(&self, reference: Option) { - let mut feed = self.feed.lock().unwrap(); + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); feed.selection = Some(reference); feed.joining = true; } /// Fan one reference timeline record out to every living rendition, and into replay history. pub fn push(&self, index: u64, entry: Entry, reference: &Reference) { - let mut feed = self.feed.lock().unwrap(); + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); if !feed.accepts(reference) || feed.failed.is_some() { return; } @@ -342,7 +359,7 @@ impl Fanout { /// Remove records that left the source timeline window from every rendition window. pub fn pop(&self, range: std::ops::Range, reference: &Reference) { - let mut feed = self.feed.lock().unwrap(); + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); let Some(through) = feed .history .iter() @@ -370,7 +387,7 @@ impl Fanout { /// belong to the run it named. The first one only labels the run already flowing, so an embedder that supplies /// it right after construction loses nothing to a race with the first records. pub fn set_generation(&self, generation: Option>) { - let mut feed = self.feed.lock().unwrap(); + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); if feed.generation == generation { return; } @@ -397,7 +414,7 @@ impl Fanout { /// Clear stale rows after source records were skipped before this reader saw them. pub fn skip(&self, reference: Option<&Reference>) { - let mut feed = self.feed.lock().unwrap(); + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); if let Some(reference) = reference && (feed.joining || !feed.accepts(reference)) { @@ -420,7 +437,7 @@ impl Fanout { /// End every window for the selected watcher, ignoring a superseded watcher's completion. /// `None` ends them when the source ended with no reference selected, so no watcher will. pub fn finish(&self, reference: Option<&Reference>) -> bool { - let mut feed = self.feed.lock().unwrap(); + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); let selected = match reference { Some(reference) => feed.accepts(reference), None => feed.selection.as_ref().is_none_or(Option::is_none), @@ -443,11 +460,14 @@ impl Fanout { /// Fail every window with `err` when `reference`'s timeline failed, so no playlist reads as /// finished. A superseded reader counts only in history mode while rows it must pop are still - /// listed; a live window trims a superseded reference's rows on its own. - pub fn fail(&self, reference: &Reference, err: crate::Error) -> bool { - let mut feed = self.feed.lock().unwrap(); - let listed = feed.accepts(reference) - || (feed.window.is_none() && feed.history.iter().any(|row| row.reference == *reference)); + /// listed; a live window trims a superseded reference's rows on its own. `None` fails the + /// whole source when the catalog fails. + pub fn fail(&self, reference: Option<&Reference>, err: crate::Error) -> bool { + let mut feed = self.feed.write().unwrap_or_else(|_| unreachable!("fanout is open")); + let listed = reference.is_none_or(|reference| { + feed.accepts(reference) + || (feed.window.is_none() && feed.history.iter().any(|row| row.reference == *reference)) + }); if feed.failed.is_some() || !listed { return false; } @@ -512,7 +532,11 @@ impl Producer { /// (and the ended/failed/closed markers) so its window matches its siblings', and start following its /// own timeline. fn register(&self, rendition: &Arc) { - let mut feed = self.fanout.feed.lock().unwrap(); + let mut feed = self + .fanout + .feed + .write() + .unwrap_or_else(|_| unreachable!("fanout is open")); rendition.label(feed.generation.clone()); // An unwindowed feed is history mode: every rendition reads its timeline from the start. rendition.watch(feed.window.is_none()); @@ -592,6 +616,7 @@ impl Producer { .collect(); for key in stale { if let Some(rendition) = current.renditions.remove(&key) { + rendition.end(); rendition.close(); } } @@ -655,15 +680,18 @@ impl Producer { /// a fresh consumer replays every current rendition as [`Event::Added`]. pub struct Consumer { state: kio::Consumer, + feed: kio::Consumer, /// The renditions this cursor has already reported, by identity, so a reconfigure (a /// replaced `Arc`) is seen as a remove followed by an add. seen: BTreeMap>, } impl Consumer { - /// The next rendition change; `None` once the source broadcast closes. - pub async fn next(&mut self) -> Option { - let event = kio::wait(|waiter| self.poll_next(waiter)).await?; + /// The next rendition change; `None` only after the catalog and reference finish cleanly. + pub async fn next(&mut self) -> crate::Result> { + let Some(event) = kio::wait(|waiter| self.poll_next(waiter)).await? else { + return Ok(None); + }; match &event { Event::Added(rendition) => { self.seen @@ -673,17 +701,36 @@ impl Consumer { self.seen.remove(&(*kind, name.clone())); } } - Some(event) + Ok(Some(event)) } - fn poll_next(&self, waiter: &kio::Waiter) -> Poll> { - match self + fn poll_next(&self, waiter: &kio::Waiter) -> Poll>> { + let finished = match self .state .poll(waiter, |current| next_event(¤t.renditions, &self.seen)) { - Poll::Ready(Ok(event)) => Poll::Ready(Some(event)), - // Closed with no pending diff: no more changes. - Poll::Ready(Err(_)) => Poll::Ready(None), + Poll::Ready(Ok(event)) => return Poll::Ready(Ok(Some(event))), + Poll::Ready(Err(state)) => { + match &state.finished { + Some(result) => result.clone()?, + None => return Poll::Ready(Err(crate::Error::IncompleteRecording)), + } + true + } + Poll::Pending => false, + }; + // A failed reference cannot wait for an otherwise live catalog to close. + match self.feed.poll(waiter, |feed| { + if let Some(err) = &feed.failed { + Poll::Ready(Err(err.clone())) + } else if feed.ended && finished { + Poll::Ready(Ok(None)) + } else { + Poll::Pending + } + }) { + Poll::Ready(Ok(result)) => Poll::Ready(result), + Poll::Ready(Err(_)) => Poll::Ready(Err(crate::Error::IncompleteRecording)), Poll::Pending => Poll::Pending, } } @@ -717,6 +764,68 @@ fn next_event(current: &BTreeMap>, seen: &BTreeMap Entry { let record = hang::timeline::Record::new( sequence, @@ -739,7 +848,7 @@ mod tests { } fn rows(fanout: &Fanout) -> Vec<(u64, u64, Duration)> { - let feed = fanout.feed.lock().unwrap(); + let feed = fanout.feed.read(); feed.history .iter() .map(|row| (row.segment, row.discontinuity, Duration::from(row.period))) @@ -800,7 +909,7 @@ mod tests { fanout.skip(Some(&old)); fanout.push(1, entry(1, 2_000), &old); fanout.push(1, entry(1, 2_000), &replacement); - assert_eq!(fanout.feed.lock().unwrap().history.len(), 2); + assert_eq!(fanout.feed.read().history.len(), 2); } #[test] @@ -808,9 +917,9 @@ mod tests { let err = || crate::Error::NoVariants; // Live: a superseded reader's rows trim with the window, so its failure changes nothing. let (fanout, new) = handed_off(5); - let old = fanout.feed.lock().unwrap().history[0].reference.clone(); - assert!(!fanout.fail(&old, err())); - assert!(fanout.fail(&new, err())); + let old = fanout.feed.read().history[0].reference.clone(); + assert!(!fanout.fail(Some(&old), err())); + assert!(fanout.fail(Some(&new), err())); assert!(!fanout.finish(Some(&new)), "a failed feed never finishes"); // History: a retained reader fails the feed while rows it must pop stay listed. @@ -824,7 +933,7 @@ mod tests { if popped { fanout.pop(0..1, &old); } - fanout.fail(&old, err()) + fanout.fail(Some(&old), err()) }; assert!(history(false)); assert!(!history(true)); @@ -846,19 +955,19 @@ mod tests { fanout.push(0, entry(0, 0), &new); fanout.push(6, entry(6, 12_000), &new); { - let feed = fanout.feed.lock().unwrap(); + let feed = fanout.feed.read(); assert_eq!(feed.history.len(), 2); assert_eq!(feed.history.back().unwrap().segment, 6); assert!(!feed.ended); assert_eq!(feed.history.back().unwrap().discontinuity, 0); } fanout.pop(0..6, &new); - assert_eq!(fanout.feed.lock().unwrap().history.len(), 2); + assert_eq!(fanout.feed.read().history.len(), 2); fanout.pop(0..6, &old); - assert_eq!(fanout.feed.lock().unwrap().history.len(), 1); + assert_eq!(fanout.feed.read().history.len(), 1); fanout.pop(6..7, &new); - assert!(fanout.feed.lock().unwrap().history.is_empty()); + assert!(fanout.feed.read().history.is_empty()); fanout.push(7, entry(7, 14_000), &new); - assert_eq!(fanout.feed.lock().unwrap().history.back().unwrap().segment, 7); + assert_eq!(fanout.feed.read().history.back().unwrap().segment, 7); } } diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 451edd2cb22..85bbc421239 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -20,7 +20,7 @@ use std::collections::VecDeque; use std::sync::Arc; -use std::task::Poll; +use std::task::{Poll, ready}; use std::time::{Duration, SystemTime}; use bytes::Bytes; @@ -51,6 +51,8 @@ struct State { failed: Option, /// Bumped whenever the window is cleared, so a latch on what it listed can tell it is stale. resets: u64, + /// Total rows appended, so a cursor can detect eviction even before its first read. + produced: u64, } /// Numbers the broadcast's content timeline: the sequence bumps wherever the timeline breaks. @@ -223,21 +225,17 @@ impl State { } } - /// The first segment past `after`, for a cursor. - /// - /// Rows are complete the moment they arrive. Segments evicted from the front of the - /// window before the cursor reached them are skipped: the cursor resumes at the oldest - /// row still in the window. A reset starts a new window, whose numbers may restart lower. - fn next_after(&self, after: Option<&(u64, u64)>) -> Next { - let next = self.rows.iter().find(|r| match after { - Some((resets, after)) => *resets != self.resets || r.segment > *after, - None => true, - }); - match next { + /// The row at a cursor's append position, refusing any lost prefix or reset. + fn next_at(&self, position: (u64, u64)) -> Result { + let first = self.produced - self.rows.len() as u64; + if position.0 != self.resets || position.1 < first { + return Err(crate::Error::RecordingLagged); + } + Ok(match self.rows.get((position.1 - first) as usize) { Some(row) => Next::Ready(row.clone()), None if self.ended || self.failed.is_some() => Next::Done, None => Next::Pending, - } + }) } } @@ -252,6 +250,7 @@ impl Producer { ended: false, failed: None, resets: 0, + produced: 0, }), } } @@ -279,6 +278,7 @@ impl Producer { state.discontinuity = row.discontinuity; state.rows.push_back(row); + state.produced += 1; while let Some(window) = window { let rest = match state.rows.get(1) { @@ -347,8 +347,8 @@ impl Producer { } /// Close the channel: no more rows will arrive. A [`Consumer`] drains the segments it - /// can still see and then ends; the serve path keeps reading the frozen window. Call after - /// [`end`](Self::end) when the timeline is over, or on its own when a rendition is retired. + /// can still see and then errors unless [`end`](Self::end) or [`fail`](Self::fail) confirmed + /// how it ended. The serve path keeps reading the frozen window. pub fn close(&self) { let _ = self.state.close(); } @@ -455,10 +455,12 @@ impl Producer { /// A cursor over segments, starting from the oldest still in the window. pub fn subscribe(&self, rendition: Arc) -> Consumer { + let consumer = self.state.consume(); + let state = self.state.read(); Consumer { - state: self.state.consume(), + state: consumer, rendition, - after: None, + position: (state.resets, state.produced - state.rows.len() as u64), feed: None, } } @@ -495,9 +497,8 @@ pub struct Segment { pub struct Consumer { state: kio::Consumer, rendition: Arc, - /// Last fetched or skipped segment, with its window reset epoch; errors leave it unchanged - /// so callers can retry. - after: Option<(u64, u64)>, + /// The reset epoch and next append position; errors leave it unchanged. + position: (u64, u64), feed: Option, } @@ -533,14 +534,14 @@ impl Consumer { /// denied referenced broadcast fails every call with its authorization error. /// /// Waits for the next segment to resolve on this rendition, then FETCHes and transmuxes its - /// frames. A segment whose groups already left the relay cache (or that is a gap for this - /// rendition) is skipped, resuming at the next one, rather than surfaced as an error; a real - /// fetch/transmux failure is returned, leaving the cursor to retry it on the next call. + /// frames. A gap for this rendition is skipped. Missing window rows or unavailable media + /// fail the recording instead of silently omitting segments. Fetch/transmux errors leave + /// the cursor in place so the caller can retry. /// A bound instance's retained media drains after End or Restart; a replacement's media /// is never read under the old rows. pub async fn next(&mut self) -> Result> { loop { - let Some((row, resets)) = kio::wait(|waiter| self.poll_next(waiter)).await? else { + let Some(row) = kio::wait(|waiter| self.poll_next(waiter)).await? else { return Ok(None); }; kio::wait(|waiter| { @@ -556,44 +557,46 @@ impl Consumer { { return Err(err); } - let media = self.rendition.fetch(&row).await?; - self.after = Some((resets, row.segment)); - if let Some(media) = media { - return Ok(Some(Segment { - segment: row.segment, - media, - duration: row.duration, - program_date_time: self.rendition.wall_clock(row.pts), - discontinuity: row.discontinuity, - })); + if self.rendition.is_gap(&row) { + self.position.1 += 1; + continue; } + let media = self + .rendition + .fetch(&row) + .await? + .ok_or(crate::Error::SegmentUnavailable { segment: row.segment })?; + self.position.1 += 1; + return Ok(Some(Segment { + segment: row.segment, + media, + duration: row.duration, + program_date_time: self.rendition.wall_clock(row.pts), + discontinuity: row.discontinuity, + })); } } - fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>> { + fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>> { self.poll_feed(waiter); // A refused binding never held media, so there is nothing to drain. `poll_feed` registered // the wake for a refusal that arrives while the window waits for rows. if let Err(err) = self.rendition.authorize() { return Poll::Ready(Err(err)); } - let poll = self - .state - .poll(waiter, |state| match state.next_after(self.after.as_ref()) { - Next::Ready(row) => Poll::Ready(Some((row, state.resets))), - Next::Done => Poll::Ready(None), + let found = match ready!(self.state.poll(waiter, |state| -> Poll>> { + match state.next_at(self.position)? { + Next::Ready(row) => Poll::Ready(Ok(Some(row))), + Next::Done => Poll::Ready(Ok(None)), Next::Pending => Poll::Pending, - }); - // A closed window (the rendition was retired) has nothing more, and its guard must drop - // before `check` reads the window again. - let found = match poll { - Poll::Ready(found) => found.unwrap_or(None), - Poll::Pending => return Poll::Pending, + } + })) { + Ok(found) => found?, + Err(_) => return Poll::Ready(Err(crate::Error::IncompleteRecording)), }; if found.is_some() { return Poll::Ready(Ok(found)); } - // Nothing left to drain: end where the playlist does, failing if it fails. Poll::Ready(self.rendition.check().map(|()| None)) } @@ -729,7 +732,7 @@ mod tests { #[test] fn a_continuous_timeline_keeps_one_discontinuity() { - // A cursor that skips a record (uncached, or a gap for its rendition) must not mark the + // A cursor that skips a declared gap for its rendition must not mark the // next one: a sibling that fetched it wouldn't, and players require renditions to agree. let mut timeline = Discontinuities::default(); assert_eq!(timeline.stamp(secs(0), secs(2)), 0); @@ -807,53 +810,56 @@ mod tests { } #[test] - fn next_after_walks_segments() { + fn cursor_walks_segments() { let live = Producer::new(); let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 2_000), window); live.push(row(1, 1, 2_000, 2_000), window); - let Next::Ready(first) = live.state.read().next_after(None) else { + let Next::Ready(first) = live.state.read().next_at((0, 0)).unwrap() else { panic!("expected a segment"); }; assert_eq!(first.segment, 0); - let after = |segment| Some((0, segment)); - let Next::Ready(second) = live.state.read().next_after(after(0).as_ref()) else { + let next = |segment| (0, segment); + let Next::Ready(second) = live.state.read().next_at(next(1)).unwrap() else { panic!("expected a segment"); }; assert_eq!(second.segment, 1); assert!( - matches!(live.state.read().next_after(after(1).as_ref()), Next::Pending), + matches!(live.state.read().next_at(next(2)).unwrap(), Next::Pending), "nothing further while live" ); live.end(); - assert!(matches!(live.state.read().next_after(after(1).as_ref()), Next::Done)); + assert!(matches!(live.state.read().next_at(next(2)).unwrap(), Next::Done)); } - /// A cleared window may restart its numbering lower, so a cursor past a higher number of the - /// old window still yields the new one's rows. #[test] - fn a_new_reference_restarts_the_cursor() { + fn cursors_refuse_lost_rows_and_resets() { let live = Producer::new(); - let window = Some(Duration::from_secs(30)); - live.push(row(5, 5, 10_000, 2_000), window); - let Next::Ready(old) = live.state.read().next_after(None) else { - panic!("expected a segment"); - }; - + for segment in 5..8 { + live.push(row(segment, segment, segment * 2_000, 2_000), None); + } + // One cursor consumed the first row, another has not read anything yet. + live.retire(5); + assert!(matches!( + live.state.read().next_at((0, 0)), + Err(crate::Error::RecordingLagged) + )); + assert!(matches!(live.state.read().next_at((0, 1)), Ok(Next::Ready(_)))); + live.retire(7); + assert!(matches!( + live.state.read().next_at((0, 1)), + Err(crate::Error::RecordingLagged) + )); + assert!(matches!(live.state.read().next_at((0, 3)), Ok(Next::Pending))); live.clear(); - let switched = Row { - reference: Arc::new((Kind::Video, "other".to_string(), "other".to_string())), - ..row(0, 0, 0, 2_000) - }; - live.push(switched, window); - let after = Some((0, old.segment)); - let Next::Ready(next) = live.state.read().next_after(after.as_ref()) else { - panic!("the new reference's first row follows"); - }; - assert_eq!(next.segment, 0); - assert_ne!(tag(&next.reference), tag(&old.reference), "its URLs carry another tag"); + live.push(row(0, 0, 0, 2_000), None); + assert!(matches!( + live.state.read().next_at((0, 3)), + Err(crate::Error::RecordingLagged) + )); + assert!(matches!(live.state.read().next_at((1, 3)), Ok(Next::Ready(_)))); } #[test] @@ -870,12 +876,12 @@ mod tests { }; live.push(switched, window); assert_eq!(live.window().sequence, 5); - let Next::Ready(next) = live.state.read().next_after(Some(&(0, 5))) else { + let Next::Ready(next) = live.state.read().next_at((0, 1)).unwrap() else { panic!("the replacement's row follows"); }; assert_eq!(next.segment, 6); assert_ne!(tag(&next.reference), tag(&old_reference)); - assert!(matches!(live.state.read().next_after(Some(&(0, 6))), Next::Pending)); + assert!(matches!(live.state.read().next_at((0, 2)).unwrap(), Next::Pending)); live.retire(5); assert_eq!(live.window().sequence, 6); assert!(live.row(&tag(&old_reference), 5).is_none()); diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index ff2b085a988..027a30231bd 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -830,7 +830,7 @@ mod tests { // Recorded as soon as the catalog lists it with its timeline, as a recorder does. let mut renditions = broadcaster.renditions(); let rendition = match tokio::time::timeout(TIMEOUT, renditions.next()).await { - Ok(Some(crate::export::renditions::Event::Added(rendition))) => rendition, + Ok(Ok(Some(crate::export::renditions::Event::Added(rendition)))) => rendition, _ => panic!("{version}: the catalog never listed the rendition"), }; let mut cursor = rendition.segments();