diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 58e3cbef7fe..7289f5d963c 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -94,7 +94,9 @@ The producer drains its counters every interval (one second by default) and writes only when a track's frame changed, so silence means nothing moved, not that the producer is gone. The track ends when the producer does, or at depth 1 or more when its group's broadcast is unannounced; the group may -return later as a new broadcast. +return later as a new broadcast. A frame too large to cache (32 MB) ends its +track with a `FrameTooLarge` error, while the `.z` flavor, which compresses +under the limit, keeps updating. ### Totals diff --git a/quest/m1/README.md b/quest/m1/README.md index 5df3713e3e0..b89f6222fab 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -122,8 +122,6 @@ lands before the cut. - [Binding track resolver](/quest/m1/binding-track-helper.md) - binding apps subscribe to a rendition without inlining `track` or the key - [Exporters follow a rendition to a new track](/quest/m1/export-rendition-rebind.md) - the fMP4, TS, MKV and FLV exporters rebuild a rendition whose resolved `track` or `broadcast` changes under the same ID -- [Stats fail loud](/quest/m1/stats-frame-too-large.md) - a stats track whose frame outgrows the cache ends with FrameTooLarge instead of serving a stale snapshot - - [C++ and OBS package proof](/quest/m1/cpp-package-check.md) - the installed artifacts pass before their first release ## Related diff --git a/quest/m1/stats-frame-too-large.md b/quest/m1/stats-frame-too-large.md deleted file mode 100644 index 72437489ef2..00000000000 --- a/quest/m1/stats-frame-too-large.md +++ /dev/null @@ -1,20 +0,0 @@ -# [XS] Stats tracks fail loud on an oversized frame - -## Goal - -A stats track whose next frame exceeds the cache limit ends with -`FrameTooLarge` instead of silently serving its previous snapshot, so a -reader learns the track stopped tracking the relay. - -## Plan - -- `TrackPair::publish` in `rs/moq-stats/src/produce.rs` gets - `FrameTooLarge` from `update` (payload over `group::MAX_CACHE_BYTES`) and - only logs it at debug level; readers keep the stale frame indefinitely. - `publisher.json` near 50k paths is the likely first hit. -- Abort only the flavor whose `update` failed (log at warn) and keep its - sibling publishing: DEFLATE can shrink an oversized plain frame or - slightly grow a fitting one. Test that the overflowing flavor ends with - `FrameTooLarge` while the other keeps updating. Prefix tracks get the same - rule from [Stats prefix tracks](/quest/m2/stats-prefix-tracks.md). -- Found in #5719. diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index ad348957a2c..e90a1e7307c 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -474,14 +474,26 @@ impl Snapshot { } /// Write one frame. `Some(len)` is the payload written; `None` means the - /// value was unchanged and the encoder emitted nothing. + /// value was unchanged, or the track already ended, and nothing was written. + /// + /// A frame over the cache limit aborts the track with + /// [`moq_net::Error::FrameTooLarge`]: skipping it would leave readers on the + /// previous snapshot with no sign it stopped tracking. fn update(&mut self, value: &Frame) -> moq_json::Result> { + if self.track.is_closed() { + return Ok(None); + } let Some(frame) = self.encoder.update(value)? else { return Ok(None); }; let len = frame.payload.len(); if len as u64 > moq_net::group::MAX_CACHE_BYTES { - return Err(moq_net::Error::FrameTooLarge.into()); + let err = moq_net::Error::FrameTooLarge; + if let Some(group) = self.group.take() { + let _ = group.abort(err.clone()); + } + let _ = self.track.clone().abort(err.clone()); + return Err(err.into()); } if frame.keyframe { if let Some(group) = self.group.take() { @@ -568,20 +580,22 @@ impl TrackPair { } /// Publish this drain's entries on both flavors (`{}` when there are none) - /// and clear them for the next drain; moq-json skips unchanged values. + /// and clear them for the next drain; moq-json skips unchanged values. A + /// flavor that fails ends alone: DEFLATE can shrink a frame too large for + /// the plain track, or grow one that just fits. fn publish(&mut self, name: &str) { self.frame.entries.sort_unstable_by(|a, b| a.0.cmp(&b.0)); let plain = match self.plain.update(&self.frame) { Ok(len) => len, Err(err) => { - tracing::debug!(?err, name, "stats: failed to write frame"); + tracing::warn!(?err, name, "stats: failed to write frame"); None } }; let compressed = match self.compressed.update(&self.frame) { Ok(len) => len, Err(err) => { - tracing::debug!(?err, name, "stats: failed to write compressed frame"); + tracing::warn!(?err, name, "stats: failed to write compressed frame"); None } }; @@ -808,10 +822,10 @@ impl TrackFamily { plain: String, pending: PendingPair, ) { - // Defensive only: a request racing the pair's creation is fulfilled by - // `create_track` (queued) or adopted by [`Self::flush`] (parked), so it - // never reaches this with the pair already live. Rejecting is still - // safe there - the requester's retry resolves against the live track. + // A request racing the pair's creation is fulfilled by `create_track` + // (queued) or adopted by [`Self::push`] (parked), so a live pair here + // means the requested flavor ended, such as on a frame too large to + // cache. It cannot update again, so refuse it. if self.tracks.contains_key(&plain) { pending.reject(moq_net::Error::NotFound); return; @@ -1197,8 +1211,8 @@ impl GroupPublisher { /// Hold a requested root's presence track, publishing its current counters /// so the subscription resolves at once. Cumulative from here on. fn adopt_presence(&mut self, report: &Report, root: PathOwned, pending: PendingPair) { - // Defensive only: a request for a live track is served by the broadcast - // itself. Rejecting is still safe; the retry finds the live track. + // A request for a live track is served by the broadcast itself, so a + // live pair here means the requested flavor ended and cannot update. if self.presence.contains_key(&root) { pending.reject(moq_net::Error::NotFound); return; @@ -1397,6 +1411,57 @@ mod tests { } } + /// A flavor whose frame outgrows the cache ends with `FrameTooLarge` instead + /// of leaving readers on its last snapshot, while its sibling, which DEFLATE + /// shrinks under the limit, keeps updating. + #[tokio::test] + async fn oversized_frame_aborts_only_its_flavor() { + use futures::FutureExt; + type Value = BTreeMap; + + let broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut pair = TrackPair::::create(&broadcast, "t.json", Arc::new(AtomicU64::new(0))).unwrap(); + let mut publish = |key: &str, value: u64| { + pair.frame.entries.push((PathOwned::from(key.to_string()), value)); + pair.publish("t.json"); + }; + publish("small", 1); + + let read = |name: &str, compression| { + let track = consumer + .track(name) + .unwrap() + .subscribe(None) + .now_or_never() + .unwrap() + .unwrap(); + let mut config = moq_json::snapshot::consumer::Config::default(); + config.compression = compression; + moq_json::snapshot::Consumer::::new(track, config) + }; + let mut plain = read("t.json", moq_json::Compression::None); + let mut compressed = read("t.json.z", moq_json::Compression::Deflate); + let next = |reader: &mut moq_json::snapshot::Consumer| reader.next().now_or_never().unwrap(); + let small = Value::from([("small".to_string(), 1)]); + assert_eq!(next(&mut plain).unwrap(), Some(small.clone())); + assert_eq!(next(&mut compressed).unwrap(), Some(small)); + + let huge = "a".repeat(moq_net::group::MAX_CACHE_BYTES as usize); + publish(&huge, 2); + assert!(matches!( + next(&mut plain), + Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) + )); + assert_eq!(next(&mut compressed).unwrap(), Some(Value::from([(huge, 2)]))); + + publish("small", 3); + assert_eq!( + next(&mut compressed).unwrap(), + Some(Value::from([("small".to_string(), 3)])) + ); + } + /// Build an origin producer, spawning its driver on the ambient runtime. fn produce_origin() -> moq_net::origin::Producer { let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default());