From dee30930fbe7faa7823da79bada3174364e024cf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 15 Aug 2026 16:12:01 -0700 Subject: [PATCH 01/32] feat(moq-net): enforce the subscriber latency budget `Subscription::latency` was metadata: moq-net put it on the wire and never acted on it, so only `moq_mux::container::Consumer` ever skipped a stale group. A relay hop, a `moq-json` track, a catalog, or a raw `track::Subscriber` got no skipping whatever budget it declared. That matters now because #2852 deliberately reverted #2785 on dev: a takeover boundary becomes subscription demand again, unbounded. `start`/`end` are filters rather than requests for reliability, so the ceiling on a takeover backlog has to come from the latency budget instead. See #2784. Enforcement lives in one place, `TrackState::is_stale`, reached by every read that hands content over: `poll_recv_group`, `poll_next_group`, and the frame-level `poll_read_frame`. Both ends of a subscription get there, since a relay serves downstream through `track::Subscriber` (#2771) and a local consumer reads the same way. Neither end alone suffices: `Latency::merge` takes the most tolerant budget, so what reaches a publisher is only ever as tight as its most patient viewer, and the subscriber applies it again where its own budget is exact. `fetch_group` stays exempt, naming one old group rather than following a live edge. An explicit `start` gets no exemption either, which is what lets a publisher apply the budget without having to tell a requested start from a resume-synthesized one. Drift is measured in presentation time: a group's first frame timestamp against the highest-sequence group above it that has one. Both are stamped once, when a group's first frame is created, so a backlog delivered as a burst still reads as its true age. The anchor must sit strictly above the candidate, or backfill and the tail of a rewound timeline convict the groups that follow them. It is resolved once per poll rather than per candidate: the highest-sequence stamped group in range is the anchor for every group below it, so walking a backlog of N groups off stays linear rather than quadratic. The anchor is capped by what the subscription could actually be served, since a reader that cannot jump past a boundary is not late against what lies beyond it. That means the read cursor's `end_at`, the requested `Subscription::end`, and -- for a spliced segment, which is deliberately left uncapped so its completion stays visible -- the cap its reader imposes from outside. A group parked above a cap is re-checked when the cap rises, since the live edge moved while it waited. `clamp_latency` becomes the single clamp point, applied to the aggregate the publisher sees and to one subscriber's own budget alike; the two agree because `min` distributes over the `max` that combines them. Skips are counted as `stats.stale`, disjoint from `groups`. They accumulate on the cursor and are drained by the `track::Subscriber` that owns the stats scope: a spliced subscriber reads through untagged per-segment cursors, so that wrapper is the only place attribution happens exactly once. Accepted limits, documented rather than fixed: - On pre-Lite05 drafts a receiver can only stamp frames on arrival, so the measure is burst-blind there: thirty seconds of backlog delivered in three reads as three. The publisher's copy is stamped as it produces, so the gate there still holds, but only at aggregate granularity. - A group that has presented nothing is never stale, and neither is one on a track where nothing newer has. With no timestamp there is nothing to measure and the frames may simply not have arrived; the publisher's retention window bounds such a group instead. This is narrower than the draft's Expiration section, which also wants a wall-clock backstop. `moq_mux::container::Consumer::with_latency` and `set_latency` only ever set a local field, so a tolerance never reached the subscription and the transport would now skip the very groups the consumer was told to wait for. They push it through, which also fixes moq-rtmp's 2s default being silently ignored. Tests that assert every group arrives now say so with a budget, which is stronger than the default REAL_TIME promises: the cluster failover test, the resume backfill test, and the moq-mux/moq-audio/moq-cli/moq-video batch pipelines that write a whole broadcast before reading it. Co-Authored-By: Claude Opus 5 --- doc/bin/relay/config.md | 18 +- doc/concept/layer/moq-lite.md | 5 + rs/moq-audio/tests/roundtrip.rs | 4 + rs/moq-cli/src/publish.rs | 11 +- rs/moq-mux/src/codec/h264/export.rs | 5 +- rs/moq-mux/src/codec/h265/export.rs | 5 +- rs/moq-mux/src/container/consumer.rs | 34 +- rs/moq-mux/src/container/flv/export_test.rs | 12 +- rs/moq-mux/src/container/fmp4/export_test.rs | 10 +- rs/moq-mux/src/container/fmp4/import_test.rs | 40 +- rs/moq-mux/src/container/mkv/export_test.rs | 13 +- rs/moq-mux/src/container/producer.rs | 32 +- rs/moq-mux/src/container/ts/export_test.rs | 22 +- rs/moq-mux/src/container/ts/import.rs | 19 +- rs/moq-mux/src/container/ts/import_test.rs | 19 +- rs/moq-mux/src/import/track.rs | 7 +- rs/moq-net/src/latency.rs | 42 +- rs/moq-net/src/model/group.rs | 32 ++ rs/moq-net/src/model/origin.rs | 55 ++ rs/moq-net/src/model/resume.rs | 185 ++++++- rs/moq-net/src/model/subscription.rs | 8 +- rs/moq-net/src/model/track.rs | 549 +++++++++++++++++-- rs/moq-net/src/stats.rs | 28 + rs/moq-relay/tests/goaway_cluster.rs | 13 +- rs/moq-video/src/decode/consumer.rs | 7 +- 25 files changed, 1062 insertions(+), 113 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 24055bf89a3..61473bddff2 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -400,14 +400,14 @@ counterpart no traffic can flow, so the entry is dropped: "broadcasts": 1, "broadcasts_closed": 0, "subscriptions": 5, "subscriptions_closed": 2, "fetches": 3, - "bytes": 12345, "frames": 678, "groups": 9, "datagrams": 2 + "bytes": 12345, "frames": 678, "groups": 9, "datagrams": 2, "stale": 4 }, "anon/foo": { "announced": 1, "announced_closed": 0, "announced_bytes": 8, "broadcasts": 1, "broadcasts_closed": 0, "subscriptions": 2, "subscriptions_closed": 0, "fetches": 0, - "bytes": 234, "frames": 12, "groups": 1, "datagrams": 0 + "bytes": 234, "frames": 12, "groups": 1, "datagrams": 0, "stale": 0 } } ``` @@ -417,6 +417,7 @@ Field semantics: - `announced` / `announced_closed`: cumulative count of every broadcast announce/unannounce event on this `(tier, role)` slot, regardless of whether any subscription happened. Use this for "all known broadcasts". + - `announced_bytes`: cumulative broadcast-name length summed over each model-visible announce and unannounce of this broadcast. It counts the name, not the encoded message size, so a broadcast isn't charged for hop chains or @@ -424,6 +425,7 @@ Field semantics: Separate from `bytes`, which is media payload. Announce control traffic that never enters the model (auth-rejected or unmatched-prefix announcements) is not counted. + - `broadcasts` / `broadcasts_closed`: per-(broadcast, session) subscription sentinel. The first active subscription a peer session opens for a broadcast bumps `broadcasts`; the last one it closes bumps @@ -431,13 +433,16 @@ Field semantics: broadcasts_closed` is the number of distinct sessions currently subscribed to the broadcast (i.e. viewers on the egress side), which is typically what billing and UI want. + - `subscriptions` / `subscriptions_closed`: cumulative count of track-level subscriptions opened and dropped. + - `fetches`: cumulative one-shot group fetches requested by a calling session, counted once per coalesced fetch when the request is issued, so a fetch that resolves to "not found" still counts. It is separate from `subscriptions` and the viewer sentinel; the fetched payload still flows into `bytes` / `frames` / `groups`. + - `bytes` / `frames` / `groups`: cumulative payload counters, bumped as groups/frames are read out of the model on the egress side and written into it on the ingress side. Egress bytes are counted when read out of the model @@ -445,12 +450,21 @@ Field semantics: still count. For a fan-out egress reader (e.g. an HLS/DASH muxer) this is bytes read once per segment at the broadcast origin, not per downstream HTTP client. + - `datagrams`: cumulative single-frame groups delivered over an unreliable QUIC datagram (moq-lite-05+ on a datagram-capable transport). A subset of `groups`: each datagram also counts there, and its payload in `frames` / `bytes`. Counted when the datagram enters or leaves the model, so an egress datagram dropped by congestion or an oversized body still counts. +- `stale`: cumulative groups skipped because they drifted further behind the live + edge than the subscriber's latency budget allows, so the relay never put them on + the wire. Disjoint from `groups`: a skipped group is not delivered, and none of + its payload reaches `frames` / `bytes` either. A steady rate here means + subscribers are consistently behind the live edge, which is normal for a + real-time subscription during congestion and a problem for one that asked to + tolerate more. + The session tracks (`sessions.json` and any `/sessions.json`) instead map each auth root to a `{ sessions, sessions_closed }` snapshot. `sessions` bumps when a session authenticated under that root connects and diff --git a/doc/concept/layer/moq-lite.md b/doc/concept/layer/moq-lite.md index 70ed9415871..fc52cfd76df 100644 --- a/doc/concept/layer/moq-lite.md +++ b/doc/concept/layer/moq-lite.md @@ -116,6 +116,11 @@ Each Subscription consists of a few properties: - **Group Order**: The order in which groups are delivered. Defaults to descending; higher IDs are delivered first. - **Subscriber Max Latency**: The maximum age of a non-latest group before it is skipped. Defaults to zero, so stale groups are skipped immediately. +That age is measured on the media timeline (a group's first timestamp against the newest one), not by when it arrived, so a backlog delivered as a burst still counts as old. +Both ends apply it: the publisher skips a group rather than sending it, and the subscriber skips it again as it reads. +The publisher only ever sees the most tolerant budget across its subscribers, which is why the subscriber applies it too. +Asking for old groups with `Group Start` does not exempt them: a start bounds what you are sent, and a subscriber that wants history has to raise its budget to match. + The publisher also keeps old groups around for a best-effort **Publisher Max Latency** cache window so relays and late subscribers can still fetch them. This defaults to 5 seconds. The subscriber's maximum latency is bounded by this window: a group can't be waited for longer than it's actually kept around. diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 525f4f16990..15ed7df05e1 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -65,6 +65,10 @@ async fn opus_round_trip_48k_stereo() { let cfg = snapshot.audio.renditions.get("audio").expect("audio rendition"); let mut config = decode::Config::default(); config.format = Format::F32; + // The whole clip is encoded before anything decodes it. The default REAL_TIME + // budget is enforced on the subscription, so without a tolerance the decoder would + // take the live edge and drop the rest of the batch. + config.latency = moq_mux::Latency::max(Duration::from_secs(3600)); let mut consumer = decode::Consumer::new(&broadcast_consumer, cfg, "audio", config) .await .unwrap(); diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index a7f12db9048..069626e8b1c 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -560,11 +560,18 @@ mod tests { Export::with_ts(moq_mux::Source::new(origin.consume(), "cli"), CatalogFormat::Hang) .await .unwrap() - .with_latency(moq_mux::Latency::REAL_TIME), + .with_latency(moq_mux::Latency::max(BATCH)), ) .await } + /// A drift budget no test timeline comes close to, so the exporter reads every group. + /// + /// These tests publish a whole feed before exporting it, which the default + /// [`Latency::REAL_TIME`](moq_mux::Latency::REAL_TIME) collapses to the live edge: + /// completeness has to be asked for, exactly as a real recorder does. + const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + /// Full CLI round-trip: a TS feed with undecoded streams goes through `Publish` /// (which selects the `mpegts` catalog) and the subscribe-side `Export::with_ts`, /// and the SCTE-35 section and the verbatim PES survive with their PIDs, framing, @@ -597,7 +604,7 @@ mod tests { Export::with_ts(moq_mux::Source::new(origin.consume(), "cli"), CatalogFormat::Hang) .await .unwrap() - .with_latency(moq_mux::Latency::REAL_TIME), + .with_latency(moq_mux::Latency::max(BATCH)), ) .await; diff --git a/rs/moq-mux/src/codec/h264/export.rs b/rs/moq-mux/src/codec/h264/export.rs index 880adc34dba..a3fc2a54acf 100644 --- a/rs/moq-mux/src/codec/h264/export.rs +++ b/rs/moq-mux/src/codec/h264/export.rs @@ -293,7 +293,10 @@ mod tests { // Consumer side: run the exporter. let consumer = broadcast.consume(); - let mut export = Export::new(crate::source::announced(&consumer), Once(Some(catalog))); + // The whole track is written before the exporter runs, so it needs a budget + // wide enough to read it: the default skips everything but the live edge. + let mut export = Export::new(crate::source::announced(&consumer), Once(Some(catalog))) + .with_latency(crate::Latency::max(std::time::Duration::from_secs(3600))); let frame0 = export.next().await.unwrap().expect("first frame"); let frame1 = export.next().await.unwrap().expect("second frame"); diff --git a/rs/moq-mux/src/codec/h265/export.rs b/rs/moq-mux/src/codec/h265/export.rs index d87bd98fcf8..89135ed7308 100644 --- a/rs/moq-mux/src/codec/h265/export.rs +++ b/rs/moq-mux/src/codec/h265/export.rs @@ -279,7 +279,10 @@ mod tests { track.finish().unwrap(); let consumer = broadcast.consume(); - let mut export = Export::new(crate::source::announced(&consumer), Once(Some(catalog))); + // The whole track is written before the exporter runs, so it needs a budget + // wide enough to read it: the default skips everything but the live edge. + let mut export = Export::new(crate::source::announced(&consumer), Once(Some(catalog))) + .with_latency(crate::Latency::max(std::time::Duration::from_secs(3600))); let frame0 = export.next().await.unwrap().expect("first frame"); let frame1 = export.next().await.unwrap().expect("second frame"); diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 6c4898c23ea..baa35f3f84c 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -148,7 +148,7 @@ impl Consumer { /// skipped. [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (the default) skips /// aggressively: any group with a newer alternative is dropped. pub fn with_latency(mut self, latency: crate::Latency) -> Self { - self.latency = latency; + self.set_latency(latency); self } @@ -480,6 +480,11 @@ impl Consumer { /// Set the latency tolerance mid-stream. pub fn set_latency(&mut self, latency: crate::Latency) { self.latency = latency; + // The transport enforces the same budget on the subscription itself, so a + // tolerance set here has to reach it: otherwise moq-net skips the very groups + // this consumer was told to wait for, before they ever get here. + let subscription = self.track.subscription().with_latency(latency); + let _ = self.track.update(subscription); } } @@ -761,6 +766,22 @@ mod tests { Ok(frames) } + /// Wrap `track` in a consumer whose skipping is entirely its own: the subscription + /// gets a budget no test timeline reaches, so the transport hands over every group + /// and what the consumer does with them is what the test measures. + /// + /// Both layers enforce the same budget, and normally should: the transport skipping + /// a group the consumer was going to skip anyway just saves the bandwidth. These + /// tests are the exception, since they are about the consumer's half of it. + fn consumer_only(track: moq_net::track::Subscriber, latency: Latency) -> Consumer { + let control = track.control(); + let consumer = Consumer::new(track, Container::Legacy).with_latency(latency); + control + .update(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(3600)))) + .unwrap(); + consumer + } + // ---- Basic Reading ---- #[tokio::test] @@ -868,7 +889,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::REAL_TIME); + let mut consumer = consumer_only(consumer_track, Latency::REAL_TIME); // Group 0 at ts 0 keeps timestamps monotonic with sequence (groups 1-9 follow at // g*50 ms), so the test exercises latency skipping and not rewind detection. @@ -907,8 +928,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = consumer_only(consumer_track, Latency::max(Duration::from_millis(100))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1851,8 +1871,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = consumer_only(consumer_track, Latency::max(Duration::from_millis(100))); // Group 0: stalled at ts=0, NOT finished let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1890,8 +1909,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = consumer_only(consumer_track, Latency::max(Duration::from_millis(100))); // Group 0: finished normally write_group(&mut track, 0, &[ts(0), ts(20_000)]); diff --git a/rs/moq-mux/src/container/flv/export_test.rs b/rs/moq-mux/src/container/flv/export_test.rs index 97a2647d3ff..fa809128206 100644 --- a/rs/moq-mux/src/container/flv/export_test.rs +++ b/rs/moq-mux/src/container/flv/export_test.rs @@ -10,6 +10,13 @@ use hang::catalog::{AudioCodec, VideoCodec}; use super::{Export, Import}; +/// A drift budget no test timeline comes close to, so the exporter reads every group. +/// +/// These tests write or import a whole broadcast and only then export it, which the +/// exporter's default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the +/// live edge: completeness has to be asked for, exactly as a real recorder does. +const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + /// A minimal `AVCDecoderConfigurationRecord` (profile 0x42, level 0x1f, one SPS + PPS). fn avcc() -> Vec { let sps = [0x67u8, 0x42, 0xc0, 0x1f]; @@ -169,7 +176,10 @@ async fn export_emits_sequence_headers_and_frames() { importer.decode(&bytes::BytesMut::from(synth_flv().as_slice())).unwrap(); catalog.finish().unwrap(); - let exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); + let exporter = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_latency(crate::Latency::max(BATCH)); let exported = drain_export(exporter, importer).await; let tags = parse_tags(&exported); diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index d35dbd0cf94..eab3bf37d6e 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -7,6 +7,13 @@ use mp4_atom::{DecodeMaybe, Encode}; use crate::container::test_util::{Live, PPS, SPS, raw_frame, video_frame}; +/// A drift budget no test timeline comes close to, so the exporter reads every group. +/// +/// These tests write or import a whole broadcast and only then export it, which the +/// exporter's default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the +/// live edge: completeness has to be asked for, exactly as a real recorder does. +const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + /// Avc3-shape source (catalog `Container::Legacy`, `H264 { inline: true }`, /// `description: None`) → fMP4 / CMAF export must synthesize a valid init /// segment from the codec config the Avc1 transform builds on the wire. @@ -530,7 +537,8 @@ async fn next_chunk_reports_segment_metadata() { // Sub-GOP cap so GOP 0 splits into two parts (the trailing part non-independent). let mut exporter = crate::container::fmp4::Export::new(live.source(), live.catalog_stream().await) - .with_fragment_duration(std::time::Duration::from_millis(20)); + .with_fragment_duration(std::time::Duration::from_millis(20)) + .with_latency(crate::Latency::max(BATCH)); // First emit is the init segment, which carries no segmenting metadata to assert on. chunk_now(&mut exporter) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 68f2bd6ad10..3332774d283 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2,6 +2,18 @@ use futures::FutureExt; use hang::catalog::Container; use mp4_atom::{Decode, Encode}; +/// A drift budget no test timeline comes close to, so every group is read. +/// +/// These tests write a whole batch of groups up front and only then read them, which the +/// default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) budget collapses to the live +/// edge: completeness has to be asked for. +const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + +/// A subscription with that budget, for a test asserting every group is delivered. +fn subscribe_all() -> moq_net::track::Subscription { + moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(BATCH)) +} + /// Drain every group currently buffered on the consumer without waiting for new ones. /// Used in tests where the producer is still alive after writing. #[cfg(test)] @@ -631,8 +643,18 @@ async fn segmented_source_groups_per_segment() { let snapshot = catalog.snapshot(); let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - let mut video_track = consumer.track(&video_name).unwrap().subscribe(None).await.unwrap(); - let mut audio_track = consumer.track(&audio_name).unwrap().subscribe(None).await.unwrap(); + let mut video_track = consumer + .track(&video_name) + .unwrap() + .subscribe(subscribe_all()) + .await + .unwrap(); + let mut audio_track = consumer + .track(&audio_name) + .unwrap() + .subscribe(subscribe_all()) + .await + .unwrap(); // Three 1s segments: 2 video fragments each (only the first an IDR) and 4 audio fragments, // so a per-fragment grouping and a per-segment one can't be confused. @@ -867,8 +889,18 @@ async fn a_single_leading_styp_still_segments_on_keyframes() { let snapshot = catalog.snapshot(); let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - let mut video_track = consumer.track(&video_name).unwrap().subscribe(None).await.unwrap(); - let mut audio_track = consumer.track(&audio_name).unwrap().subscribe(None).await.unwrap(); + let mut video_track = consumer + .track(&video_name) + .unwrap() + .subscribe(subscribe_all()) + .await + .unwrap(); + let mut audio_track = consumer + .track(&audio_name) + .unwrap() + .subscribe(subscribe_all()) + .await + .unwrap(); // One styp up front, then four segments' worth of fragments each opening on an IDR. fmp4.decode(&styp()).unwrap(); diff --git a/rs/moq-mux/src/container/mkv/export_test.rs b/rs/moq-mux/src/container/mkv/export_test.rs index a840b268cf6..f2fd70d2e2f 100644 --- a/rs/moq-mux/src/container/mkv/export_test.rs +++ b/rs/moq-mux/src/container/mkv/export_test.rs @@ -13,6 +13,13 @@ use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock}; use crate::container::test_util::{IDR, Live, PPS, SPS, raw_frame, video_frame}; +/// A drift budget no test timeline comes close to, so the exporter reads every group. +/// +/// These tests import a whole file and only then export it, which the exporter's default +/// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the live edge: +/// completeness has to be asked for, exactly as a real recorder does. +const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + #[tokio::test(start_paused = true)] async fn export_header_roundtrip_vp9_opus() { // Build a tiny synthetic WebM with one VP9 video track and one Opus audio track. @@ -367,7 +374,8 @@ async fn export_emits_blocks_for_each_frame() { let mut exporter = crate::container::mkv::Export::new(crate::source::announced(&consumer), catalog_stream) // Use per-frame clustering so each frame is observable as its own // Cluster chunk; batching is exercised in a dedicated test below. - .with_fragment_duration(std::time::Duration::ZERO); + .with_fragment_duration(std::time::Duration::ZERO) + .with_latency(crate::Latency::max(BATCH)); let mut exported: Vec = Vec::new(); let mut importer = Some(importer); @@ -605,7 +613,8 @@ async fn export_fragment_duration_batches_blocks() { .await .expect("catalog consumer"); let mut exporter = crate::container::mkv::Export::new(crate::source::announced(&consumer), catalog_stream) - .with_fragment_duration(std::time::Duration::from_secs(2)); + .with_fragment_duration(std::time::Duration::from_secs(2)) + .with_latency(crate::Latency::max(BATCH)); let mut exported: Vec = Vec::new(); let mut importer = Some(importer); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index fb2ca4569d3..e1e4a8b3f9e 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -398,6 +398,19 @@ mod tests { .unwrap() } + /// Subscribe with a budget wide enough to read a whole batch. + /// + /// These tests write every group up front and only then read, which the default + /// [`Latency::REAL_TIME`](moq_net::Latency::REAL_TIME) budget collapses to the live + /// edge: completeness has to be asked for. + fn subscribe_all(track: &moq_net::track::Producer) -> moq_net::track::Subscriber { + track.subscribe(moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(BATCH))) + } + + /// A drift budget no test timeline comes close to, so nothing is skipped. Clamped to + /// the track's own retention window, which is what actually bounds it. + const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + fn frame(timestamp_us: u64, keyframe: bool) -> Frame { Frame { timestamp: Timestamp::from_micros(timestamp_us).unwrap(), @@ -544,8 +557,11 @@ mod tests { /// consumer can see the break instead of inferring continuity from adjacent sequences. #[tokio::test] async fn discontinuity_publishes_an_empty_group() { - let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + // The resumed clock jumps forty minutes, so both the retention window and the + // drift budget have to cover it or the pre-discontinuity group reads as ancient. + let info = hang::container::track_info().with_latency_max(BATCH); + let track = track_producer("test", info); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); @@ -581,7 +597,7 @@ mod tests { #[tokio::test] async fn keyframe_closes_group_immediately() { let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); // first frame must be a keyframe @@ -600,7 +616,7 @@ mod tests { #[tokio::test] async fn needs_keyframe_drives_audio_grouping() { let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); // Drive grouping off `needs_keyframe`, as the audio importers do: the first frame of each @@ -624,7 +640,7 @@ mod tests { #[tokio::test] async fn cut_closes_immediately() { let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); @@ -642,7 +658,7 @@ mod tests { #[allow(deprecated)] async fn deprecated_finish_group_still_closes() { let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); @@ -677,7 +693,7 @@ mod tests { #[tokio::test] async fn seek_uses_explicit_sequence() { let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); // seq 0 @@ -692,7 +708,7 @@ mod tests { #[tokio::test] async fn seek_clears_pending_after_use() { let track = track_producer("test", hang::container::track_info()); - let consumer = track.subscribe(None); + let consumer = subscribe_all(&track); let mut producer = Producer::new(track, Container::Legacy); producer.seek(5).unwrap(); diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index b3c18908c3f..ad812bc415e 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -60,12 +60,20 @@ fn length_prefixed(nals: &[&[u8]]) -> Bytes { /// finished, retained tracks; that means it never reaches a hard end-of-stream, /// so we pull until a `next()` blocks (`Pending`, surfaced as a timeout under /// paused time) or the stream ends. +/// A drift budget no test timeline comes close to, so the exporter reads every group. +/// +/// These tests write a whole broadcast up front and only then export it, which the +/// exporter's default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the +/// live edge: completeness has to be asked for, exactly as a real recorder does. +const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { drain_with(Export::new(crate::source::announced(&consumer)).await.unwrap()).await } /// `drain` for an exporter built with an explicit catalog extension. -async fn drain_with(mut exporter: Export) -> BytesMut { +async fn drain_with(exporter: Export) -> BytesMut { + let mut exporter = exporter.with_latency(crate::Latency::max(BATCH)); let mut out = BytesMut::new(); // `while let Ok` stops on the first timeout (`Pending`: no more output). while let Ok(res) = tokio::time::timeout(std::time::Duration::from_secs(1), exporter.next()).await { @@ -639,7 +647,8 @@ async fn export_scte35_roundtrip() { let name = scte_track(&snapshot).expect("a scte35 track"); let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy); + let mut scte_reader = + crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); let frame = scte_reader .read() .await @@ -734,7 +743,8 @@ async fn export_pes_verbatim_roundtrip() { let name = name.clone(); let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy); + let mut reader = + crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); let frame = reader .read() .await @@ -802,7 +812,8 @@ async fn scte35_without_video_export_is_rejected() { /// Subscribe to a track and read every retained frame payload it holds. async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec> { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy); + let mut reader = + crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -1118,7 +1129,8 @@ fn scte_track(snap: &crate::catalog::hang::Catalog) -> Option Vec<(Vec, Timestamp)> { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy); + let mut reader = + crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); let mut cues = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 61e81675509..964c17788ba 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2305,6 +2305,13 @@ impl Read for Feed { #[cfg(test)] mod test { + + /// A drift budget no test timeline comes close to, so the reader sees every group. + /// + /// These tests import a whole file and only then read it back, which the default + /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) budget collapses to the live + /// edge: completeness has to be asked for. + const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); use mpeg2ts::es::StreamType; use super::SectionReassembler; @@ -2639,7 +2646,7 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); import.finish().unwrap(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::REAL_TIME); + let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(BATCH)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -2871,7 +2878,8 @@ mod test { .expect("an audio track") .clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) + .with_latency(Latency::max(BATCH)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { @@ -3681,7 +3689,8 @@ mod test { .subscribe(None) .await .unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) + .with_latency(Latency::max(BATCH)); let published = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await; assert!( matches!(published, Ok(Ok(Some(_)))), @@ -3936,7 +3945,7 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::REAL_TIME); + let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(BATCH)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -4081,7 +4090,7 @@ mod test { assert_eq!(track.pid, DATA_PID, "recorded the original PID"); let track = consumer.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::REAL_TIME); + let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(BATCH)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("verbatim read timed out") diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 8ec7be5dbaa..48f84580ee7 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -5,6 +5,13 @@ use bytes::BytesMut; +/// A drift budget no test timeline comes close to, so the reader sees every group. +/// +/// These tests import a whole file and only then read it back, which the default +/// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) budget collapses to the live edge: +/// completeness has to be asked for. +const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + /// Decode a whole TS buffer into a fresh broadcast and return the catalog. fn import_ts(data: &[u8]) -> crate::catalog::hang::Catalog { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -143,7 +150,8 @@ async fn import_opus_frames() { .clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) + .with_latency(crate::Latency::max(BATCH)); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -284,7 +292,8 @@ async fn import_export_import_roundtrip() { // subscribe to the finished, retained tracks. let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await - .unwrap(); + .unwrap() + .with_latency(crate::Latency::max(BATCH)); let mut out = BytesMut::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_secs(1), exporter.next()).await { match res.expect("exporter error") { @@ -337,7 +346,8 @@ async fn survives_midstream_join() { // The track resumes at the keyframe: the leading delta was dropped, the IDR // anchors the one and only group. let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) + .with_latency(crate::Latency::max(BATCH)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { frames.push(frame); @@ -381,7 +391,8 @@ async fn kyrion_dirtystart_extracts_real_cues() { .map(|(name, _)| name.clone()) .expect("scte35 track"); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) + .with_latency(crate::Latency::max(BATCH)); let mut cues = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { cues.push((frame.payload.to_vec(), frame.timestamp)); diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index d642af3c4a6..0932345fdb8 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -888,7 +888,12 @@ mod tests { catalog: &crate::catalog::Producer, ) -> (crate::codec::opus::Import, moq_net::track::Subscriber) { let track = broadcast.create_track("audio", hang::container::track_info()).unwrap(); - let subscriber = track.subscribe(None); + // Every group is written before anything reads, which the default + // REAL_TIME budget would collapse to the live edge. + let subscriber = track.subscribe( + moq_net::track::Subscription::default() + .with_latency(moq_net::Latency::max(std::time::Duration::from_secs(3600))), + ); let config = crate::codec::opus::Config::new(48_000, 2); let import = crate::codec::opus::Import::new(track, catalog.reserve(), config.into()).unwrap(); (import, subscriber) diff --git a/rs/moq-net/src/latency.rs b/rs/moq-net/src/latency.rs index 2155eff5524..f193ec865a4 100644 --- a/rs/moq-net/src/latency.rs +++ b/rs/moq-net/src/latency.rs @@ -4,12 +4,6 @@ use std::time::Duration; /// How far a subscriber may drift behind the live edge before a group is skipped. /// -/// One budget, enforced in two places: the publisher's cache drops a group that exceeds it -/// (this is `Subscriber Max Latency` on the wire, see -/// [`Subscription::latency`](crate::track::Subscription::latency)), and a receiver that -/// buffers applies the same bound locally while reordering. A group is skipped once either -/// measure exceeds it. -/// /// Only a ceiling today: a stalled group is skipped once newer data is [`max`](Self::max) /// ahead of it. [`Latency::REAL_TIME`] (the default) skips aggressively, so any group with /// a newer alternative is dropped. @@ -18,9 +12,43 @@ use std::time::Duration; /// ahead, so raising it buys a stalled group more time to arrive rather than padding a /// buffer. /// +/// # Where it is enforced +/// +/// At both ends of a subscription, and neither alone is enough. A publisher skips a group +/// that has drifted past the budget instead of putting it on the wire, which is what bounds +/// a backlog before it costs bandwidth. But what reaches a publisher is the aggregate +/// across every subscriber, which [`merge`](Self::merge) resolves in favor of the most +/// tolerant one, so that gate is only ever as tight as the most patient viewer. The +/// subscriber applies the same budget again as it reads, where its own budget is the only +/// one in play and the semantics are exact. +/// +/// This bounds a *subscription*. [`track::Consumer::fetch_group`](crate::track::Consumer::fetch_group) +/// is exempt: it names one old group explicitly, so there is no live edge to be late +/// against. +/// +/// # How drift is measured +/// +/// In presentation time: a group's timestamp against the newest one above it. Both are +/// stamped once, when a group's first frame is created, so a backlog delivered as a burst +/// still reads as its true age rather than as the few seconds it took to arrive. +/// +/// A group that has yet to present a frame is never skipped, and neither is one on a track +/// where nothing newer has: with no timestamp there is nothing to measure, and the frames +/// may simply not have arrived yet. Such a group is bounded by the publisher's retention +/// window instead, which aborts it once it ages out. +/// +/// Protocols whose wire can't carry a timestamp (pre-Lite05 moq-lite, moq-transport without +/// the Timestamp property) have their frames stamped on receipt instead, which makes the +/// measure burst-blind on the receiving side: thirty seconds of backlog delivered in three +/// reads as three. The publisher's copy is stamped as it produces, so the gate there still +/// holds; it is just the coarser of the two. +/// +/// # Not a retention window +/// /// Distinct from [`track::Info::latency_max`](crate::track::Info::latency_max), which is a /// *retention* bound (how long the publisher keeps a group, the inverse of an HTTP -/// `Cache-Control: max-age`) rather than a drift budget. +/// `Cache-Control: max-age`) rather than a drift budget. A budget is clamped to it, since +/// waiting for a group longer than it is kept around cannot produce it. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[non_exhaustive] pub struct Latency { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index ce028dd0244..c4084bf9484 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -124,6 +124,13 @@ pub(crate) struct GroupState { // against the byte budget tracks evict toward. charge: cache::Charge, + // The first frame's timestamp, recorded once and never revised: the group's + // presentation start. Kept here rather than read off `frames` so it survives a + // front eviction, and so an abort doesn't erase where the group sat in time. + // `None` until the first frame is written, which is the only honest answer: an + // empty group has not presented anything yet. + timestamp: Option, + // Once finalized, the total number of frames the group will ever contain. Recorded // at finish so the count outlives an abort that clears the cache. pub(crate) fin: Option, @@ -187,6 +194,11 @@ impl GroupState { } } + /// Record where the group starts in presentation time, on the first frame only. + fn stamp(&mut self, timestamp: Timestamp) { + self.timestamp.get_or_insert(timestamp); + } + /// Evict completed frames from the front until within the byte budget. fn evict(&mut self) { while self.cache > MAX_GROUP_CACHE { @@ -405,6 +417,7 @@ impl Producer { state.frames.push_back(Frame { timestamp, payload }); state.next_index = next_index; state.committed = state.next_index; + state.stamp(timestamp); state.evict(); drop(state); @@ -452,6 +465,9 @@ impl Producer { buf: buf.clone(), }); state.next_index = next_index; + // Opening the frame is enough: the header carries the timestamp, so the group's + // place in time is known before a single payload byte streams in. + state.stamp(timestamp); state.evict(); drop(state); @@ -501,6 +517,9 @@ impl Producer { buf: buf.clone(), }); state.next_index = next_index; + // Opening the frame is enough: the header carries the timestamp, so the group's + // place in time is known before a single payload byte streams in. + state.stamp(timestamp); state.evict(); drop(state); @@ -603,6 +622,19 @@ impl Producer { self.state.read().committed } + /// Where the group starts in presentation time: its first frame's timestamp, + /// or `None` while no frame has been opened. + /// + /// Stamped once, when the group's first frame arrives, so it measures the group's + /// place in the media timeline rather than when it happened to be delivered. That + /// is what lets the track tell a burst of old content apart from live content (see + /// [`track::Subscriber`]). On protocols whose wire can't carry a timestamp the + /// receiver stamps frames with [`Timestamp::now`], which makes this the local + /// receive time instead: an estimate that a burst compresses. + pub(crate) fn timestamp(&self) -> Option { + self.state.read().timestamp + } + /// The group's full cached footprint (payload plus fixed overhead), used by the /// track to size this group as an eviction victim. pub(crate) fn cache_size(&self) -> u64 { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ff51c45cd84..142e1593a6b 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3844,6 +3844,61 @@ mod tests { assert_eq!(entry.subscriber.fetches, 0, "ingress cannot fetch"); } + /// A group the drift budget skips is counted, not silently dropped: without a + /// counter a skip is indistinguishable from loss, and `groups` alone would just + /// quietly under-report. + #[tokio::test] + async fn test_stats_counts_stale_skips() { + use crate::Timestamp; + use crate::stats::{Config, Registry, Tier}; + use bytes::Bytes; + + tokio::time::pause(); + + let registry = Registry::new(Config::new()); + let ctx = registry.tier(Tier::default()).session("acme"); + + let origin = Origin::random().produce(); + let ingress = origin.clone().with_stats(ctx.clone()); + let egress = origin.consume().with_stats(ctx.clone()); + + let mut announced = egress.announced(); + let source = ingress.create_broadcast("demo", announce()).unwrap(); + let mut dynamic = source.dynamic(); + settle().await; + settle().await; + + let broadcast = announced.next().await.unwrap().broadcast.unwrap(); + let subscribing = broadcast.track("video").unwrap().subscribe(None); + let mut producer = accept_track(&mut dynamic, "video").await; + settle().await; + let mut sub = subscribing.await.unwrap(); + + // Three seconds of media, delivered as a burst before the subscriber reads. + for second in 0..3 { + let mut group = producer.append_group().unwrap(); + group + .write_frame( + Timestamp::from_millis(second * 1000).unwrap(), + Bytes::from_static(b"hi"), + ) + .unwrap(); + group.finish().unwrap(); + } + + // The default REAL_TIME budget takes the live edge and writes the other two off. + let group = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(group.sequence, 2); + settle().await; + + let report = registry.report(); + let entry = report.traffic.iter().find(|e| e.path.as_str() == "demo").unwrap(); + assert_eq!(entry.publisher.stale, 2, "two groups skipped on the way out"); + assert_eq!(entry.publisher.groups, 1, "only the live edge was delivered"); + assert_eq!(entry.subscriber.stale, 0, "ingress wrote all three"); + assert_eq!(entry.subscriber.groups, 3); + } + /// `Subscriber::read_frame` collapses a group to its first frame. The paths it /// delegates to (plain and spliced) build their own *unmetered* group consumers, /// so the wrapper is the only place that can attribute the read: exactly one diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 06ce9e6b184..1824fa0be2f 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1328,7 +1328,13 @@ impl Subscriber { /// Resolve a segment's pending subscription, if any. Ready once the segment is /// `Active` or `Done`; a rejected or closed track becomes `Done` (stall, not /// error). Never consumes groups, so terminal-state pollers can share it. - fn poll_activate(seg: &mut SegmentSub, prefs: &Subscription, min_sequence: u64, waiter: &kio::Waiter) -> Poll<()> { + fn poll_activate( + seg: &mut SegmentSub, + prefs: &Subscription, + min_sequence: u64, + stale_cap: Option, + waiter: &kio::Waiter, + ) -> Poll<()> { if let SubState::Pending(pending) = &mut seg.sub { match pending.poll_ok(waiter) { Poll::Ready(Ok(mut sub)) => { @@ -1338,6 +1344,7 @@ impl Subscriber { // this subscriber, never on the inner cursor: an inner cap would // park groups there and hide the segment's completion. sub.start_at(seg.first_group().max(min_sequence)); + sub.set_stale_cap(stale_cap); let _ = sub.update(slice(prefs, seg.start, seg.end)); seg.sub = SubState::Active(sub); } @@ -1355,12 +1362,13 @@ impl Subscriber { seg: &mut SegmentSub, prefs: &Subscription, min_sequence: u64, + stale_cap: Option, waiter: &kio::Waiter, ) -> Poll> { loop { match &mut seg.sub { SubState::Pending(_) => { - ready!(Self::poll_activate(seg, prefs, min_sequence, waiter)); + ready!(Self::poll_activate(seg, prefs, min_sequence, stale_cap, waiter)); } SubState::Active(sub) => match sub.poll_recv_group(waiter) { Poll::Ready(Ok(Some(group))) => { @@ -1450,6 +1458,14 @@ impl Subscriber { .parked .remove(&sequence) .expect("parked key just observed"); + // The cap rising widens the live edge too, so a group parked while it + // was current can be a backlog by the time it is owed again. Re-check it + // rather than hand back content the budget has since given up on. + if let SubState::Active(sub) = &mut self.segments[index].sub + && matches!(sub.poll_stale(&group, waiter), Poll::Ready(Ok(true))) + { + continue; + } // Folded into a group already handed out: try the next parked one. if let Some(group) = self.hand_out(index, group) { self.next_sequence = self.next_sequence.max(sequence.saturating_add(1)); @@ -1458,7 +1474,13 @@ impl Subscriber { } loop { - let polled = Self::poll_segment(&mut self.segments[index], &self.last_prefs, min_sequence, waiter); + let polled = Self::poll_segment( + &mut self.segments[index], + &self.last_prefs, + min_sequence, + end_sequence, + waiter, + ); match polled { Poll::Ready(Some(group)) => { if beyond_cap(group.sequence) { @@ -1583,7 +1605,7 @@ impl Subscriber { // be woken when it activates. let mut pending_activation = false; if let Some(seg) = self.segments.last_mut() { - if Self::poll_activate(seg, &self.last_prefs, self.min_sequence, waiter).is_pending() { + if Self::poll_activate(seg, &self.last_prefs, self.min_sequence, self.end_sequence, waiter).is_pending() { pending_activation = true; } else if let SubState::Active(sub) = &mut seg.sub { match sub.poll_recv_datagram(waiter) { @@ -1638,7 +1660,13 @@ impl Subscriber { let Some(seg) = self.segments.last_mut() else { return Poll::Ready(Ok(0)); }; - ready!(Self::poll_activate(seg, &self.last_prefs, self.min_sequence, waiter)); + ready!(Self::poll_activate( + seg, + &self.last_prefs, + self.min_sequence, + self.end_sequence, + waiter + )); match &mut seg.sub { SubState::Done(count) => Poll::Ready(Ok(count.unwrap_or(0))), SubState::Active(sub) => match ready!(sub.poll_finished(waiter)) { @@ -1679,6 +1707,13 @@ impl Subscriber { /// re-offers it. pub fn end_at(&mut self, sequence: impl Into>) { self.end_sequence = sequence.into(); + // The cap bounds each segment's drift anchor as well as this reader's own + // delivery: a segment must not measure against groups this cap hides. + for seg in &mut self.segments { + if let SubState::Active(sub) = &mut seg.sub { + sub.set_stale_cap(self.end_sequence); + } + } } /// The shared preferences channel, so `track::SubscriberControl` can wrap it. @@ -1686,6 +1721,21 @@ impl Subscriber { self.prefs.clone() } + /// Take the groups every segment's drift budget skipped since the last call. + /// + /// The segments are subscribed untagged, so their skips have nowhere to be counted + /// until they reach the [`track::Subscriber`] that owns the stats scope. A retired + /// segment's tail is lost with it, which is the same bound its groups already had. + pub(crate) fn take_stale(&mut self) -> u64 { + let mut stale = 0; + for seg in &mut self.segments { + if let SubState::Active(sub) = &mut seg.sub { + stale += sub.take_stale(); + } + } + stale + } + /// Replace this subscriber's preferences; each segment's demand is re-derived /// on the next poll. pub fn update(&mut self, subscription: Subscription) { @@ -1708,9 +1758,10 @@ impl Subscriber { #[cfg(test)] mod test { use super::*; - use crate::{Timestamp, broadcast}; + use crate::{Latency, Timestamp, broadcast}; use futures::FutureExt; use std::sync::Arc; + use std::time::Duration; fn track_pair(name: &str) -> (track::Producer, track::Consumer) { let producer = track::Producer::new(Arc::new(broadcast::Info::default()), name, None); @@ -1718,12 +1769,32 @@ mod test { (producer, consumer) } + /// [`track_pair`] with explicit publisher properties, for tests that care about the + /// retention window a subscriber's drift budget is clamped to. + fn track_pair_with(name: &str, info: track::Info) -> (track::Producer, track::Consumer) { + let producer = track::Producer::new(Arc::new(broadcast::Info::default()), name, info); + let consumer = producer.consume(); + (producer, consumer) + } + fn write_group(producer: &mut track::Producer, sequence: u64, payload: &str) { let mut group = producer.create_group(group::Info { sequence }).unwrap(); group.write_frame(Timestamp::ZERO, payload.as_bytes().to_vec()).unwrap(); group.finish().unwrap(); } + /// Like [`write_group`], but placing the group on a real media timeline so the + /// drift budget has something to measure. Most tests here are about splicing + /// mechanics and use [`write_group`], which leaves every group at the same instant + /// and therefore never stale. + fn write_group_at(producer: &mut track::Producer, sequence: u64, payload: &str, at: Duration) { + let mut group = producer.create_group(group::Info { sequence }).unwrap(); + group + .write_frame(at.try_into().unwrap(), payload.as_bytes().to_vec()) + .unwrap(); + group.finish().unwrap(); + } + fn recv(sub: &mut Subscriber) -> u64 { sub.recv_group() .now_or_never() @@ -1880,6 +1951,98 @@ mod test { assert_eq!(recv(&mut sub), 2); } + /// A route that stalls while a replacement runs on leaves the boundary trailing, + /// so the takeover asks the new route to replay a backlog. That backlog is bounded + /// by the drift budget and nothing else: a REAL_TIME subscriber jumps to the live + /// edge, while one that declared a budget covering the gap reads it whole. + #[tokio::test] + async fn a_takeover_backlog_is_bounded_by_the_budget() { + // Both routes retain a minute, so the budget is the only thing bounding the + // backlog (a budget past the publisher's window is clamped to it). + let retain = track::Info::default().with_latency_max(Duration::from_secs(60)); + let (mut track_a, consumer_a) = track_pair_with("a", retain.clone()); + let (mut track_b, consumer_b) = track_pair_with("b", retain); + + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut live = producer.consume().subscribe(None); + let mut patient = producer + .consume() + .subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(60)))); + + write_group_at(&mut track_a, 0, "a0", Duration::ZERO); + assert_eq!(recv(&mut live), 0); + assert_eq!(recv(&mut patient), 0); + + // The old route stalls; the replacement resumes at the boundary and replays + // half a minute of media as fast as the wire allows. + track_a.abort(Error::Dropped).unwrap(); + producer.takeover(&consumer_b).unwrap(); + recv_pending(&mut live); + for second in 1..=30 { + write_group_at(&mut track_b, second, "b", Duration::from_secs(second)); + } + + // The live subscriber takes the edge and writes the backlog off; the patient + // one asked to tolerate it and gets every group. + assert_eq!(recv(&mut live), 30); + recv_pending(&mut live); + + let mut backfill = Vec::new(); + while let Some(Ok(Some(group))) = patient.recv_group().now_or_never() { + backfill.push(group.sequence); + } + assert_eq!(backfill, (1..=30).collect::>()); + } + + /// A capped spliced subscriber measures drift against its cap, exactly like a plain + /// one. The segment is deliberately left uncapped so its completion stays visible, so + /// the cap has to reach its drift anchor by another route or the groups the reader + /// still wants read as ancient. + #[tokio::test] + async fn a_capped_spliced_subscriber_measures_drift_against_its_cap() { + let retain = track::Info::default().with_latency_max(Duration::from_secs(60)); + let (mut track_a, consumer_a) = track_pair_with("a", retain); + + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + sub.end_at(0); + + write_group_at(&mut track_a, 0, "a0", Duration::ZERO); + write_group_at(&mut track_a, 1, "a1", Duration::from_secs(30)); + + // Group 1 is past the cap, so it is not a live edge this reader can jump to. + assert_eq!(recv(&mut sub), 0); + } + + /// A group parked above the cap is re-checked when the cap rises: the live edge moved + /// while it waited, so handing it back unconditionally would deliver a backlog the + /// budget has already given up on. + #[tokio::test] + async fn a_raised_cap_rechecks_parked_groups() { + let retain = track::Info::default().with_latency_max(Duration::from_secs(60)); + let (mut track_a, consumer_a) = track_pair_with("a", retain); + + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + sub.end_at(0); + + write_group_at(&mut track_a, 0, "a0", Duration::ZERO); + assert_eq!(recv(&mut sub), 0); + + // Group 1 parks above the cap; group 2 lands half a minute further on. + write_group_at(&mut track_a, 1, "a1", Duration::from_secs(1)); + recv_pending(&mut sub); + write_group_at(&mut track_a, 2, "a2", Duration::from_secs(30)); + + // Raising the cap owes the reader group 1 again, but by now it is a backlog. + sub.end_at(None); + assert_eq!(recv(&mut sub), 2); + recv_pending(&mut sub); + } + #[tokio::test] async fn takeover_replaces_empty_segment() { let (track_a, consumer_a) = track_pair("a"); @@ -2673,9 +2836,13 @@ mod test { let mut producer = Producer::new(); producer.takeover(&consumer_a).unwrap(); - let mut sub = producer - .consume() - .subscribe(Subscription::default().with_start(Position::group(0))); + // The budget is what makes the backfill readable: an explicit start says which + // groups the publisher should send, not that the subscriber will wait for them. + let mut sub = producer.consume().subscribe( + Subscription::default() + .with_start(Position::group(0)) + .with_latency(Latency::max(Duration::from_secs(60))), + ); recv_pending(&mut sub); assert_eq!(track_a.subscription().unwrap().start, Some(Position::group(0))); diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index b24cc99ac95..2a1a5993a4d 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -20,9 +20,11 @@ pub struct Subscription { /// skipped); a larger ceiling tolerates that much reordering before giving up on the /// older group. /// - /// This is the `Subscriber Max Latency` on the wire, enforced by the publisher's - /// cache. Receivers that buffer (e.g. a jitter buffer) enforce the same budget - /// locally, and a group is skipped once either measure exceeds it. + /// This is the `Subscriber Max Latency` on the wire. It is enforced at both ends: + /// the publisher skips a group rather than sending it, and this subscriber skips it + /// again as it reads. Clamped to the publisher's + /// [`Info::latency_max`](crate::track::Info::latency_max) retention window; stored + /// here verbatim, so what was asked for stays readable. pub latency: crate::Latency, /// First [`Position`] the publisher should deliver, or `None` to start at the latest /// group. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 0b2b4f21ccb..a5a90d1a9c3 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -348,6 +348,7 @@ impl TrackState { &self, index: usize, next_sequence: u64, + drift: Drift, waiter: &kio::Waiter, ) -> Poll>> { let start = index.saturating_sub(self.offset); @@ -364,6 +365,12 @@ impl TrackState { // incarnation, delivered (if at all) at its own arrival position. continue; } + // The drift budget bounds this the same as the group-level reads: a frame + // from a group the subscription has given up on is no fresher for being + // read one frame at a time. + if self.is_stale(*sequence, drift.edge, drift.budget) { + continue; + } let mut consumer = slot.group.consume(); match consumer.poll_read_frame(waiter) { @@ -490,6 +497,67 @@ impl TrackState { self.info.as_ref().map(|info| info.latency_max) } + /// The live edge a subscription bounded at `cap` measures drift against: the + /// highest-sequence group at or below it that has presented a frame, and that + /// frame's timestamp. + /// + /// One scan answers a whole poll. The anchor for any candidate is "the newest + /// stamped group above it", and the highest-sequence stamped group in range is above + /// every candidate below it and above none at or past it, so it is that anchor for + /// all of them. Recomputing per candidate would make walking a backlog of N groups + /// off in one poll cost O(N^2). + /// + /// Capped because a group can only be late relative to data that would actually be + /// served in its place: a subscription ending at a takeover boundary can't jump past + /// it, so the groups it still wants aren't stale just because the route ran on. + fn live_edge(&self, cap: Option) -> Option<(u64, Timestamp)> { + let mut edge: Option<(u64, Timestamp)> = None; + for slot in self.lookup.values() { + let group = &slot.group; + if cap.is_some_and(|cap| group.sequence > cap) || group.is_aborted() { + continue; + } + if let Some(timestamp) = group.timestamp() + && edge.is_none_or(|(seq, _)| group.sequence > seq) + { + edge = Some((group.sequence, timestamp)); + } + } + edge + } + + /// Whether the group at `sequence` has drifted further behind `edge` than `budget` + /// tolerates, so a subscriber should skip it rather than hand it over. + /// + /// Measured in presentation time: both ends are stamped once, when a group's first + /// frame is created, so a backlog delivered as a burst still reads as its true age + /// rather than as the moment it happened to arrive. A group that has presented + /// nothing is never stale, and neither is one with nothing newer stamped above it: + /// with no timestamp there is nothing to measure, and the frames may simply not have + /// arrived yet. Such a group is bounded by the publisher's retention window instead, + /// which aborts it once it ages out (see [`Self::evict_expired`]). + /// + /// The edge must sit strictly above the candidate. The live edge is never late + /// against itself, and backfill or the tail of a rewound timeline can carry a high + /// timestamp on a low sequence without being an edge at all. + fn is_stale(&self, sequence: u64, edge: Option<(u64, Timestamp)>, budget: Duration) -> bool { + let Some((edge_sequence, stamped)) = edge else { + return false; + }; + if edge_sequence <= sequence { + return false; + } + let Some(slot) = self.lookup.get(&sequence) else { + return false; + }; + let Some(timestamp) = slot.group.timestamp() else { + return false; + }; + // Newer data below this group's timestamp isn't drift, and a scale that can't + // subtract carries no information either way. + matches!(stamped.checked_sub(timestamp), Ok(age) if Duration::from(age) > budget) + } + /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] /// once it can never be served. A missing group is a failure ([`Error::NotFound`]), not an /// end-of-stream. The handler side (a rejection, or no [`Dynamic`] at all) lives @@ -1298,6 +1366,8 @@ impl Producer { next_sequence: 0, end_sequence: None, parked: BTreeMap::new(), + stale_cap: None, + stale: 0, }), // A producer-side (in-process) subscribe is not egress: stay untagged. stats: stats::Scope::default(), @@ -1638,18 +1708,44 @@ fn snapshot_subscription(subs: &kio::Shared, bound: Option, requested: Option, outer: Option) -> Option { + // Ends are exclusive, so one at the head of a group excludes that whole group. + let requested = requested.map(|end| match end.frame { + 0 => end.group.saturating_sub(1), + _ => end.group, + }); + super::subscription::min_some(super::subscription::min_some(cursor, requested), outer) +} + +/// Clamp a drift budget to the publisher's retention window: nobody can wait for a late +/// group longer than the publisher keeps it around. /// /// The single clamp point. Subscribers hold their preferences verbatim, so what they asked -/// for stays readable, and clamping the aggregate is equivalent to clamping each subscriber -/// first (`min` distributes over the `max` that combines them). `bound` is `None` on a track -/// whose info isn't known yet (an unaccepted [`Request`]), which imposes no window. -fn clamp_combined(combined: Option, bound: Option) -> Option { - let mut combined = combined?; +/// for stays readable, and it is applied here on both sides of the aggregate: to the +/// combined request the publisher sees ([`clamp_combined`]) and to one subscriber's own +/// budget when it decides a group is stale ([`TrackState::is_stale`]). Those agree because +/// `min` distributes over the `max` that combines them. `bound` is `None` on a track whose +/// info isn't known yet (an unaccepted [`Request`]), which imposes no window. +fn clamp_latency(mut latency: crate::Latency, bound: Option) -> crate::Latency { if let Some(bound) = bound { - combined.latency.max = combined.latency.max.min(bound); + latency.max = latency.max.min(bound); } + latency +} + +/// Clamp the aggregate's latency budget to the publisher's window; see [`clamp_latency`]. +fn clamp_combined(combined: Option, bound: Option) -> Option { + let mut combined = combined?; + combined.latency = clamp_latency(combined.latency, bound); Some(combined) } @@ -2130,6 +2226,8 @@ impl Subscribing { next_sequence: 0, end_sequence: None, parked: BTreeMap::new(), + stale_cap: None, + stale: 0, }), stats: self.stats.clone(), _stats_sub: self.stats.subscribe(), @@ -2430,6 +2528,15 @@ enum SubscriberKind { Spliced(Box), } +/// One poll's view of how far this subscription may drift: the clamped budget and the +/// live edge to measure a candidate group against. Resolved once, then applied to every +/// group that poll considers. +#[derive(Clone, Copy)] +struct Drift { + budget: Duration, + edge: Option<(u64, Timestamp)>, +} + /// The cursor state for a subscription over a single (per-session) track. struct PlainSubscriber { state: kio::Consumer, @@ -2454,6 +2561,16 @@ struct PlainSubscriber { /// are parked here instead of dropped). Keyed by sequence so the lowest is /// re-offered first. parked: BTreeMap, + /// A cap imposed by a reader wrapping this cursor (a [`super::resume::Subscriber`] + /// segment), folded into the drift anchor only. Delivery is still bounded by + /// `end_sequence`, which stays unset on a segment so its completion is visible. + stale_cap: Option, + /// Groups the drift budget skipped since the count was last drained. Accumulated + /// here rather than metered in place because the handle that owns the stats scope + /// is the outer [`Subscriber`], which may be reading this cursor through a + /// [`super::resume::Subscriber`] segment (untagged, so the outer wrapper is the + /// only place attribution happens once). + stale: u64, } impl PlainSubscriber { @@ -2469,6 +2586,47 @@ impl PlainSubscriber { }) } + /// Take the groups skipped since the last call, for the owner to meter. + fn take_stale(&mut self) -> u64 { + std::mem::take(&mut self.stale) + } + + /// Note a group an outer reader skipped on this cursor's behalf, so it lands in the + /// same counter as the ones skipped here. + fn note_stale(&mut self) { + self.stale += 1; + } + + /// This subscriber's clamped drift budget and the live edge to measure against, + /// resolved once per poll. + /// + /// Read fresh each poll, so a mid-stream [`SubscriberControl::update`] applies to the + /// very next group, and shared across every candidate that poll walks off, so + /// discarding a backlog of N groups costs one scan rather than N. Only ever + /// [`Poll::Ready`]; the track ending surfaces as the error the caller was going to + /// get anyway. + fn poll_drift(&self, waiter: &kio::Waiter) -> Poll> { + let (latency, end) = { + let prefs = self.subscription.read(); + (prefs.latency, prefs.end) + }; + let cap = servable_cap(self.end_sequence, end, self.stale_cap); + self.poll(waiter, move |state| { + Poll::Ready(Ok(Drift { + budget: clamp_latency(latency, state.latency_bound()).max, + edge: state.live_edge(cap), + })) + }) + } + + /// Whether the drift budget says to skip `group`, against a [`Drift`] already resolved + /// for this poll. + fn poll_stale(&self, group: &group::Consumer, drift: Drift, waiter: &kio::Waiter) -> Poll> { + self.poll(waiter, move |state| { + Poll::Ready(Ok(state.is_stale(group.sequence, drift.edge, drift.budget))) + }) + } + fn poll_recv_group(&mut self, waiter: &kio::Waiter) -> Poll>> { // An eviction aborts a parked group without touching any cursor this // subscriber polls, so each entry needs a waiter or this poll would never @@ -2490,35 +2648,48 @@ impl PlainSubscriber { self.parked .retain(|sequence, group| *sequence >= min_sequence && watch(group)); - // Re-offer the lowest parked group back inside the cap once it rises. - if let Some(&sequence) = self.parked.keys().next() - && self.end_sequence.is_none_or(|end| sequence <= end) - { - return Poll::Ready(Ok(self.parked.remove(&sequence))); - } + // One scan for the whole poll, so walking a backlog off stays linear in its size. + let drift = ready!(self.poll_drift(waiter))?; loop { - let Some((consumer, found_index)) = - ready!(self.poll(waiter, |state| state.poll_recv_group(self.index, self.min_sequence))?) - else { - // Parked groups survive a finished track: they become deliverable - // again if the cap rises, so the stream isn't over while any are held. - if self.parked.is_empty() { - return Poll::Ready(Ok(None)); + // Re-offer the lowest parked group back inside the cap once it rises, + // ahead of the arrival cursor: it is the oldest thing still owed. + let consumer = match self.parked.keys().next().copied() { + Some(sequence) if self.end_sequence.is_none_or(|end| sequence <= end) => { + self.parked.remove(&sequence).expect("just looked it up") } - return Poll::Pending; - }; - self.index = found_index + 1; - - // Park a group beyond the cap instead of dropping it, and keep scanning - // so an in-range group that arrived behind it still flows. - if self.end_sequence.is_some_and(|end| consumer.sequence > end) { - // Watch it from the moment it parks: the retain pass above already - // ran, so an entry admitted here would otherwise sit unwatched for - // the rest of this poll, and an abort could wake nobody. - if watch(&consumer) { - self.parked.insert(consumer.sequence, consumer); + _ => { + let Some((consumer, found_index)) = + ready!(self.poll(waiter, |state| state.poll_recv_group(self.index, self.min_sequence))?) + else { + // Parked groups survive a finished track: they become deliverable + // again if the cap rises, so the stream isn't over while any are held. + if self.parked.is_empty() { + return Poll::Ready(Ok(None)); + } + return Poll::Pending; + }; + self.index = found_index + 1; + + // Park a group beyond the cap instead of dropping it, and keep scanning + // so an in-range group that arrived behind it still flows. + if self.end_sequence.is_some_and(|end| consumer.sequence > end) { + // Watch it from the moment it parks: the retain pass above already + // ran, so an entry admitted here would otherwise sit unwatched for + // the rest of this poll, and an abort could wake nobody. + if watch(&consumer) { + self.parked.insert(consumer.sequence, consumer); + } + continue; + } + consumer } + }; + + // Drop a group the drift budget has given up on and keep scanning, so one + // poll walks a whole backlog off rather than handing it out group by group. + if ready!(self.poll_stale(&consumer, drift, waiter))? { + self.stale += 1; continue; } return Poll::Ready(Ok(Some(consumer))); @@ -2538,19 +2709,30 @@ impl PlainSubscriber { } fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Poll>> { - let floor = self.next_sequence.max(self.min_sequence); - let Some(group) = ready!(self.poll(waiter, |state| state.poll_next_in_range(floor, self.end_sequence))?) else { - return Poll::Ready(Ok(None)); - }; - self.next_sequence = group.sequence.saturating_add(1); - Poll::Ready(Ok(Some(group))) + let drift = ready!(self.poll_drift(waiter))?; + + loop { + let floor = self.next_sequence.max(self.min_sequence); + let Some(group) = ready!(self.poll(waiter, |state| state.poll_next_in_range(floor, self.end_sequence))?) + else { + return Poll::Ready(Ok(None)); + }; + // Advance before the budget check: a skipped group is consumed, not retried. + self.next_sequence = group.sequence.saturating_add(1); + if ready!(self.poll_stale(&group, drift, waiter))? { + self.stale += 1; + continue; + } + return Poll::Ready(Ok(Some(group))); + } } fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { + let drift = ready!(self.poll_drift(waiter))?; let lower = self.min_sequence.max(self.next_sequence); - let Some((frame, found_index, sequence)) = - ready!(self.poll(waiter, |state| { state.poll_read_frame(self.index, lower, waiter) })?) - else { + let Some((frame, found_index, sequence)) = ready!(self.poll(waiter, |state| { + state.poll_read_frame(self.index, lower, drift, waiter) + })?) else { return Poll::Ready(Ok(None)); }; @@ -2608,6 +2790,58 @@ impl Subscriber { &self.broadcast } + /// Attribute the groups the drift budget skipped since the last read. + /// + /// Drained rather than metered where the skip happens, for the same reason a + /// delivered group is metered here: a spliced subscriber reads through untagged + /// per-segment cursors, so this wrapper is the one place that counts exactly once. + fn count_stale(&mut self, meter: &stats::Meter) { + // An untagged subscriber leaves the count where it is. A spliced reader polls + // its segments through this same method, and those segments are untagged, so + // draining here would throw the count away before the tagged handle wrapping + // them ever sees it. + if meter.is_tracked() { + meter.stale(self.take_stale()); + } + } + + /// Take the groups the drift budget skipped since the last call, so a nesting + /// handle (a spliced subscriber reading this one as a segment) can attribute them. + pub(crate) fn take_stale(&mut self) -> u64 { + match &mut self.inner { + SubscriberKind::Plain(plain) => plain.take_stale(), + SubscriberKind::Spliced(spliced) => spliced.take_stale(), + } + } + + /// Bound the drift anchor from outside, for a reader that caps this subscriber + /// without capping its cursor. + /// + /// A [`super::resume::Subscriber`] segment is deliberately left uncapped + /// ([`Self::end_at`] would park boundary-crossing groups where its completion can't + /// be seen), so its own cap has to reach the anchor this way or the segment measures + /// drift against groups its reader will never be served. + pub(crate) fn set_stale_cap(&mut self, cap: Option) { + if let SubscriberKind::Plain(plain) = &mut self.inner { + plain.stale_cap = cap; + } + } + + /// Whether the drift budget says to skip `group`, for a reader holding a group this + /// subscriber handed it earlier (a parked one, re-offered once a cap rose). Counts + /// the skip here so it reaches the stats with the rest. + pub(crate) fn poll_stale(&mut self, group: &group::Consumer, waiter: &kio::Waiter) -> Poll> { + let SubscriberKind::Plain(plain) = &mut self.inner else { + return Poll::Ready(Ok(false)); + }; + let drift = ready!(plain.poll_drift(waiter))?; + let stale = ready!(plain.poll_stale(group, drift, waiter))?; + if stale { + plain.note_stale(); + } + Poll::Ready(Ok(stale)) + } + /// Create a handle for updating this subscriber's delivery preferences. pub fn control(&self) -> SubscriberControl { SubscriberControl { @@ -2620,9 +2854,19 @@ impl Subscriber { /// Poll for the next group in arrival order, without blocking. /// - /// Returns every group exactly once in the order it landed on the wire, which may be - /// out of sequence due to network reordering or loss. Use [`Self::poll_next_group`] if - /// you only want groups whose sequence number is higher than any previously returned. + /// Returns each group it delivers exactly once, in the order it landed on the wire, + /// which may be out of sequence due to network reordering or loss. Use + /// [`Self::poll_next_group`] if you only want groups whose sequence number is higher + /// than any previously returned. + /// + /// Groups are semi-reliable, and the [`Subscription::latency`] budget is the other + /// thing (alongside eviction and a moving start) that decides which of them arrive: + /// one that has drifted further behind the live edge than the budget tolerates is + /// skipped rather than handed over, so a single poll walks off a whole backlog. The + /// default is [`Latency::REAL_TIME`](crate::Latency::REAL_TIME), which takes the live + /// edge and writes the rest off; raise it to read history. [`Self::start_at`] and + /// [`Subscription::start`] are filters, not exemptions: backfill needs a budget that + /// covers it. [`Consumer::fetch_group`] is the way to ask for one old group outright. /// /// Honors the floor set by [`Self::start_at`] and the cap set by [`Self::end_at`]: /// a group beyond the cap is parked (not dropped) and re-offered once the cap rises @@ -2640,6 +2884,7 @@ impl Subscriber { SubscriberKind::Plain(plain) => plain.poll_recv_group(waiter), SubscriberKind::Spliced(spliced) => spliced.poll_recv_group(waiter), }; + self.count_stale(&meter); res.map(|res| res.map(|group| group.map(|group| group.with_meter(meter)))) } @@ -2693,6 +2938,9 @@ impl Subscriber { /// produces a monotonically increasing sequence at the cost of dropping out-of-order /// groups. Use [`Self::poll_recv_group`] to see every group in arrival order instead. /// + /// The [`Subscription::latency`] budget applies here too, on the same terms; see + /// [`Self::poll_recv_group`]. + /// /// Honors the cap set by [`Self::end_at`]: groups with sequence past the cap are left /// in the producer's cache and become eligible again if the cap is raised or removed. pub fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Poll>> { @@ -2701,6 +2949,7 @@ impl Subscriber { SubscriberKind::Plain(plain) => plain.poll_next_group(waiter), SubscriberKind::Spliced(spliced) => spliced.poll_next_group(waiter), }; + self.count_stale(&meter); res.map(|res| res.map(|group| group.map(|group| group.with_meter(meter)))) } @@ -2722,6 +2971,7 @@ impl Subscriber { SubscriberKind::Plain(plain) => plain.poll_read_frame(waiter), SubscriberKind::Spliced(spliced) => spliced.poll_read_frame(waiter), }; + self.count_stale(&meter); // This helper collapses a group to its first frame: count the group, the one // frame, and the bytes actually read. if let Poll::Ready(Ok(Some(frame))) = &res { @@ -3593,6 +3843,215 @@ mod test { assert_eq!(producer.subscription().unwrap().latency.max, Duration::from_secs(2)); } + /// Append a finished group presenting at `millis`, so the track carries a media + /// timeline for the drift budget to measure against. + fn append_at(producer: &mut Producer, millis: u64) -> u64 { + let mut group = producer.append_group().unwrap(); + group + .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + group.finish().unwrap(); + group.sequence + } + + /// Every group the subscriber can read right now, in delivery order. + fn drain(subscriber: &mut Subscriber) -> Vec { + let mut sequences = Vec::new(); + while let Some(Ok(Some(group))) = subscriber.recv_group().now_or_never() { + sequences.push(group.sequence); + } + sequences + } + + #[test] + fn real_time_skips_a_backlog_to_the_live_edge() { + let mut producer = track_producer("test", None); + for second in 0..5 { + append_at(&mut producer, second * 1000); + } + + // The default budget is REAL_TIME: a subscriber joining a track that already + // holds five seconds of history takes the live edge, not the history. This is + // the ceiling a takeover backlog runs into. + let mut subscriber = producer.subscribe(None); + assert_eq!(drain(&mut subscriber), vec![4]); + + // And it stays caught up: the next group is live when it lands. + append_at(&mut producer, 5000); + assert_eq!(drain(&mut subscriber), vec![5]); + } + + #[test] + fn a_budget_admits_groups_within_it() { + let mut producer = track_producer("test", None); + for second in 0..5 { + append_at(&mut producer, second * 1000); + } + + // Two seconds of tolerance keeps the groups presenting within 2s of the live + // edge (2s, 3s, 4s) and drops the two below it. + let mut subscriber = + producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(2)))); + assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); + } + + #[test] + fn drift_is_measured_in_presentation_time_not_arrival_time() { + let mut producer = track_producer("test", None); + // A relay ingesting a backlog creates every group at once, so arrival time says + // they are all equally fresh. Their timestamps say otherwise, which is the whole + // point of measuring in presentation time. + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + let mut subscriber = + producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_millis(1500)))); + assert_eq!(drain(&mut subscriber), vec![2, 3]); + } + + #[test] + fn latency_max_bounds_the_budget() { + // The publisher only keeps a group around for 500ms, so a subscriber asking to + // wait ten seconds for one still gives up at 500ms: the same clamp the aggregate + // applies, on the subscriber's own side of it. + let mut producer = track_producer("test", Info::default().with_latency_max(Duration::from_millis(500))); + append_at(&mut producer, 0); + append_at(&mut producer, 1000); + + let mut subscriber = + producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + assert_eq!(drain(&mut subscriber), vec![1]); + } + + #[test] + fn an_explicit_start_gets_no_exemption_from_the_budget() { + let mut producer = track_producer("test", None); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + // Asking to start at the beginning is a filter, not a request for reliability. + // Backfill needs a budget that covers it; without one the live edge still wins. + let mut subscriber = producer.subscribe(Subscription::default().with_start(Position::group(0))); + subscriber.start_at(0); + assert_eq!(drain(&mut subscriber), vec![3]); + + let mut patient = producer.subscribe( + Subscription::default() + .with_start(Position::group(0)) + .with_latency(Latency::max(Duration::from_secs(10))), + ); + patient.start_at(0); + assert_eq!(drain(&mut patient), vec![0, 1, 2, 3]); + } + + #[test] + fn next_group_skips_stale_groups_too() { + let mut producer = track_producer("test", None); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + // Sequence-ordered reads run the same budget: a relay serves in arrival order, + // but a decoder reading by sequence must not be handed a backlog either. + let mut subscriber = producer.subscribe(None); + let group = subscriber.next_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(group.sequence, 3); + assert!(subscriber.next_group().now_or_never().is_none()); + } + + #[test] + fn fetch_ignores_the_budget() { + let mut producer = track_producer("test", None); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + // A fetch names one old group explicitly, so there is no live edge to drift + // from: the budget bounds a subscription, not a request for a specific group. + let consumer = producer.consume(); + let group = consumer.fetch_group(0, None).now_or_never().unwrap().unwrap(); + assert_eq!(group.sequence, 0); + } + + #[test] + fn a_group_that_has_presented_nothing_is_never_stale() { + let mut producer = track_producer("test", None); + let mut stalled = producer.append_group().unwrap(); + append_at(&mut producer, 10_000); + + // The stalled group has no timestamp, so there is nothing to measure and no + // reason to assume it is old. It is still handed over, and the reader (or the + // publisher's own retention window) decides how long to wait on it. + let mut subscriber = producer.subscribe(None); + assert_eq!(drain(&mut subscriber), vec![0, 1]); + + stalled.finish().unwrap(); + } + + #[tokio::test] + async fn read_frame_honors_the_budget() { + let mut producer = track_producer("test", None); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + // The frame-level helper is the group-level read with the group thrown away, so + // it gives up on a backlog on the same terms. Reading one frame at a time makes + // nothing fresher. + let mut subscriber = producer.subscribe(None); + let frame = subscriber.read_frame().await.unwrap().expect("a frame"); + assert_eq!(frame.timestamp, Timestamp::from_millis(3000).unwrap()); + assert!( + subscriber.read_frame().now_or_never().is_none(), + "the skipped groups are gone, not queued" + ); + } + + #[test] + fn a_lower_sequence_is_never_the_live_edge() { + let mut producer = track_producer("test", None); + // A high timestamp on a lower sequence is not a live edge: backfill served on + // demand sits there, and so does the tail of a timeline the publisher rewound. + // Only groups above the candidate anchor the measure, so neither can convict + // the group that follows it. + let mut straggler = producer.create_group(0u64.into()).unwrap(); + straggler + .write_frame(Timestamp::from_millis(60_000).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + straggler.finish().unwrap(); + + let mut rewound = producer.create_group(1u64.into()).unwrap(); + rewound + .write_frame(Timestamp::from_millis(0).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + rewound.finish().unwrap(); + + let mut subscriber = producer.subscribe(None); + assert_eq!(drain(&mut subscriber), vec![0, 1]); + } + + #[test] + fn a_capped_subscriber_measures_drift_against_its_cap() { + let mut producer = track_producer("test", None); + append_at(&mut producer, 0); + append_at(&mut producer, 1000); + + // Capped at group 0: the route running on past the cap is data this subscriber + // can never be served, so it isn't a live edge to be late against. This is what + // keeps a spliced segment from dropping the groups either side of a takeover + // boundary. + let mut subscriber = producer.subscribe(Subscription::default().with_end(Position::after_group(0))); + subscriber.end_at(0); + assert_eq!(drain(&mut subscriber), vec![0]); + + // Raising the cap re-offers the parked group, now measured against the wider + // edge it just admitted. + subscriber.end_at(None); + assert_eq!(drain(&mut subscriber), vec![1]); + } + #[test] fn subscriber_control_updates_while_read_future_is_pending() { let producer = track_producer("test", None); diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index 7130bf00f3c..e44ea426faa 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -151,6 +151,9 @@ pub(crate) struct Counters { groups: AtomicU64, // Subset of `groups` carried over an unreliable QUIC datagram. datagrams: AtomicU64, + // Groups the drift budget gave up on before they were delivered. Disjoint from + // `groups`, which counts only what was handed over. + stale: AtomicU64, } impl Counters { @@ -174,6 +177,7 @@ impl Counters { let frames = self.frames.load(Ordering::Relaxed); let groups = self.groups.load(Ordering::Relaxed); let datagrams = self.datagrams.load(Ordering::Relaxed); + let stale = self.stale.load(Ordering::Relaxed); Traffic { announced, announced_closed, @@ -187,6 +191,7 @@ impl Counters { frames, groups, datagrams, + stale, } } } @@ -257,6 +262,11 @@ pub struct Traffic { /// A subset of `groups`: each one also counts there and its payload in /// `frames` / `bytes`. pub datagrams: u64, + /// Cumulative groups skipped because they drifted past a subscriber's + /// [`Latency`](crate::Latency) budget. Disjoint from `groups`: a skipped group is + /// never handed over, so none of its payload reaches `frames` / `bytes` either. + /// A steady rate here means subscribers are consistently behind the live edge. + pub stale: u64, } impl Traffic { @@ -274,6 +284,7 @@ impl Traffic { self.frames += other.frames; self.groups += other.groups; self.datagrams += other.datagrams; + self.stale += other.stale; } /// True while the broadcast is announced (an announce guard is open). @@ -997,6 +1008,23 @@ impl Meter { } } + /// Whether this meter attributes anything, i.e. the broadcast is tracked and the + /// handle was tagged. A caller holding a count that has to land exactly once can + /// keep it rather than drop it into an untagged meter. + pub(crate) fn is_tracked(&self) -> bool { + self.counters.is_some() + } + + /// Bump `stale` by `n` (groups skipped before delivery by the drift budget). + pub(crate) fn stale(&self, n: u64) { + if n == 0 { + return; + } + if let Some(counters) = self.counters() { + counters.stale.fetch_add(n, Ordering::Relaxed); + } + } + /// Bump `bytes` by `n`. pub(crate) fn bytes(&self, n: u64) { if n == 0 { diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index aac238499e5..3196b6a197b 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -453,11 +453,18 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { // A live (unordered) subscription transmits the newest group first, so a // back-to-back burst legally arrives inverted; `ordered` is the protocol's // way to ask every hop for sequence-order transmission instead. + // + // The latency budget is the other half of that ask: this test asserts every group + // arrives exactly once, which is more than the default REAL_TIME budget promises. + // A subscriber that wants completeness across a failover has to say how far behind + // the live edge it is willing to sit (clamped to what the publisher retains). let mut sub = within( "subscribe to the video track", - bc.track("video") - .expect("track handle") - .subscribe(moq_net::track::Subscription::default().with_ordered(true)), + bc.track("video").expect("track handle").subscribe( + moq_net::track::Subscription::default() + .with_ordered(true) + .with_latency(moq_net::Latency::max(Duration::from_secs(60))), + ), ) .await .expect("subscribe"); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index bd382608d5c..c0140a3c6e3 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -131,7 +131,12 @@ mod tests { .await .unwrap(); let source = moq_mux::Source::new(origin.consume(), "test"); - let mut export = moq_mux::container::fmp4::Export::new(source, catalog); + // Both frames are encoded before the export runs, so the exporter needs a budget + // wide enough to read them: its REAL_TIME default keeps only the live edge, and + // the second `next()` would then block forever waiting for a group that was + // skipped. + let mut export = moq_mux::container::fmp4::Export::new(source, catalog) + .with_latency(moq_mux::Latency::max(std::time::Duration::from_secs(3600))); let init = export.next().await.unwrap().expect("CMAF init"); let fragment = export.next().await.unwrap().expect("CMAF fragment"); From ca7cf0cb25d0774686a71d2571b625687a8c9efe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 12:27:21 -0700 Subject: [PATCH 02/32] fix(moq-net): revalidate the drift anchor and count frame-level skips Follow-ups from a second adversarial review pass over the enforcement itself. Resolving the live edge once per poll made walking a backlog off linear, but it also split the anchor's lock from the candidate's: an eviction or a re-served sequence in between would convict a group on content that is no longer there to jump to, permanently discarding it. The anchor now carries the slot incarnation it was read from, and a candidate is only convicted while that incarnation is still cached and unaborted. Failing safe is right here, since the next poll resolves a fresh anchor anyway. `poll_read_frame` skipped stale groups without counting them, so `stats.stale` drained zero after a frame-level reader discarded a backlog and the metric hid exactly the drops it exists to report. The scan now reports what it stepped over, counted only on the scan that returns a frame: its cursor jumps past those groups, so one that ends `Pending` leaves them to be counted exactly once by the one that eventually succeeds. That return outgrew a tuple, so it is a `FrameHit` now. Co-Authored-By: Claude Opus 5 --- rs/moq-net/src/model/track.rs | 121 +++++++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 15 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index a5a90d1a9c3..43f9b39025e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -225,6 +225,17 @@ pub(crate) struct TrackState { fetch: kio::Shared, } +/// What a frame-level scan found: the frame, where it came from so the cursor can +/// advance past it, and how many groups the drift budget made the scan step over on the +/// way there (counted here rather than skipped silently, since a silent drop is +/// indistinguishable from loss). +struct FrameHit { + frame: frame::Frame, + index: usize, + sequence: u64, + stale: u64, +} + /// A cached group plus its bookkeeping in the track's `lookup` map. /// /// Access times and the evictable-population sample live in the group's own @@ -350,9 +361,13 @@ impl TrackState { next_sequence: u64, drift: Drift, waiter: &kio::Waiter, - ) -> Poll>> { + ) -> Poll>> { let start = index.saturating_sub(self.offset); let mut pending_seen = false; + // Groups the budget gave up on below the frame this returns. Counted only on the + // winning scan, whose cursor jumps past them, so a scan that ends Pending leaves + // them to be counted exactly once by the one that eventually succeeds. + let mut stale = 0; for (i, (sequence, stamp)) in self.arrival.iter().enumerate().skip(start) { if *sequence < next_sequence { continue; @@ -369,13 +384,19 @@ impl TrackState { // from a group the subscription has given up on is no fresher for being // read one frame at a time. if self.is_stale(*sequence, drift.edge, drift.budget) { + stale += 1; continue; } let mut consumer = slot.group.consume(); match consumer.poll_read_frame(waiter) { Poll::Ready(Ok(Some(frame))) => { - return Poll::Ready(Ok(Some((frame, self.offset + i, *sequence)))); + return Poll::Ready(Ok(Some(FrameHit { + frame, + index: self.offset + i, + sequence: *sequence, + stale, + }))); } Poll::Ready(Ok(None)) => continue, // A single group failing (aborted upstream, or evicted from the @@ -510,17 +531,21 @@ impl TrackState { /// Capped because a group can only be late relative to data that would actually be /// served in its place: a subscription ending at a takeover boundary can't jump past /// it, so the groups it still wants aren't stale just because the route ran on. - fn live_edge(&self, cap: Option) -> Option<(u64, Timestamp)> { - let mut edge: Option<(u64, Timestamp)> = None; + fn live_edge(&self, cap: Option) -> Option { + let mut edge: Option = None; for slot in self.lookup.values() { let group = &slot.group; if cap.is_some_and(|cap| group.sequence > cap) || group.is_aborted() { continue; } if let Some(timestamp) = group.timestamp() - && edge.is_none_or(|(seq, _)| group.sequence > seq) + && edge.is_none_or(|edge| group.sequence > edge.sequence) { - edge = Some((group.sequence, timestamp)); + edge = Some(Edge { + sequence: group.sequence, + stamp: slot.stamp, + timestamp, + }); } } edge @@ -540,11 +565,22 @@ impl TrackState { /// The edge must sit strictly above the candidate. The live edge is never late /// against itself, and backfill or the tail of a rewound timeline can carry a high /// timestamp on a low sequence without being an edge at all. - fn is_stale(&self, sequence: u64, edge: Option<(u64, Timestamp)>, budget: Duration) -> bool { - let Some((edge_sequence, stamped)) = edge else { + fn is_stale(&self, sequence: u64, edge: Option, budget: Duration) -> bool { + let Some(edge) = edge else { return false; }; - if edge_sequence <= sequence { + if edge.sequence <= sequence { + return false; + } + // The anchor was resolved under an earlier lock, so confirm it is still the + // group it was: an eviction or a re-served sequence in between would otherwise + // convict a candidate on content that is no longer servable. Failing safe + // (delivering) is right, since the next poll resolves a fresh anchor. + let live = self + .lookup + .get(&edge.sequence) + .is_some_and(|slot| slot.stamp == edge.stamp && !slot.group.is_aborted()); + if !live { return false; } let Some(slot) = self.lookup.get(&sequence) else { @@ -555,7 +591,7 @@ impl TrackState { }; // Newer data below this group's timestamp isn't drift, and a scale that can't // subtract carries no information either way. - matches!(stamped.checked_sub(timestamp), Ok(age) if Duration::from(age) > budget) + matches!(edge.timestamp.checked_sub(timestamp), Ok(age) if Duration::from(age) > budget) } /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] @@ -2534,7 +2570,18 @@ enum SubscriberKind { #[derive(Clone, Copy)] struct Drift { budget: Duration, - edge: Option<(u64, Timestamp)>, + edge: Option, +} + +/// The group a poll's drift is measured against, identified well enough to tell it apart +/// from whatever may occupy its sequence by the time a candidate is judged. +#[derive(Clone, Copy)] +struct Edge { + sequence: u64, + /// The slot incarnation this timestamp was read from, so an eviction or a re-served + /// sequence between resolving the anchor and using it is detectable. + stamp: u32, + timestamp: Timestamp, } /// The cursor state for a subscription over a single (per-session) track. @@ -2730,15 +2777,16 @@ impl PlainSubscriber { fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { let drift = ready!(self.poll_drift(waiter))?; let lower = self.min_sequence.max(self.next_sequence); - let Some((frame, found_index, sequence)) = ready!(self.poll(waiter, |state| { + let Some(hit) = ready!(self.poll(waiter, |state| { state.poll_read_frame(self.index, lower, drift, waiter) })?) else { return Poll::Ready(Ok(None)); }; - self.index = found_index + 1; - self.next_sequence = sequence.saturating_add(1); - Poll::Ready(Ok(Some(frame))) + self.stale += hit.stale; + self.index = hit.index + 1; + self.next_sequence = hit.sequence.saturating_add(1); + Poll::Ready(Ok(Some(hit.frame))) } } @@ -4009,6 +4057,49 @@ mod test { ); } + /// The live edge is resolved once per poll and applied to every group that poll + /// walks off, so it has to be revalidated before it convicts anything: an eviction + /// in between would otherwise discard a group on the strength of content that is no + /// longer there to jump to. + #[test] + fn an_evicted_live_edge_convicts_nothing() { + let mut producer = track_producer("test", None); + append_at(&mut producer, 0); + let edge = append_at(&mut producer, 30_000); + + let state = producer.state.read(); + let drift = Drift { + budget: Duration::ZERO, + edge: state.live_edge(None), + }; + assert!(state.is_stale(0, drift.edge, drift.budget), "stale against a live edge"); + drop(state); + + // The edge dies between resolving it and judging the candidate. + let slot = producer.modify().unwrap().lookup.remove(&edge).unwrap(); + let _ = slot.group.abort(Error::Evicted); + + let state = producer.state.read(); + assert!( + !state.is_stale(0, drift.edge, drift.budget), + "a vanished edge is no reason to drop what is left" + ); + } + + #[tokio::test] + async fn read_frame_counts_what_it_skips() { + let mut producer = track_producer("test", None); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + // A silent skip is indistinguishable from loss whichever read did it, so the + // frame-level path reports its drops like the group-level ones. + let mut subscriber = producer.subscribe(None); + subscriber.read_frame().await.unwrap().expect("a frame"); + assert_eq!(subscriber.take_stale(), 3); + } + #[test] fn a_lower_sequence_is_never_the_live_edge() { let mut producer = track_producer("test", None); From 5d800e1c233202094b0c3173237e658d0731c87f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 12:38:50 -0700 Subject: [PATCH 03/32] fix(moq-net): stop the requested end from capping the local live edge `Subscription::end` is a request to the publisher, folded in with every other subscriber's, and it does not filter the handle that asked for it: another unbounded subscriber widens the aggregate and the groups arrive here anyway. Using it to cap the drift anchor therefore pinned the live edge at the requested end while delivery ran past it, and every group above that end had nothing newer to be late against. A subscriber asking for a short range could be handed an arbitrarily old backlog under the default real-time budget. Only the read cursor (`end_at`) and the cap a wrapping reader imposes bound what this handle can actually be served, so only those bound the anchor. That leaves a spliced segment's boundary, which reached the anchor through the sliced `Subscription::end` before: it rides down with the reader's own cap now, which is where it belonged, and a boundary that moves updates it. Co-Authored-By: Claude Opus 5 --- rs/moq-net/src/model/resume.rs | 27 ++++++++++++++---- rs/moq-net/src/model/track.rs | 50 +++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 1824fa0be2f..56b13c92b7a 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1273,7 +1273,10 @@ impl Subscriber { Some(existing) => { if existing.end != segment.end { existing.end = segment.end; + let cap = Self::stale_cap(existing, self.end_sequence); if let SubState::Active(sub) = &mut existing.sub { + // The boundary bounds the drift anchor as well as the demand. + sub.set_stale_cap(cap); // Shrink the demand so the session can cap upstream. The // read bounds stay on this subscriber (see `poll_recv_group`): // an inner `end_at` would park boundary-crossing groups in the @@ -1325,6 +1328,18 @@ impl Subscriber { Some(group.into_spliced(spliced)) } + /// The highest sequence a segment could hand its reader: the reader's own cap and + /// the segment's boundary, whichever is lower. + /// + /// A segment's inner cursor is deliberately left uncapped (an inner `end_at` would + /// park boundary-crossing groups where its completion can't be seen), so this is how + /// both bounds reach the drift anchor. Without them a segment measures staleness + /// against groups it will never surface: the route running past the boundary, or the + /// reader's own cap holding content back. + fn stale_cap(seg: &SegmentSub, end_sequence: Option) -> Option { + min_some(end_sequence, seg.last_group()) + } + /// Resolve a segment's pending subscription, if any. Ready once the segment is /// `Active` or `Done`; a rejected or closed track becomes `Done` (stall, not /// error). Never consumes groups, so terminal-state pollers can share it. @@ -1332,7 +1347,7 @@ impl Subscriber { seg: &mut SegmentSub, prefs: &Subscription, min_sequence: u64, - stale_cap: Option, + end_sequence: Option, waiter: &kio::Waiter, ) -> Poll<()> { if let SubState::Pending(pending) = &mut seg.sub { @@ -1344,7 +1359,7 @@ impl Subscriber { // this subscriber, never on the inner cursor: an inner cap would // park groups there and hide the segment's completion. sub.start_at(seg.first_group().max(min_sequence)); - sub.set_stale_cap(stale_cap); + sub.set_stale_cap(Self::stale_cap(seg, end_sequence)); let _ = sub.update(slice(prefs, seg.start, seg.end)); seg.sub = SubState::Active(sub); } @@ -1362,13 +1377,13 @@ impl Subscriber { seg: &mut SegmentSub, prefs: &Subscription, min_sequence: u64, - stale_cap: Option, + end_sequence: Option, waiter: &kio::Waiter, ) -> Poll> { loop { match &mut seg.sub { SubState::Pending(_) => { - ready!(Self::poll_activate(seg, prefs, min_sequence, stale_cap, waiter)); + ready!(Self::poll_activate(seg, prefs, min_sequence, end_sequence, waiter)); } SubState::Active(sub) => match sub.poll_recv_group(waiter) { Poll::Ready(Ok(Some(group))) => { @@ -1709,9 +1724,11 @@ impl Subscriber { self.end_sequence = sequence.into(); // The cap bounds each segment's drift anchor as well as this reader's own // delivery: a segment must not measure against groups this cap hides. + let end_sequence = self.end_sequence; for seg in &mut self.segments { + let cap = Self::stale_cap(seg, end_sequence); if let SubState::Active(sub) = &mut seg.sub { - sub.set_stale_cap(self.end_sequence); + sub.set_stale_cap(cap); } } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 43f9b39025e..00424a79a48 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1744,22 +1744,22 @@ fn snapshot_subscription(subs: &kio::Shared, bound: Option, requested: Option, outer: Option) -> Option { - // Ends are exclusive, so one at the head of a group excludes that whole group. - let requested = requested.map(|end| match end.frame { - 0 => end.group.saturating_sub(1), - _ => end.group, - }); - super::subscription::min_some(super::subscription::min_some(cursor, requested), outer) +/// of sight), so its reader passes down its own cap and the segment boundary that way. +/// Without it, a segment would anchor drift on groups it can never hand over. +fn servable_cap(cursor: Option, outer: Option) -> Option { + super::subscription::min_some(cursor, outer) } /// Clamp a drift budget to the publisher's retention window: nobody can wait for a late @@ -2653,11 +2653,8 @@ impl PlainSubscriber { /// [`Poll::Ready`]; the track ending surfaces as the error the caller was going to /// get anyway. fn poll_drift(&self, waiter: &kio::Waiter) -> Poll> { - let (latency, end) = { - let prefs = self.subscription.read(); - (prefs.latency, prefs.end) - }; - let cap = servable_cap(self.end_sequence, end, self.stale_cap); + let latency = self.subscription.read().latency; + let cap = servable_cap(self.end_sequence, self.stale_cap); self.poll(waiter, move |state| { Poll::Ready(Ok(Drift { budget: clamp_latency(latency, state.latency_bound()).max, @@ -4123,6 +4120,21 @@ mod test { assert_eq!(drain(&mut subscriber), vec![0, 1]); } + #[test] + fn a_requested_end_does_not_cap_the_live_edge() { + let mut producer = track_producer("test", None); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + + // `Subscription::end` is a request to the publisher, folded in with every other + // subscriber's, and it does not filter this handle: the groups above it arrive + // anyway. Capping the live edge with it would pin the edge below them, leaving + // everything past it with nothing newer to be late against. + let mut subscriber = producer.subscribe(Subscription::default().with_end(Position::after_group(1))); + assert_eq!(drain(&mut subscriber), vec![3]); + } + #[test] fn a_capped_subscriber_measures_drift_against_its_cap() { let mut producer = track_producer("test", None); From 8783a69f7fd35f37f184170683c1e5b04a0e56c1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 16:50:57 -0700 Subject: [PATCH 04/32] fix(moq-net): address latency enforcement review Co-Authored-By: Codex --- doc/bin/relay/config.md | 20 +++-- rs/moq-audio/tests/roundtrip.rs | 2 +- rs/moq-cli/src/publish.rs | 12 +-- rs/moq-mux/src/codec/h264/export.rs | 2 +- rs/moq-mux/src/codec/h265/export.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 18 ++-- rs/moq-mux/src/container/flv/export_test.rs | 8 +- rs/moq-mux/src/container/fmp4/export_test.rs | 10 +-- rs/moq-mux/src/container/fmp4/import_test.rs | 7 +- rs/moq-mux/src/container/mkv/export_test.rs | 10 ++- rs/moq-mux/src/container/producer.rs | 15 ++-- rs/moq-mux/src/container/source.rs | 23 +++++- rs/moq-mux/src/container/ts/export_test.rs | 24 +++--- rs/moq-mux/src/container/ts/import.rs | 15 ++-- rs/moq-mux/src/container/ts/import_test.rs | 13 +-- rs/moq-mux/src/import/track.rs | 2 +- rs/moq-net/src/model/group.rs | 28 +++++++ rs/moq-net/src/model/origin.rs | 7 +- rs/moq-net/src/model/resume.rs | 76 ++++++++++++++--- rs/moq-net/src/model/track.rs | 87 +++++++++++++------- rs/moq-net/src/stats.rs | 85 +++++++++++++++---- rs/moq-video/src/decode/consumer.rs | 2 +- 22 files changed, 334 insertions(+), 134 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 61473bddff2..06c09c57513 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -400,14 +400,16 @@ counterpart no traffic can flow, so the entry is dropped: "broadcasts": 1, "broadcasts_closed": 0, "subscriptions": 5, "subscriptions_closed": 2, "fetches": 3, - "bytes": 12345, "frames": 678, "groups": 9, "datagrams": 2, "stale": 4 + "bytes": 12345, "frames": 678, "groups": 9, "datagrams": 2, + "stale": { "bytes": 456, "frames": 23, "groups": 4, "datagrams": 0 } }, "anon/foo": { "announced": 1, "announced_closed": 0, "announced_bytes": 8, "broadcasts": 1, "broadcasts_closed": 0, "subscriptions": 2, "subscriptions_closed": 0, "fetches": 0, - "bytes": 234, "frames": 12, "groups": 1, "datagrams": 0, "stale": 0 + "bytes": 234, "frames": 12, "groups": 1, "datagrams": 0, + "stale": { "bytes": 0, "frames": 0, "groups": 0, "datagrams": 0 } } } ``` @@ -457,13 +459,13 @@ Field semantics: when the datagram enters or leaves the model, so an egress datagram dropped by congestion or an oversized body still counts. -- `stale`: cumulative groups skipped because they drifted further behind the live - edge than the subscriber's latency budget allows, so the relay never put them on - the wire. Disjoint from `groups`: a skipped group is not delivered, and none of - its payload reaches `frames` / `bytes` either. A steady rate here means - subscribers are consistently behind the live edge, which is normal for a - real-time subscription during congestion and a problem for one that asked to - tolerate more. +- `stale`: cumulative `{ bytes, frames, groups, datagrams }` skipped because the + content drifted further behind the live edge than the subscriber's latency + budget allows, so the relay never put it on the wire. These are disjoint from + the top-level payload counters, which remain the backwards-compatible shape for + delivered content. A steady rate here means subscribers are consistently behind + the live edge, which is normal for a real-time subscription during congestion + and a problem for one that asked to tolerate more. The session tracks (`sessions.json` and any `/sessions.json`) instead map each auth root to a `{ sessions, sessions_closed }` snapshot. `sessions` diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 15ed7df05e1..312ac0ccbb1 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -68,7 +68,7 @@ async fn opus_round_trip_48k_stereo() { // The whole clip is encoded before anything decodes it. The default REAL_TIME // budget is enforced on the subscription, so without a tolerance the decoder would // take the live edge and drop the rest of the batch. - config.latency = moq_mux::Latency::max(Duration::from_secs(3600)); + config.latency = moq_mux::Latency::max(Duration::from_secs(30)); let mut consumer = decode::Consumer::new(&broadcast_consumer, cfg, "audio", config) .await .unwrap(); diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 069626e8b1c..ea478e21ef6 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -560,17 +560,17 @@ mod tests { Export::with_ts(moq_mux::Source::new(origin.consume(), "cli"), CatalogFormat::Hang) .await .unwrap() - .with_latency(moq_mux::Latency::max(BATCH)), + .with_latency(moq_mux::Latency::max(RECORDING_LATENCY)), ) .await } - /// A drift budget no test timeline comes close to, so the exporter reads every group. - /// - /// These tests publish a whole feed before exporting it, which the default + /// The media track's full retention window, so an exporter started after publishing + /// can still read every retained group. These tests publish a whole feed before + /// exporting it, which the default /// [`Latency::REAL_TIME`](moq_mux::Latency::REAL_TIME) collapses to the live edge: /// completeness has to be asked for, exactly as a real recorder does. - const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); /// Full CLI round-trip: a TS feed with undecoded streams goes through `Publish` /// (which selects the `mpegts` catalog) and the subscribe-side `Export::with_ts`, @@ -604,7 +604,7 @@ mod tests { Export::with_ts(moq_mux::Source::new(origin.consume(), "cli"), CatalogFormat::Hang) .await .unwrap() - .with_latency(moq_mux::Latency::max(BATCH)), + .with_latency(moq_mux::Latency::max(RECORDING_LATENCY)), ) .await; diff --git a/rs/moq-mux/src/codec/h264/export.rs b/rs/moq-mux/src/codec/h264/export.rs index a3fc2a54acf..08d2bc36940 100644 --- a/rs/moq-mux/src/codec/h264/export.rs +++ b/rs/moq-mux/src/codec/h264/export.rs @@ -296,7 +296,7 @@ mod tests { // The whole track is written before the exporter runs, so it needs a budget // wide enough to read it: the default skips everything but the live edge. let mut export = Export::new(crate::source::announced(&consumer), Once(Some(catalog))) - .with_latency(crate::Latency::max(std::time::Duration::from_secs(3600))); + .with_latency(crate::Latency::max(std::time::Duration::from_secs(30))); let frame0 = export.next().await.unwrap().expect("first frame"); let frame1 = export.next().await.unwrap().expect("second frame"); diff --git a/rs/moq-mux/src/codec/h265/export.rs b/rs/moq-mux/src/codec/h265/export.rs index 89135ed7308..765057f995a 100644 --- a/rs/moq-mux/src/codec/h265/export.rs +++ b/rs/moq-mux/src/codec/h265/export.rs @@ -282,7 +282,7 @@ mod tests { // The whole track is written before the exporter runs, so it needs a budget // wide enough to read it: the default skips everything but the live edge. let mut export = Export::new(crate::source::announced(&consumer), Once(Some(catalog))) - .with_latency(crate::Latency::max(std::time::Duration::from_secs(3600))); + .with_latency(crate::Latency::max(std::time::Duration::from_secs(30))); let frame0 = export.next().await.unwrap().expect("first frame"); let frame1 = export.next().await.unwrap().expect("second frame"); diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index baa35f3f84c..d3088a67235 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -766,18 +766,18 @@ mod tests { Ok(frames) } - /// Wrap `track` in a consumer whose skipping is entirely its own: the subscription - /// gets a budget no test timeline reaches, so the transport hands over every group - /// and what the consumer does with them is what the test measures. + /// Wrap `track` so only the container consumer performs latency skips. The + /// transport gets the media track's full retention window and hands over every + /// retained group; what the container does with them is what the test measures. /// /// Both layers enforce the same budget, and normally should: the transport skipping /// a group the consumer was going to skip anyway just saves the bandwidth. These /// tests are the exception, since they are about the consumer's half of it. - fn consumer_only(track: moq_net::track::Subscriber, latency: Latency) -> Consumer { + fn container_latency_only(track: moq_net::track::Subscriber, latency: Latency) -> Consumer { let control = track.control(); let consumer = Consumer::new(track, Container::Legacy).with_latency(latency); control - .update(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(3600)))) + .update(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(30)))) .unwrap(); consumer } @@ -889,7 +889,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = consumer_only(consumer_track, Latency::REAL_TIME); + let mut consumer = container_latency_only(consumer_track, Latency::REAL_TIME); // Group 0 at ts 0 keeps timestamps monotonic with sequence (groups 1-9 follow at // g*50 ms), so the test exercises latency skipping and not rewind detection. @@ -928,7 +928,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = consumer_only(consumer_track, Latency::max(Duration::from_millis(100))); + let mut consumer = container_latency_only(consumer_track, Latency::max(Duration::from_millis(100))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1871,7 +1871,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = consumer_only(consumer_track, Latency::max(Duration::from_millis(100))); + let mut consumer = container_latency_only(consumer_track, Latency::max(Duration::from_millis(100))); // Group 0: stalled at ts=0, NOT finished let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1909,7 +1909,7 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = consumer_only(consumer_track, Latency::max(Duration::from_millis(100))); + let mut consumer = container_latency_only(consumer_track, Latency::max(Duration::from_millis(100))); // Group 0: finished normally write_group(&mut track, 0, &[ts(0), ts(20_000)]); diff --git a/rs/moq-mux/src/container/flv/export_test.rs b/rs/moq-mux/src/container/flv/export_test.rs index fa809128206..732ae46af1a 100644 --- a/rs/moq-mux/src/container/flv/export_test.rs +++ b/rs/moq-mux/src/container/flv/export_test.rs @@ -12,10 +12,12 @@ use super::{Export, Import}; /// A drift budget no test timeline comes close to, so the exporter reads every group. /// -/// These tests write or import a whole broadcast and only then export it, which the +/// The media track's full retention window, so an exporter started after publishing +/// can still read every retained group. These tests write or import a whole broadcast +/// and only then export it, which the /// exporter's default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the /// live edge: completeness has to be asked for, exactly as a real recorder does. -const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); +const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); /// A minimal `AVCDecoderConfigurationRecord` (profile 0x42, level 0x1f, one SPS + PPS). fn avcc() -> Vec { @@ -179,7 +181,7 @@ async fn export_emits_sequence_headers_and_frames() { let exporter = Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let exported = drain_export(exporter, importer).await; let tags = parse_tags(&exported); diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index eab3bf37d6e..c5085fafed2 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -7,12 +7,12 @@ use mp4_atom::{DecodeMaybe, Encode}; use crate::container::test_util::{Live, PPS, SPS, raw_frame, video_frame}; -/// A drift budget no test timeline comes close to, so the exporter reads every group. -/// -/// These tests write or import a whole broadcast and only then export it, which the +/// The media track's full retention window, so an exporter started after publishing +/// can still read every retained group. These tests write or import a whole broadcast +/// and only then export it, which the /// exporter's default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the /// live edge: completeness has to be asked for, exactly as a real recorder does. -const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); +const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); /// Avc3-shape source (catalog `Container::Legacy`, `H264 { inline: true }`, /// `description: None`) → fMP4 / CMAF export must synthesize a valid init @@ -538,7 +538,7 @@ async fn next_chunk_reports_segment_metadata() { // Sub-GOP cap so GOP 0 splits into two parts (the trailing part non-independent). let mut exporter = crate::container::fmp4::Export::new(live.source(), live.catalog_stream().await) .with_fragment_duration(std::time::Duration::from_millis(20)) - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); // First emit is the init segment, which carries no segmenting metadata to assert on. chunk_now(&mut exporter) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 3332774d283..fef38aa66ff 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -4,14 +4,15 @@ use mp4_atom::{Decode, Encode}; /// A drift budget no test timeline comes close to, so every group is read. /// -/// These tests write a whole batch of groups up front and only then read them, which the +/// The media track's full retention window, so a reader started after publishing can +/// still read every retained group. These tests write every group up front, which the /// default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) budget collapses to the live /// edge: completeness has to be asked for. -const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); +const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); /// A subscription with that budget, for a test asserting every group is delivered. fn subscribe_all() -> moq_net::track::Subscription { - moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(BATCH)) + moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(RECORDING_LATENCY)) } /// Drain every group currently buffered on the consumer without waiting for new ones. diff --git a/rs/moq-mux/src/container/mkv/export_test.rs b/rs/moq-mux/src/container/mkv/export_test.rs index f2fd70d2e2f..39935745d24 100644 --- a/rs/moq-mux/src/container/mkv/export_test.rs +++ b/rs/moq-mux/src/container/mkv/export_test.rs @@ -15,10 +15,12 @@ use crate::container::test_util::{IDR, Live, PPS, SPS, raw_frame, video_frame}; /// A drift budget no test timeline comes close to, so the exporter reads every group. /// -/// These tests import a whole file and only then export it, which the exporter's default +/// The media track's full retention window, so an exporter started after publishing +/// can still read every retained group. These tests import a whole file and only then +/// export it, which the exporter's default /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the live edge: /// completeness has to be asked for, exactly as a real recorder does. -const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); +const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); #[tokio::test(start_paused = true)] async fn export_header_roundtrip_vp9_opus() { @@ -375,7 +377,7 @@ async fn export_emits_blocks_for_each_frame() { // Use per-frame clustering so each frame is observable as its own // Cluster chunk; batching is exercised in a dedicated test below. .with_fragment_duration(std::time::Duration::ZERO) - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut exported: Vec = Vec::new(); let mut importer = Some(importer); @@ -614,7 +616,7 @@ async fn export_fragment_duration_batches_blocks() { .expect("catalog consumer"); let mut exporter = crate::container::mkv::Export::new(crate::source::announced(&consumer), catalog_stream) .with_fragment_duration(std::time::Duration::from_secs(2)) - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut exported: Vec = Vec::new(); let mut importer = Some(importer); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index e1e4a8b3f9e..00e97a5d573 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -404,12 +404,12 @@ mod tests { /// [`Latency::REAL_TIME`](moq_net::Latency::REAL_TIME) budget collapses to the live /// edge: completeness has to be asked for. fn subscribe_all(track: &moq_net::track::Producer) -> moq_net::track::Subscriber { - track.subscribe(moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(BATCH))) + track.subscribe(moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(RECORDING_LATENCY))) } - /// A drift budget no test timeline comes close to, so nothing is skipped. Clamped to - /// the track's own retention window, which is what actually bounds it. - const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + /// The media track's full retention window, so readers started after publishing + /// can still consume every retained group. + const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); fn frame(timestamp_us: u64, keyframe: bool) -> Frame { Frame { @@ -559,9 +559,12 @@ mod tests { async fn discontinuity_publishes_an_empty_group() { // The resumed clock jumps forty minutes, so both the retention window and the // drift budget have to cover it or the pre-discontinuity group reads as ancient. - let info = hang::container::track_info().with_latency_max(BATCH); + let discontinuity_latency = std::time::Duration::from_secs(41 * 60); + let info = hang::container::track_info().with_latency_max(discontinuity_latency); let track = track_producer("test", info); - let consumer = subscribe_all(&track); + let consumer = track.subscribe( + moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(discontinuity_latency)), + ); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 1d9b01dd57f..ab2acb24941 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -228,7 +228,8 @@ impl ExportSource { Poll::Pending => return Poll::Pending, } }; - self.state = SourceState::Subscribing(broadcast.track(&name)?.subscribe(None)); + let subscription = moq_net::track::Subscription::default().with_latency(self.latency); + self.state = SourceState::Subscribing(broadcast.track(&name)?.subscribe(subscription)); } // Resolve the subscription before reading any frames. @@ -436,4 +437,24 @@ mod tests { .unwrap_or_else(|| panic!("{reference:?} should keep the rendition")); } } + + /// The requested budget must be present on the first SUBSCRIBE. Updating it + /// after SUBSCRIBE_OK cannot recover backlog the publisher already skipped. + #[tokio::test] + async fn latency_is_sent_with_the_initial_subscription() { + let live = Live::avc3(); + let latency = crate::Latency::max(std::time::Duration::from_secs(10)); + let mut export = ExportSource::for_video(&live.source(), live.track.name(), &video(None), latency).unwrap(); + + let observed = kio::wait(|waiter| { + let _ = export.poll_read(waiter); + match live.track.subscription() { + Some(subscription) => Poll::Ready(subscription.latency), + None => Poll::Pending, + } + }) + .await; + + assert_eq!(observed, latency); + } } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index ad812bc415e..24c50ff066b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -62,10 +62,12 @@ fn length_prefixed(nals: &[&[u8]]) -> Bytes { /// paused time) or the stream ends. /// A drift budget no test timeline comes close to, so the exporter reads every group. /// -/// These tests write a whole broadcast up front and only then export it, which the +/// The media track's full retention window, so an exporter started after publishing +/// can still read every retained group. These tests write a whole broadcast up front +/// and only then export it, which the /// exporter's default [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) collapses to the /// live edge: completeness has to be asked for, exactly as a real recorder does. -const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); +const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { drain_with(Export::new(crate::source::announced(&consumer)).await.unwrap()).await @@ -73,7 +75,7 @@ async fn drain(consumer: moq_net::broadcast::Consumer) -> BytesMut { /// `drain` for an exporter built with an explicit catalog extension. async fn drain_with(exporter: Export) -> BytesMut { - let mut exporter = exporter.with_latency(crate::Latency::max(BATCH)); + let mut exporter = exporter.with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut out = BytesMut::new(); // `while let Ok` stops on the first timeout (`Pending`: no more output). while let Ok(res) = tokio::time::timeout(std::time::Duration::from_secs(1), exporter.next()).await { @@ -647,8 +649,8 @@ async fn export_scte35_roundtrip() { let name = scte_track(&snapshot).expect("a scte35 track"); let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut scte_reader = - crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); + let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy) + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let frame = scte_reader .read() .await @@ -743,8 +745,8 @@ async fn export_pes_verbatim_roundtrip() { let name = name.clone(); let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = - crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); + let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let frame = reader .read() .await @@ -812,8 +814,8 @@ async fn scte35_without_video_export_is_rejected() { /// Subscribe to a track and read every retained frame payload it holds. async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec> { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = - crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); + let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -1129,8 +1131,8 @@ fn scte_track(snap: &crate::catalog::hang::Catalog) -> Option Vec<(Vec, Timestamp)> { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = - crate::container::Consumer::new(track, HangContainer::Legacy).with_latency(crate::Latency::max(BATCH)); + let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut cues = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 964c17788ba..a530cff802e 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2308,10 +2308,11 @@ mod test { /// A drift budget no test timeline comes close to, so the reader sees every group. /// - /// These tests import a whole file and only then read it back, which the default + /// The media track's full retention window, so a reader started after importing can + /// still read every retained group. These tests import a whole file first, which the default /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) budget collapses to the live /// edge: completeness has to be asked for. - const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); + const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); use mpeg2ts::es::StreamType; use super::SectionReassembler; @@ -2646,7 +2647,7 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); import.finish().unwrap(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(BATCH)); + let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(RECORDING_LATENCY)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -2879,7 +2880,7 @@ mod test { .clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(BATCH)); + .with_latency(Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { @@ -3690,7 +3691,7 @@ mod test { .await .unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(BATCH)); + .with_latency(Latency::max(RECORDING_LATENCY)); let published = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await; assert!( matches!(published, Ok(Ok(Some(_)))), @@ -3945,7 +3946,7 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(BATCH)); + let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(RECORDING_LATENCY)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -4090,7 +4091,7 @@ mod test { assert_eq!(track.pid, DATA_PID, "recorded the original PID"); let track = consumer.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(BATCH)); + let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(RECORDING_LATENCY)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("verbatim read timed out") diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 48f84580ee7..2d86ff83401 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -7,10 +7,11 @@ use bytes::BytesMut; /// A drift budget no test timeline comes close to, so the reader sees every group. /// -/// These tests import a whole file and only then read it back, which the default +/// The media track's full retention window, so a reader started after importing can +/// still read every retained group. These tests import a whole file first, which the default /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) budget collapses to the live edge: /// completeness has to be asked for. -const BATCH: std::time::Duration = std::time::Duration::from_secs(3600); +const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); /// Decode a whole TS buffer into a fresh broadcast and return the catalog. fn import_ts(data: &[u8]) -> crate::catalog::hang::Catalog { @@ -151,7 +152,7 @@ async fn import_opus_frames() { let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -293,7 +294,7 @@ async fn import_export_import_roundtrip() { let mut exporter = crate::container::ts::Export::new(crate::source::announced(&consumer)) .await .unwrap() - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut out = BytesMut::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_secs(1), exporter.next()).await { match res.expect("exporter error") { @@ -347,7 +348,7 @@ async fn survives_midstream_join() { // anchors the one and only group. let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { frames.push(frame); @@ -392,7 +393,7 @@ async fn kyrion_dirtystart_extracts_real_cues() { .expect("scte35 track"); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(crate::Latency::max(BATCH)); + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut cues = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { cues.push((frame.payload.to_vec(), frame.timestamp)); diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 0932345fdb8..d3f1bd4b8e7 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -892,7 +892,7 @@ mod tests { // REAL_TIME budget would collapse to the live edge. let subscriber = track.subscribe( moq_net::track::Subscription::default() - .with_latency(moq_net::Latency::max(std::time::Duration::from_secs(3600))), + .with_latency(moq_net::Latency::max(std::time::Duration::from_secs(30))), ); let config = crate::codec::opus::Config::new(48_000, 2); let import = crate::codec::opus::Import::new(track, catalog.reserve(), config.into()).unwrap(); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index c4084bf9484..56e2c6cc2f7 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -140,6 +140,16 @@ pub(crate) struct GroupState { } impl GroupState { + /// Content still available to a reader of this group. + fn content(&self) -> stats::Content { + stats::Content { + bytes: self.cache, + frames: self.next_index.saturating_sub(self.offset) as u64, + groups: 1, + datagrams: 0, + } + } + /// Resolve the source for the frame at `index`: a completed frame (whole) or the /// in-flight tail (streamed). Used by [`Consumer::poll_next_frame`]. fn poll_frame_source(&self, index: usize) -> Poll>> { @@ -635,6 +645,11 @@ impl Producer { self.state.read().timestamp } + /// Snapshot the content a subscriber would discard by skipping this group. + pub(crate) fn content(&self) -> stats::Content { + self.state.read().content() + } + /// The group's full cached footprint (payload plus fixed overhead), used by the /// track to size this group as an eviction victim. pub(crate) fn cache_size(&self) -> u64 { @@ -868,6 +883,19 @@ impl std::ops::Deref for Consumer { } impl Consumer { + /// Snapshot the content this cursor would discard if its group were skipped. + pub(crate) fn content(&self) -> stats::Content { + match &self.inner { + ConsumerKind::Plain(plain) => plain.state.read().content(), + // Drift is evaluated before a segment copy is wrapped as a spliced group. + // Keep the group count honest if a future caller reaches this fallback. + ConsumerKind::Spliced(_) => stats::Content { + groups: 1, + ..Default::default() + }, + } + } + /// Rebuild this consumer as the head of a group assembled across route changes, /// keeping the group's identity and its track's properties. See [`super::resume`]. pub(crate) fn into_spliced(self, spliced: super::resume::Group) -> Self { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 142e1593a6b..ba8112b3833 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3893,9 +3893,12 @@ mod tests { let report = registry.report(); let entry = report.traffic.iter().find(|e| e.path.as_str() == "demo").unwrap(); - assert_eq!(entry.publisher.stale, 2, "two groups skipped on the way out"); + assert_eq!(entry.publisher.stale.groups, 2, "two groups skipped on the way out"); + assert_eq!(entry.publisher.stale.frames, 2); + assert_eq!(entry.publisher.stale.bytes, 4); + assert_eq!(entry.publisher.stale.datagrams, 0); assert_eq!(entry.publisher.groups, 1, "only the live edge was delivered"); - assert_eq!(entry.subscriber.stale, 0, "ingress wrote all three"); + assert_eq!(entry.subscriber.stale.groups, 0, "ingress wrote all three"); assert_eq!(entry.subscriber.groups, 3); } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 56b13c92b7a..92ae0e158c7 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1049,6 +1049,9 @@ struct SegmentSub { start: Option, end: Option, sub: SubState, + /// A completed segment's cursor, retained while parked groups may need their + /// latency budget re-evaluated after the outer cap rises. + terminal: Option, /// The producer dropped this segment (pruned, or replaced before producing). /// The cursor drains what it already holds, then retires; see /// [`Self::retired`]. @@ -1062,6 +1065,23 @@ struct SegmentSub { } impl SegmentSub { + /// The cursor that can evaluate staleness, whether the segment is live or has + /// already completed. + fn stale_sub_mut(&mut self) -> Option<&mut track::Subscriber> { + match &mut self.sub { + SubState::Active(sub) => Some(sub), + _ => self.terminal.as_mut(), + } + } + + /// Move an active cursor into terminal retention and mark the segment done. + fn complete(&mut self, count: Option) { + let previous = std::mem::replace(&mut self.sub, SubState::Done(count)); + if let SubState::Active(sub) = previous { + self.terminal = Some(sub); + } + } + /// The first group this segment can serve, for the underlying read cursor. fn first_group(&self) -> u64 { self.start.map_or(0, |start| start.group) @@ -1178,6 +1198,7 @@ impl Subscriber { } if seg.pruned { seg.sub = SubState::Done(None); + seg.terminal = None; seg.parked.clear(); cut -= 1; } @@ -1203,8 +1224,9 @@ impl Subscriber { }; self.last_prefs = prefs; for seg in &mut self.segments { - if let SubState::Active(sub) = &mut seg.sub { - let _ = sub.update(slice(&self.last_prefs, seg.start, seg.end)); + let prefs = slice(&self.last_prefs, seg.start, seg.end); + if let Some(sub) = seg.stale_sub_mut() { + let _ = sub.update(prefs); } } } @@ -1274,7 +1296,7 @@ impl Subscriber { if existing.end != segment.end { existing.end = segment.end; let cap = Self::stale_cap(existing, self.end_sequence); - if let SubState::Active(sub) = &mut existing.sub { + if let Some(sub) = existing.stale_sub_mut() { // The boundary bounds the drift anchor as well as the demand. sub.set_stale_cap(cap); // Shrink the demand so the session can cap upstream. The @@ -1298,6 +1320,7 @@ impl Subscriber { start: segment.start, end: segment.end, sub: SubState::Pending(sub), + terminal: None, pruned: false, parked: BTreeMap::new(), }); @@ -1402,13 +1425,13 @@ impl Subscriber { Poll::Ready(count) => count, Poll::Pending => None, }; - seg.sub = SubState::Done(count); + seg.complete(count); return Poll::Ready(None); } // A dead segment stalls the logical track rather than erroring; // the next switch resumes it. Poll::Ready(Err(_)) => { - seg.sub = SubState::Done(None); + seg.complete(None); return Poll::Ready(None); } // An empty cursor on a pruned segment is NOT proof it drained: @@ -1476,7 +1499,7 @@ impl Subscriber { // The cap rising widens the live edge too, so a group parked while it // was current can be a backlog by the time it is owed again. Re-check it // rather than hand back content the budget has since given up on. - if let SubState::Active(sub) = &mut self.segments[index].sub + if let Some(sub) = self.segments[index].stale_sub_mut() && matches!(sub.poll_stale(&group, waiter), Poll::Ready(Ok(true))) { continue; @@ -1686,11 +1709,11 @@ impl Subscriber { SubState::Done(count) => Poll::Ready(Ok(count.unwrap_or(0))), SubState::Active(sub) => match ready!(sub.poll_finished(waiter)) { Ok(count) => { - seg.sub = SubState::Done(Some(count)); + seg.complete(Some(count)); Poll::Ready(Ok(count)) } Err(_) => { - seg.sub = SubState::Done(None); + seg.complete(None); Poll::Ready(Ok(0)) } }, @@ -1727,7 +1750,7 @@ impl Subscriber { let end_sequence = self.end_sequence; for seg in &mut self.segments { let cap = Self::stale_cap(seg, end_sequence); - if let SubState::Active(sub) = &mut seg.sub { + if let Some(sub) = seg.stale_sub_mut() { sub.set_stale_cap(cap); } } @@ -1743,11 +1766,11 @@ impl Subscriber { /// The segments are subscribed untagged, so their skips have nowhere to be counted /// until they reach the [`track::Subscriber`] that owns the stats scope. A retired /// segment's tail is lost with it, which is the same bound its groups already had. - pub(crate) fn take_stale(&mut self) -> u64 { - let mut stale = 0; + pub(crate) fn take_stale(&mut self) -> crate::stats::Content { + let mut stale = crate::stats::Content::default(); for seg in &mut self.segments { - if let SubState::Active(sub) = &mut seg.sub { - stale += sub.take_stale(); + if let Some(sub) = seg.stale_sub_mut() { + stale.add(sub.take_stale()); } } stale @@ -2060,6 +2083,33 @@ mod test { recv_pending(&mut sub); } + /// Completing the segment must not discard the cursor that evaluates parked + /// groups. A later cap increase still uses the terminal track's live edge. + #[tokio::test] + async fn a_raised_cap_rechecks_parked_groups_after_segment_finish() { + let retain = track::Info::default().with_latency_max(Duration::from_secs(60)); + let (mut track_a, consumer_a) = track_pair_with("a", retain); + + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + sub.end_at(0); + + write_group_at(&mut track_a, 0, "a0", Duration::ZERO); + assert_eq!(recv(&mut sub), 0); + + write_group_at(&mut track_a, 1, "a1", Duration::from_secs(1)); + write_group_at(&mut track_a, 2, "a2", Duration::from_secs(30)); + track_a.finish().unwrap(); + // Drive the inner cursor through its clean end while both newer groups are + // parked above the cap. + recv_pending(&mut sub); + + sub.end_at(None); + assert_eq!(recv(&mut sub), 2, "the stale parked group is skipped after finish"); + recv_pending(&mut sub); + } + #[tokio::test] async fn takeover_replaces_empty_segment() { let (track_a, consumer_a) = track_pair("a"); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 00424a79a48..c56db9515ec 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -225,15 +225,18 @@ pub(crate) struct TrackState { fetch: kio::Shared, } -/// What a frame-level scan found: the frame, where it came from so the cursor can -/// advance past it, and how many groups the drift budget made the scan step over on the -/// way there (counted here rather than skipped silently, since a silent drop is -/// indistinguishable from loss). +/// What a frame-level scan found, including content skipped on the way to either a +/// frame or the clean end of the track. +struct FrameScan { + hit: Option, + stale: stats::Content, +} + +/// A frame found by a frame-level scan and the cursor position it came from. struct FrameHit { frame: frame::Frame, index: usize, sequence: u64, - stale: u64, } /// A cached group plus its bookkeeping in the track's `lookup` map. @@ -361,13 +364,13 @@ impl TrackState { next_sequence: u64, drift: Drift, waiter: &kio::Waiter, - ) -> Poll>> { + ) -> Poll> { let start = index.saturating_sub(self.offset); let mut pending_seen = false; // Groups the budget gave up on below the frame this returns. Counted only on the // winning scan, whose cursor jumps past them, so a scan that ends Pending leaves // them to be counted exactly once by the one that eventually succeeds. - let mut stale = 0; + let mut stale = stats::Content::default(); for (i, (sequence, stamp)) in self.arrival.iter().enumerate().skip(start) { if *sequence < next_sequence { continue; @@ -384,19 +387,21 @@ impl TrackState { // from a group the subscription has given up on is no fresher for being // read one frame at a time. if self.is_stale(*sequence, drift.edge, drift.budget) { - stale += 1; + stale.add(slot.group.content()); continue; } let mut consumer = slot.group.consume(); match consumer.poll_read_frame(waiter) { Poll::Ready(Ok(Some(frame))) => { - return Poll::Ready(Ok(Some(FrameHit { - frame, - index: self.offset + i, - sequence: *sequence, + return Poll::Ready(Ok(FrameScan { + hit: Some(FrameHit { + frame, + index: self.offset + i, + sequence: *sequence, + }), stale, - }))); + })); } Poll::Ready(Ok(None)) => continue, // A single group failing (aborted upstream, or evicted from the @@ -414,7 +419,7 @@ impl TrackState { if pending_seen { Poll::Pending } else if self.is_complete() { - Poll::Ready(Ok(None)) + Poll::Ready(Ok(FrameScan { hit: None, stale })) } else if let Some(err) = &self.abort { Poll::Ready(Err(err.clone())) } else { @@ -1403,7 +1408,7 @@ impl Producer { end_sequence: None, parked: BTreeMap::new(), stale_cap: None, - stale: 0, + stale: stats::Content::default(), }), // A producer-side (in-process) subscribe is not egress: stay untagged. stats: stats::Scope::default(), @@ -2263,7 +2268,7 @@ impl Subscribing { end_sequence: None, parked: BTreeMap::new(), stale_cap: None, - stale: 0, + stale: stats::Content::default(), }), stats: self.stats.clone(), _stats_sub: self.stats.subscribe(), @@ -2617,7 +2622,7 @@ struct PlainSubscriber { /// is the outer [`Subscriber`], which may be reading this cursor through a /// [`super::resume::Subscriber`] segment (untagged, so the outer wrapper is the /// only place attribution happens once). - stale: u64, + stale: stats::Content, } impl PlainSubscriber { @@ -2634,14 +2639,14 @@ impl PlainSubscriber { } /// Take the groups skipped since the last call, for the owner to meter. - fn take_stale(&mut self) -> u64 { + fn take_stale(&mut self) -> stats::Content { std::mem::take(&mut self.stale) } /// Note a group an outer reader skipped on this cursor's behalf, so it lands in the /// same counter as the ones skipped here. - fn note_stale(&mut self) { - self.stale += 1; + fn note_stale(&mut self, group: &group::Consumer) { + self.stale.add(group.content()); } /// This subscriber's clamped drift budget and the live edge to measure against, @@ -2733,7 +2738,7 @@ impl PlainSubscriber { // Drop a group the drift budget has given up on and keep scanning, so one // poll walks a whole backlog off rather than handing it out group by group. if ready!(self.poll_stale(&consumer, drift, waiter))? { - self.stale += 1; + self.stale.add(consumer.content()); continue; } return Poll::Ready(Ok(Some(consumer))); @@ -2764,7 +2769,7 @@ impl PlainSubscriber { // Advance before the budget check: a skipped group is consumed, not retried. self.next_sequence = group.sequence.saturating_add(1); if ready!(self.poll_stale(&group, drift, waiter))? { - self.stale += 1; + self.stale.add(group.content()); continue; } return Poll::Ready(Ok(Some(group))); @@ -2774,13 +2779,14 @@ impl PlainSubscriber { fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { let drift = ready!(self.poll_drift(waiter))?; let lower = self.min_sequence.max(self.next_sequence); - let Some(hit) = ready!(self.poll(waiter, |state| { + let scan = ready!(self.poll(waiter, |state| { state.poll_read_frame(self.index, lower, drift, waiter) - })?) else { + })?); + self.stale.add(scan.stale); + let Some(hit) = scan.hit else { return Poll::Ready(Ok(None)); }; - self.stale += hit.stale; self.index = hit.index + 1; self.next_sequence = hit.sequence.saturating_add(1); Poll::Ready(Ok(Some(hit.frame))) @@ -2852,7 +2858,7 @@ impl Subscriber { /// Take the groups the drift budget skipped since the last call, so a nesting /// handle (a spliced subscriber reading this one as a segment) can attribute them. - pub(crate) fn take_stale(&mut self) -> u64 { + pub(crate) fn take_stale(&mut self) -> stats::Content { match &mut self.inner { SubscriberKind::Plain(plain) => plain.take_stale(), SubscriberKind::Spliced(spliced) => spliced.take_stale(), @@ -2882,7 +2888,7 @@ impl Subscriber { let drift = ready!(plain.poll_drift(waiter))?; let stale = ready!(plain.poll_stale(group, drift, waiter))?; if stale { - plain.note_stale(); + plain.note_stale(group); } Poll::Ready(Ok(stale)) } @@ -4094,7 +4100,32 @@ mod test { // frame-level path reports its drops like the group-level ones. let mut subscriber = producer.subscribe(None); subscriber.read_frame().await.unwrap().expect("a frame"); - assert_eq!(subscriber.take_stale(), 3); + let stale = subscriber.take_stale(); + assert_eq!(stale.groups, 3); + assert_eq!(stale.frames, 3); + } + + #[tokio::test] + async fn read_frame_counts_stale_groups_at_eof() { + let mut producer = track_producer("test", None); + append_at(&mut producer, 0); + append_at(&mut producer, 1000); + + // The live edge starts past frame zero, so the frame helper cannot read it. + // The two older groups are still stale and must be reported when the scan + // reaches the clean end without returning a frame. + let mut edge = producer.append_group().unwrap(); + edge.start_at(1).unwrap(); + edge.write_frame(Timestamp::from_millis(2000).unwrap(), b"edge".to_vec()) + .unwrap(); + edge.finish().unwrap(); + producer.finish().unwrap(); + + let mut subscriber = producer.subscribe(None); + assert!(subscriber.read_frame().await.unwrap().is_none()); + let stale = subscriber.take_stale(); + assert_eq!(stale.groups, 2); + assert_eq!(stale.frames, 2); } #[test] diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index e44ea426faa..9403aab5144 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -151,9 +151,36 @@ pub(crate) struct Counters { groups: AtomicU64, // Subset of `groups` carried over an unreliable QUIC datagram. datagrams: AtomicU64, - // Groups the drift budget gave up on before they were delivered. Disjoint from - // `groups`, which counts only what was handed over. - stale: AtomicU64, + // Content the drift budget gave up on before delivery. Disjoint from the + // top-level payload counters, which count only what was handed over. + stale: ContentCounters, +} + +/// Atomic backing for one [`Content`] readout. +#[derive(Default, Debug)] +struct ContentCounters { + bytes: AtomicU64, + frames: AtomicU64, + groups: AtomicU64, + datagrams: AtomicU64, +} + +impl ContentCounters { + fn snapshot(&self) -> Content { + Content { + bytes: self.bytes.load(Ordering::Relaxed), + frames: self.frames.load(Ordering::Relaxed), + groups: self.groups.load(Ordering::Relaxed), + datagrams: self.datagrams.load(Ordering::Relaxed), + } + } + + fn add(&self, content: Content) { + self.bytes.fetch_add(content.bytes, Ordering::Relaxed); + self.frames.fetch_add(content.frames, Ordering::Relaxed); + self.groups.fetch_add(content.groups, Ordering::Relaxed); + self.datagrams.fetch_add(content.datagrams, Ordering::Relaxed); + } } impl Counters { @@ -177,7 +204,7 @@ impl Counters { let frames = self.frames.load(Ordering::Relaxed); let groups = self.groups.load(Ordering::Relaxed); let datagrams = self.datagrams.load(Ordering::Relaxed); - let stale = self.stale.load(Ordering::Relaxed); + let stale = self.stale.snapshot(); Traffic { announced, announced_closed, @@ -196,6 +223,35 @@ impl Counters { } } +/// Payload-volume counters for content with the same delivery outcome. +/// +/// This is the nested shape used by [`Traffic::stale`]. The successfully +/// delivered equivalents remain as top-level [`Traffic`] fields for wire +/// compatibility with existing stats consumers. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +#[non_exhaustive] +pub struct Content { + /// Cumulative payload bytes. + pub bytes: u64, + /// Cumulative frames. + pub frames: u64, + /// Cumulative groups. + pub groups: u64, + /// Cumulative single-frame groups carried as unreliable datagrams. + pub datagrams: u64, +} + +impl Content { + /// Fold another readout into this one, counter by counter. + pub(crate) fn add(&mut self, other: Self) { + self.bytes += other.bytes; + self.frames += other.frames; + self.groups += other.groups; + self.datagrams += other.datagrams; + } +} + /// Per-(tier, root) session gauge. One of these is shared (via `Arc`) by every /// [`Session`] guard for the same auth root on the same tier: `sessions` /// bumps on connect, `sessions_closed` on disconnect. @@ -262,11 +318,11 @@ pub struct Traffic { /// A subset of `groups`: each one also counts there and its payload in /// `frames` / `bytes`. pub datagrams: u64, - /// Cumulative groups skipped because they drifted past a subscriber's - /// [`Latency`](crate::Latency) budget. Disjoint from `groups`: a skipped group is - /// never handed over, so none of its payload reaches `frames` / `bytes` either. - /// A steady rate here means subscribers are consistently behind the live edge. - pub stale: u64, + /// Content skipped because it drifted past a subscriber's + /// [`Latency`](crate::Latency) budget. Disjoint from the top-level payload + /// counters: skipped content is never handed over. A steady rate here means + /// subscribers are consistently behind the live edge. + pub stale: Content, } impl Traffic { @@ -284,7 +340,7 @@ impl Traffic { self.frames += other.frames; self.groups += other.groups; self.datagrams += other.datagrams; - self.stale += other.stale; + self.stale.add(other.stale); } /// True while the broadcast is announced (an announce guard is open). @@ -1015,13 +1071,10 @@ impl Meter { self.counters.is_some() } - /// Bump `stale` by `n` (groups skipped before delivery by the drift budget). - pub(crate) fn stale(&self, n: u64) { - if n == 0 { - return; - } + /// Record content skipped before delivery by the drift budget. + pub(crate) fn stale(&self, content: Content) { if let Some(counters) = self.counters() { - counters.stale.fetch_add(n, Ordering::Relaxed); + counters.stale.add(content); } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index c0140a3c6e3..38cae011737 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -136,7 +136,7 @@ mod tests { // the second `next()` would then block forever waiting for a group that was // skipped. let mut export = moq_mux::container::fmp4::Export::new(source, catalog) - .with_latency(moq_mux::Latency::max(std::time::Duration::from_secs(3600))); + .with_latency(moq_mux::Latency::max(std::time::Duration::from_secs(30))); let init = export.next().await.unwrap().expect("CMAF init"); let fragment = export.next().await.unwrap().expect("CMAF fragment"); From 66e4e02269623ee7bbf5468e3b48df150207f3e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 17:07:44 -0700 Subject: [PATCH 05/32] fix(moq-mux): adapt latency test to optional rendition Co-Authored-By: Codex --- rs/moq-mux/src/container/source.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index ab2acb24941..39e24733335 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -444,7 +444,9 @@ mod tests { async fn latency_is_sent_with_the_initial_subscription() { let live = Live::avc3(); let latency = crate::Latency::max(std::time::Duration::from_secs(10)); - let mut export = ExportSource::for_video(&live.source(), live.track.name(), &video(None), latency).unwrap(); + let mut export = ExportSource::for_video(&live.source(), live.track.name(), &video(None), latency) + .unwrap() + .expect("fixture should produce a video rendition"); let observed = kio::wait(|waiter| { let _ = export.poll_read(waiter); From c700bbc9174e76f0eeb940ee41cb92c4b3be1133 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 17:49:39 -0700 Subject: [PATCH 06/32] fix(moq-net): complete latency enforcement Enforce subscriber latency against both presentation time and wall-clock arrival so stalled groups without timestamps cannot block real-time consumers. Apply configured latency to initial decoder subscriptions across the native consumers. Co-Authored-By: Codex --- doc/concept/layer/moq-lite.md | 3 +- rs/libmoq/src/consume.rs | 12 +- rs/moq-audio/src/decode/consumer.rs | 10 +- rs/moq-audio/src/decode/decoder.rs | 3 +- rs/moq-gst/src/source/imp.rs | 7 +- rs/moq-net/src/model/resume.rs | 19 +-- rs/moq-net/src/model/track.rs | 175 +++++++++++++++++----------- rs/moq-video/src/decode/consumer.rs | 6 +- rs/moq-video/src/decode/decoder.rs | 4 +- 9 files changed, 152 insertions(+), 87 deletions(-) diff --git a/doc/concept/layer/moq-lite.md b/doc/concept/layer/moq-lite.md index fc52cfd76df..974a6788e68 100644 --- a/doc/concept/layer/moq-lite.md +++ b/doc/concept/layer/moq-lite.md @@ -116,7 +116,8 @@ Each Subscription consists of a few properties: - **Group Order**: The order in which groups are delivered. Defaults to descending; higher IDs are delivered first. - **Subscriber Max Latency**: The maximum age of a non-latest group before it is skipped. Defaults to zero, so stale groups are skipped immediately. -That age is measured on the media timeline (a group's first timestamp against the newest one), not by when it arrived, so a backlog delivered as a burst still counts as old. +That age is measured both on the media timeline (a group's first timestamp against the newest stamped one) and by wall-clock arrival time, and either limit can expire the group. +The media timeline keeps a backlog delivered as a burst old, while wall-clock time backstops stalled or empty groups that have no timestamp. Both ends apply it: the publisher skips a group rather than sending it, and the subscriber skips it again as it reads. The publisher only ever sees the most tolerant budget across its subscribers, which is why the subscriber applies it too. Asking for old groups with `Group Start` does not exempt them: a start bounds what you are sent, and a subscriber that wants history has to raise its budget to match. diff --git a/rs/libmoq/src/consume.rs b/rs/libmoq/src/consume.rs index 736b775c606..b945873c0e1 100644 --- a/rs/libmoq/src/consume.rs +++ b/rs/libmoq/src/consume.rs @@ -379,7 +379,11 @@ impl Consume { let res = async move { let track = broadcast .track(&name)? - .subscribe(moq_net::track::Subscription::default().with_priority(hang::catalog::PRIORITY.video)) + .subscribe( + moq_net::track::Subscription::default() + .with_priority(hang::catalog::PRIORITY.video) + .with_latency(latency), + ) .await?; let track = moq_mux::container::Consumer::new(track, container).with_latency(latency); Self::run_track(on_frame, track, channel.1).await @@ -428,7 +432,11 @@ impl Consume { let res = async move { let track = broadcast .track(&name)? - .subscribe(moq_net::track::Subscription::default().with_priority(hang::catalog::PRIORITY.audio)) + .subscribe( + moq_net::track::Subscription::default() + .with_priority(hang::catalog::PRIORITY.audio) + .with_latency(latency), + ) .await?; let track = moq_mux::container::Consumer::new(track, container).with_latency(latency); Self::run_track(on_frame, track, channel.1).await diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index f1d414688a1..6ff14c48411 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -50,7 +50,11 @@ impl Consumer { let name = name.into(); let track = broadcast .track(&name)? - .subscribe(moq_net::track::Subscription::default().with_priority(hang::catalog::PRIORITY.audio)) + .subscribe( + moq_net::track::Subscription::default() + .with_priority(hang::catalog::PRIORITY.audio) + .with_latency(config.latency), + ) .await?; // The catalog says how the track is framed, and it is not always the legacy // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a @@ -170,23 +174,27 @@ mod tests { async fn reads_the_container_the_catalog_declares() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast.create_track("audio", hang::container::track_info()).unwrap(); + let observed = track.clone(); let subscriber = broadcast.consume(); let mut catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); catalog.container = hang::catalog::Container::Loc; let mut producer = moq_mux::container::Producer::new(track, moq_mux::catalog::hang::Container::Loc); + let latency = moq_mux::Latency::max(std::time::Duration::from_millis(250)); let mut consumer = Consumer::new( &subscriber, &catalog, "audio", Config { format: Format::F32, + latency, ..Config::new() }, ) .await .unwrap(); + assert_eq!(observed.subscription().unwrap().latency, latency); let samples = [0.25f32, -0.5, 0.75, -1.0]; let payload: Vec = samples.iter().flat_map(|sample| sample.to_le_bytes()).collect(); diff --git a/rs/moq-audio/src/decode/decoder.rs b/rs/moq-audio/src/decode/decoder.rs index d6ace02e296..9506930e956 100644 --- a/rs/moq-audio/src/decode/decoder.rs +++ b/rs/moq-audio/src/decode/decoder.rs @@ -33,7 +33,8 @@ pub struct Config { pub channels: Option, /// How far playback may drift from the live edge before skipping a stalled group. /// - /// Forwarded to [`moq_mux::container::Consumer::with_latency`]. Defaults to + /// Applied to the initial transport subscription and + /// [`moq_mux::container::Consumer::with_latency`]. Defaults to /// [`Latency::REAL_TIME`](moq_mux::Latency::REAL_TIME), which skips aggressively. /// Set [`Latency::max`](moq_mux::Latency::max) to the playout buffer you can /// tolerate (typically tens to a few hundred ms) for the best diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 1a550531a4e..e91f0606585 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -442,9 +442,10 @@ async fn reconcile( } .fetch_add(1, Ordering::Relaxed); - let track_subscriber = broadcast.track(&name)?.subscribe(None).await?; - let track = moq_mux::container::Consumer::new(track_subscriber, container) - .with_latency(moq_mux::Latency::max(Duration::from_secs(1))); + let latency = moq_mux::Latency::max(Duration::from_secs(1)); + let subscription = moq_net::track::Subscription::default().with_latency(latency); + let track_subscriber = broadcast.track(&name)?.subscribe(subscription).await?; + let track = moq_mux::container::Consumer::new(track_subscriber, container).with_latency(latency); let descriptor = TrackDescriptor { kind: d.kind, diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 92ae0e158c7..32dd70b4c96 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1817,6 +1817,11 @@ mod test { (producer, consumer) } + /// A bounded replay window for tests whose subject requires every buffered group. + fn replay() -> Subscription { + Subscription::default().with_latency(Latency::max(Duration::from_secs(30))) + } + fn write_group(producer: &mut track::Producer, sequence: u64, payload: &str) { let mut group = producer.create_group(group::Info { sequence }).unwrap(); group.write_frame(Timestamp::ZERO, payload.as_bytes().to_vec()).unwrap(); @@ -1825,8 +1830,8 @@ mod test { /// Like [`write_group`], but placing the group on a real media timeline so the /// drift budget has something to measure. Most tests here are about splicing - /// mechanics and use [`write_group`], which leaves every group at the same instant - /// and therefore never stale. + /// mechanics and use [`write_group`], paired with [`replay`] when they need every + /// buffered group rather than the real-time live edge. fn write_group_at(producer: &mut track::Producer, sequence: u64, payload: &str, at: Duration) { let mut group = producer.create_group(group::Info { sequence }).unwrap(); group @@ -1888,7 +1893,7 @@ mod test { let mut producer = Producer::new(); producer.switch(&consumer_a, None).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); write_group(&mut track_a, 0, "a0"); write_group(&mut track_a, 1, "a1"); @@ -1976,7 +1981,7 @@ mod test { // No segments yet: the takeover is unbounded. producer.takeover(&consumer_a).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); write_group(&mut track_a, 0, "a0"); write_group(&mut track_a, 1, "a1"); assert_eq!(recv(&mut sub), 0); @@ -2330,7 +2335,7 @@ mod test { let mut producer = Producer::new(); producer.switch(&consumer_a, None).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); sub.end_at(1); @@ -2379,7 +2384,7 @@ mod test { let mut producer = Producer::new(); producer.switch(&consumer_a, None).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); let next = |sub: &mut Subscriber| { kio::wait(|waiter| sub.poll_next_group(waiter)) @@ -2867,7 +2872,7 @@ mod test { let mut producer = Producer::new(); producer.takeover(&consumer_a).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); recv_pending(&mut sub); assert_eq!(track_a.subscription().unwrap().start, None); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c56db9515ec..b0229fd3dc0 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -247,6 +247,10 @@ struct FrameHit { struct Slot { group: group::Producer, + // When this group entered the track at this hop. This is the wall-clock + // backstop for groups whose first frame has not supplied a timestamp yet. + arrived: web_async::time::Instant, + // Incarnation stamp, echoed by this slot's arrival entry (if any). A re-served // sequence (an aborted group re-created by the publisher or re-fetched as // backfill) gets a fresh stamp, so a historical arrival entry can't resolve to @@ -523,49 +527,51 @@ impl TrackState { self.info.as_ref().map(|info| info.latency_max) } - /// The live edge a subscription bounded at `cap` measures drift against: the - /// highest-sequence group at or below it that has presented a frame, and that - /// frame's timestamp. + /// The live edges a subscription bounded at `cap` measures drift against: the + /// highest-sequence group for wall-clock age, plus the highest one that has + /// presented a frame for timestamp age. /// - /// One scan answers a whole poll. The anchor for any candidate is "the newest - /// stamped group above it", and the highest-sequence stamped group in range is above - /// every candidate below it and above none at or past it, so it is that anchor for - /// all of them. Recomputing per candidate would make walking a backlog of N groups - /// off in one poll cost O(N^2). + /// One scan answers a whole poll. Each highest-sequence anchor in range is above + /// every candidate below it and above none at or past it. Recomputing per candidate + /// would make walking a backlog of N groups off in one poll cost O(N^2). /// /// Capped because a group can only be late relative to data that would actually be /// served in its place: a subscription ending at a takeover boundary can't jump past /// it, so the groups it still wants aren't stale just because the route ran on. fn live_edge(&self, cap: Option) -> Option { - let mut edge: Option = None; + let mut wall: Option = None; + let mut presentation: Option = None; for slot in self.lookup.values() { let group = &slot.group; if cap.is_some_and(|cap| group.sequence > cap) || group.is_aborted() { continue; } + if wall.is_none_or(|edge| group.sequence > edge.sequence) { + wall = Some(WallEdge { + sequence: group.sequence, + stamp: slot.stamp, + arrived: slot.arrived, + }); + } if let Some(timestamp) = group.timestamp() - && edge.is_none_or(|edge| group.sequence > edge.sequence) + && presentation.is_none_or(|edge| group.sequence > edge.sequence) { - edge = Some(Edge { + presentation = Some(PresentationEdge { sequence: group.sequence, stamp: slot.stamp, timestamp, }); } } - edge + wall.map(|wall| Edge { wall, presentation }) } /// Whether the group at `sequence` has drifted further behind `edge` than `budget` /// tolerates, so a subscriber should skip it rather than hand it over. /// - /// Measured in presentation time: both ends are stamped once, when a group's first - /// frame is created, so a backlog delivered as a burst still reads as its true age - /// rather than as the moment it happened to arrive. A group that has presented - /// nothing is never stale, and neither is one with nothing newer stamped above it: - /// with no timestamp there is nothing to measure, and the frames may simply not have - /// arrived yet. Such a group is bounded by the publisher's retention window instead, - /// which aborts it once it ages out (see [`Self::evict_expired`]). + /// Presentation time distinguishes a backlog delivered as a burst from live content. + /// Wall-clock arrival time backstops it, including for an empty or stalled group that + /// has no first-frame timestamp. Either age exceeding the budget makes the group stale. /// /// The edge must sit strictly above the candidate. The live edge is never late /// against itself, and backfill or the tail of a rewound timeline can carry a high @@ -574,29 +580,36 @@ impl TrackState { let Some(edge) = edge else { return false; }; - if edge.sequence <= sequence { - return false; - } - // The anchor was resolved under an earlier lock, so confirm it is still the - // group it was: an eviction or a re-served sequence in between would otherwise - // convict a candidate on content that is no longer servable. Failing safe - // (delivering) is right, since the next poll resolves a fresh anchor. - let live = self - .lookup - .get(&edge.sequence) - .is_some_and(|slot| slot.stamp == edge.stamp && !slot.group.is_aborted()); - if !live { - return false; - } let Some(slot) = self.lookup.get(&sequence) else { return false; }; - let Some(timestamp) = slot.group.timestamp() else { - return false; - }; - // Newer data below this group's timestamp isn't drift, and a scale that can't - // subtract carries no information either way. - matches!(edge.timestamp.checked_sub(timestamp), Ok(age) if Duration::from(age) > budget) + + // The anchors were resolved under an earlier lock, so confirm each still names + // the same servable incarnation before it convicts a candidate. Failing safe + // (delivering) is right, since the next poll resolves fresh anchors. + let wall_stale = edge.wall.sequence > sequence + && self + .lookup + .get(&edge.wall.sequence) + .is_some_and(|live| live.stamp == edge.wall.stamp && !live.group.is_aborted()) + && edge + .wall + .arrived + .checked_duration_since(slot.arrived) + .is_some_and(|age| age > budget); + + let presentation_stale = edge.presentation.is_some_and(|edge| { + edge.sequence > sequence + && self + .lookup + .get(&edge.sequence) + .is_some_and(|live| live.stamp == edge.stamp && !live.group.is_aborted()) + && slot.group.timestamp().is_some_and( + |timestamp| matches!(edge.timestamp.checked_sub(timestamp), Ok(age) if Duration::from(age) > budget), + ) + }); + + wall_stale || presentation_stale } /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] @@ -782,6 +795,7 @@ impl TrackState { sequence, Slot { group: group.clone(), + arrived: web_async::time::Instant::now(), stamp, }, ); @@ -2582,6 +2596,23 @@ struct Drift { /// from whatever may occupy its sequence by the time a candidate is judged. #[derive(Clone, Copy)] struct Edge { + wall: WallEdge, + presentation: Option, +} + +/// The newest servable group, including an empty group with no media timestamp. +#[derive(Clone, Copy)] +struct WallEdge { + sequence: u64, + /// The slot incarnation this arrival time was read from, so an eviction or a re-served + /// sequence between resolving the anchor and using it is detectable. + stamp: u32, + arrived: web_async::time::Instant, +} + +/// The newest servable group that has presented at least one frame. +#[derive(Clone, Copy)] +struct PresentationEdge { sequence: u64, /// The slot incarnation this timestamp was read from, so an eviction or a re-served /// sequence between resolving the anchor and using it is detectable. @@ -3350,6 +3381,11 @@ mod test { Producer::new(Arc::new(broadcast::Info::default()), name, info) } + /// A bounded replay window for tests whose subject requires every buffered group. + fn replay() -> Subscription { + Subscription::default().with_latency(Latency::max(Duration::from_secs(30))) + } + /// Helper: count live cached groups in state. fn live_groups(state: &TrackState) -> usize { state.lookup.len() @@ -3961,6 +3997,21 @@ mod test { assert_eq!(drain(&mut subscriber), vec![2, 3]); } + #[tokio::test] + async fn wall_clock_expires_an_unstamped_group() { + tokio::time::pause(); + + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + producer.append_group().unwrap(); // seq 0 stalls before its first frame + + tokio::time::advance(Duration::from_secs(1)).await; + append_at(&mut producer, 1000); // seq 1 proves the live feed moved on + + // With the real-time budget, wall-clock age backstops the missing timestamp. + assert_eq!(drain(&mut subscriber), vec![1]); + } + #[test] fn latency_max_bounds_the_budget() { // The publisher only keeps a group around for 500ms, so a subscriber asking to @@ -4026,21 +4077,6 @@ mod test { assert_eq!(group.sequence, 0); } - #[test] - fn a_group_that_has_presented_nothing_is_never_stale() { - let mut producer = track_producer("test", None); - let mut stalled = producer.append_group().unwrap(); - append_at(&mut producer, 10_000); - - // The stalled group has no timestamp, so there is nothing to measure and no - // reason to assume it is old. It is still handed over, and the reader (or the - // publisher's own retention window) decides how long to wait on it. - let mut subscriber = producer.subscribe(None); - assert_eq!(drain(&mut subscriber), vec![0, 1]); - - stalled.finish().unwrap(); - } - #[tokio::test] async fn read_frame_honors_the_budget() { let mut producer = track_producer("test", None); @@ -4128,8 +4164,9 @@ mod test { assert_eq!(stale.frames, 2); } - #[test] - fn a_lower_sequence_is_never_the_live_edge() { + #[tokio::test] + async fn a_lower_sequence_is_never_the_live_edge() { + tokio::time::pause(); let mut producer = track_producer("test", None); // A high timestamp on a lower sequence is not a live edge: backfill served on // demand sits there, and so does the tail of a timeline the publisher rewound. @@ -4607,7 +4644,7 @@ mod test { #[tokio::test] async fn next_group_returns_arrivals_in_order() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); // Seq 3 arrives first, then seq 5. Both should be returned in arrival order. producer.create_group(group::Info { sequence: 3 }).unwrap(); @@ -4657,7 +4694,7 @@ mod test { #[tokio::test] async fn end_at_caps_next_group() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); for s in 0..6 { producer.create_group(group::Info { sequence: s }).unwrap(); @@ -4689,7 +4726,7 @@ mod test { #[tokio::test] async fn end_at_release_drains_cached_groups() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); for s in 0..6 { producer.create_group(group::Info { sequence: s }).unwrap(); @@ -4734,7 +4771,7 @@ mod test { #[tokio::test] async fn end_at_lower_than_cursor_parks_consumer() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); for s in 0..3 { producer.create_group(group::Info { sequence: s }).unwrap(); @@ -4778,7 +4815,7 @@ mod test { #[tokio::test] async fn end_at_toggling_around_late_arrivals() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); consumer.end_at(5); @@ -4824,7 +4861,7 @@ mod test { #[tokio::test] async fn end_at_parks_recv_group() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); for s in 0..3 { producer.create_group(group::Info { sequence: s }).unwrap(); @@ -4864,7 +4901,7 @@ mod test { #[tokio::test] async fn recv_group_serves_arrivals_behind_the_cap() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); consumer.end_at(1); @@ -4994,7 +5031,7 @@ mod test { #[tokio::test] async fn read_frame_returns_single_frame_per_group() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); producer.write_frame(Timestamp::ZERO, b"hello".as_slice()).unwrap(); producer.write_frame(Timestamp::ZERO, b"world".as_slice()).unwrap(); @@ -5061,7 +5098,7 @@ mod test { #[tokio::test] async fn read_frame_discards_rest_of_multi_frame_group() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); // Group 0 has two frames; only the first is returned. let mut g0 = producer.create_group(group::Info { sequence: 0 }).unwrap(); @@ -5550,7 +5587,7 @@ mod test { assert!(pool.used() <= 21_000, "usage hovers near capacity: {}", pool.used()); // A fresh subscriber skips the evicted groups entirely. - let mut subscriber = producer.subscribe(None); + let mut subscriber = producer.subscribe(replay()); assert!(subscriber.assert_group().sequence > 0, "evicted group is not delivered"); } @@ -5994,7 +6031,7 @@ mod test { producer.create_group(2u64.into()).unwrap().finish().unwrap(); producer.create_group(1u64.into()).unwrap().finish().unwrap(); - let mut subscriber = producer.subscribe(None); + let mut subscriber = producer.subscribe(replay()); assert_eq!(subscriber.assert_group().sequence, 0); assert_eq!(subscriber.assert_group().sequence, 2); assert_eq!( @@ -6115,7 +6152,7 @@ mod test { // The backfill serves by sequence, but never in arrival order. assert!(consumer.peek_group(1).is_some()); - let mut subscriber = producer.subscribe(None); + let mut subscriber = producer.subscribe(replay()); assert_eq!(subscriber.assert_group().sequence, 0); assert_eq!(subscriber.assert_group().sequence, 2); subscriber.assert_no_group(); @@ -6288,7 +6325,7 @@ mod test { assert!(consumer.peek_group(2).is_some(), "backfill is cached for later fetches"); // ...but an arrival-order subscriber only sees the live groups. - let mut subscriber = producer.subscribe(None); + let mut subscriber = producer.subscribe(replay()); assert_eq!(subscriber.assert_group().sequence, 5); assert_eq!(subscriber.assert_group().sequence, 6); subscriber.assert_no_group(); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 38cae011737..4ba34417bf5 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -40,7 +40,11 @@ impl Consumer { let name = name.into(); let track = broadcast .track(&name)? - .subscribe(moq_net::track::Subscription::default().with_priority(hang::catalog::PRIORITY.video)) + .subscribe( + moq_net::track::Subscription::default() + .with_priority(hang::catalog::PRIORITY.video) + .with_latency(config.latency), + ) .await?; // The catalog says how the track is framed, and it is not always the legacy // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index dd180c9e764..0717fd46bf5 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -45,8 +45,8 @@ pub struct Config { /// How far playback may drift from the live edge before a stalled group is /// skipped. Defaults to [`Latency::REAL_TIME`](moq_mux::Latency::REAL_TIME) /// (skip aggressively); set [`Latency::max`](moq_mux::Latency::max) to your - /// playout buffer for a softer skip. Forwarded to - /// [`moq_mux::container::Consumer::with_latency`]. + /// playout buffer for a softer skip. Applied to both the initial transport + /// subscription and [`moq_mux::container::Consumer::with_latency`]. pub latency: moq_mux::Latency, /// Ask the decoder to emit frames at this size (both dimensions even) instead /// of the stream's native one. Best effort: a hardware decoder with a From 3ac6bf33fb31b89fae941a4068a396126bfb98ad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 19:20:25 -0700 Subject: [PATCH 07/32] fix(moq-net): expire handed-out stale groups --- rs/moq-bench/src/connection.rs | 8 ++- rs/moq-net/src/latency.rs | 13 ++-- rs/moq-net/src/model/group.rs | 62 ++++++++++++++++++- rs/moq-net/src/model/track.rs | 110 +++++++++++++++++++++++++++++++-- 4 files changed, 178 insertions(+), 15 deletions(-) diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 91f78ca51e6..dbde4408c8d 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -412,6 +412,10 @@ mod tests { } } + fn replay() -> track::Subscription { + track::Subscription::default().with_latency(moq_net::Latency::max(Duration::from_secs(30))) + } + /// A produced group must start with the JSON keyframe describing the rolled /// parameters, followed by `group_size` zeroed payload frames. #[tokio::test] @@ -429,7 +433,7 @@ mod tests { // Advance past one full group (keyframe + 2 payload) into the next. tokio::time::advance(Duration::from_millis(350)).await; - let mut sub = consumer.track(TRACK).unwrap().subscribe(None).await.unwrap(); + let mut sub = consumer.track(TRACK).unwrap().subscribe(replay()).await.unwrap(); let mut group = sub.next_group().await.unwrap().expect("a group"); let keyframe = group.read_frame().await.unwrap().expect("keyframe"); @@ -464,7 +468,7 @@ mod tests { let task = tokio::spawn(produce(0, "bench/test".into(), rolled(10, 4, 0), track, stats.clone())); tokio::time::advance(Duration::from_millis(250)).await; - let mut sub = consumer.track(TRACK).unwrap().subscribe(None).await.unwrap(); + let mut sub = consumer.track(TRACK).unwrap().subscribe(replay()).await.unwrap(); let mut group = sub.next_group().await.unwrap().expect("a group"); // Just the keyframe, then the group ends. diff --git a/rs/moq-net/src/latency.rs b/rs/moq-net/src/latency.rs index f193ec865a4..636078b6b75 100644 --- a/rs/moq-net/src/latency.rs +++ b/rs/moq-net/src/latency.rs @@ -28,14 +28,11 @@ use std::time::Duration; /// /// # How drift is measured /// -/// In presentation time: a group's timestamp against the newest one above it. Both are -/// stamped once, when a group's first frame is created, so a backlog delivered as a burst -/// still reads as its true age rather than as the few seconds it took to arrive. -/// -/// A group that has yet to present a frame is never skipped, and neither is one on a track -/// where nothing newer has: with no timestamp there is nothing to measure, and the frames -/// may simply not have arrived yet. Such a group is bounded by the publisher's retention -/// window instead, which aborts it once it ages out. +/// By both presentation time and wall-clock arrival time, with either age able to expire +/// the group. Presentation time compares a group's first timestamp against the newest one +/// above it, so a backlog delivered as a burst still reads as its true age. Wall-clock time +/// compares when each group entered this hop, which backstops an empty or stalled group that +/// has not supplied a first-frame timestamp. /// /// Protocols whose wire can't carry a timestamp (pre-Lite05 moq-lite, moq-transport without /// the Timestamp property) have their frames stamped on receipt instead, which makes the diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 56e2c6cc2f7..8678558e177 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -693,6 +693,22 @@ impl Producer { // Untagged: a tagged track attaches the egress meter via `with_meter` // when it hands the consumer to a subscriber/fetch. stats: stats::Meter::default(), + expiry: None, + expired: false, + } + } + + /// Register for the first-frame timestamp while the group is still unstamped. + pub(crate) fn poll_timestamp(&self, waiter: &kio::Waiter) -> Poll<()> { + match self.state.poll(waiter, |state| { + if state.timestamp.is_some() || state.fin.is_some() || state.abort.is_some() { + Poll::Ready(()) + } else { + Poll::Pending + } + }) { + Poll::Ready(_) => Poll::Ready(()), + Poll::Pending => Poll::Pending, } } @@ -818,6 +834,18 @@ pub struct Consumer { // Egress payload meter, set by a tagged track via [`Self::with_meter`]. Empty // (no-op) for an untagged group. stats: stats::Meter, + + // Subscriber-specific drift policy. A group can become stale after the track + // hands it out, while its reader is waiting for the first or next frame. + expiry: Option>, + expired: bool, +} + +/// Subscriber-specific policy for expiring a group after it was handed out. +pub(crate) trait Expiry: Send + Sync { + /// Return whether the group is stale, registering `waiter` for anything that + /// could change the answer while it remains live. + fn is_expired(&self, waiter: &kio::Waiter) -> bool; } // `Plain` is the hot path and carries an inline frame prefetch, so boxing it to even the @@ -870,6 +898,8 @@ impl Clone for Consumer { // Inherit the meter without re-counting the group: the original already // counted it when the track handed it out. stats: self.stats.clone(), + expiry: self.expiry.clone(), + expired: self.expired, } } } @@ -904,6 +934,11 @@ impl Consumer { info: self.info, track: self.track, stats: self.stats, + // Each segment keeps its own route-specific expiry policy. Applying the + // head segment's policy to the assembled group would use the wrong edge + // after a takeover. + expiry: None, + expired: false, } } @@ -915,6 +950,20 @@ impl Consumer { self } + /// Keep applying this subscription's drift budget while the group is read. + pub(crate) fn with_expiry(mut self, expiry: Arc) -> Self { + self.expiry = Some(expiry); + self + } + + fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { + if !self.expired && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) { + self.expired = true; + self.stats.stale(self.content()); + } + self.expired + } + /// Whether the group has been aborted (including pool eviction); the abort /// dropped the cached frames, so a held consumer has nothing left to read. /// @@ -998,6 +1047,9 @@ impl Consumer { /// Returns None if the group is finished and the index is out of range, or the cursor /// passed the [`Self::end_at`] cap. pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { + if self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } let stats = self.stats.clone(); match &mut self.inner { ConsumerKind::Plain(plain) => plain.poll_next_frame(waiter, &stats), @@ -1015,6 +1067,9 @@ impl Consumer { /// Read the next frame (timestamp and payload) all at once, without blocking. pub fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { + if self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } let stats = self.stats.clone(); match &mut self.inner { ConsumerKind::Plain(plain) => plain.poll_read_frame(waiter, &stats), @@ -1031,7 +1086,9 @@ impl Consumer { /// Read the next frame (timestamp and payload) all at once. pub async fn read_frame(&mut self) -> Result> { - if let ConsumerKind::Plain(plain) = &mut self.inner { + if self.expiry.is_none() + && let ConsumerKind::Plain(plain) = &mut self.inner + { // Serve from the prefetched batch without building a future or allocating a waker. if !plain.capped() && let Some(frame) = plain.prefetch.pop() @@ -1045,6 +1102,9 @@ impl Consumer { /// Poll for the final number of frames in the group. pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { + if self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } match &mut self.inner { ConsumerKind::Plain(plain) => plain.poll(waiter, |state| state.poll_finished()), ConsumerKind::Spliced(spliced) => spliced.poll_finished(waiter), diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index b0229fd3dc0..4b1749de75f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -13,7 +13,7 @@ //! //! The track is closed with [Error] when all writers or readers are dropped. -use crate::{Error, Result, Timescale, Timestamp, coding}; +use crate::{Error, Latency, Result, Timescale, Timestamp, coding}; use crate::{broadcast, cache, frame, group, stats}; use super::{Datagram, Requests}; @@ -1404,6 +1404,7 @@ impl Producer { let info = self.state.read().info.clone().expect("producer always has info"); let subscription = kio::Producer::new(preferences); register_subscription(self.state.read(), &subscription); + let drift_cap = kio::Producer::new(None); // Hoisted: an inline `read()` guard would live to the end of the struct literal, // deadlocking against the `consume()` below. @@ -1422,6 +1423,7 @@ impl Producer { end_sequence: None, parked: BTreeMap::new(), stale_cap: None, + drift_cap, stale: stats::Content::default(), }), // A producer-side (in-process) subscribe is not egress: stay untagged. @@ -2268,6 +2270,7 @@ impl Subscribing { let info = ready!(state.poll(waiter, |state| state.poll_info())) .map_err(|e| e.abort.clone().unwrap_or(Error::Dropped))??; + let drift_cap = kio::Producer::new(None); Poll::Ready(Ok(Subscriber { name: self.name.clone(), broadcast: self.broadcast.clone(), @@ -2282,6 +2285,7 @@ impl Subscribing { end_sequence: None, parked: BTreeMap::new(), stale_cap: None, + drift_cap, stale: stats::Content::default(), }), stats: self.stats.clone(), @@ -2592,6 +2596,57 @@ struct Drift { edge: Option, } +/// Keeps one handed-out group tied to the subscription whose cursor selected it. +struct GroupExpiry { + state: kio::Consumer, + subscription: kio::Consumer, + cap: kio::Consumer>, + sequence: u64, +} + +impl group::Expiry for GroupExpiry { + fn is_expired(&self, waiter: &kio::Waiter) -> bool { + let mut latency = Latency::default(); + let _ = self.subscription.poll(waiter, |subscription| { + latency = subscription.latency; + Poll::<()>::Pending + }); + + let mut cap = None; + let _ = self.cap.poll(waiter, |current| { + cap = **current; + Poll::<()>::Pending + }); + + let mut expired = false; + let _ = self.state.poll(waiter, |state| { + let budget = clamp_latency(latency, state.latency_bound()).max; + let edge = state.live_edge(cap); + expired = state.is_stale(self.sequence, edge, budget); + + if !expired { + // A first timestamp can change the presentation-time verdict without + // mutating the track. Register on both the candidate and the current + // presentation edge so either first frame wakes the held reader. + if let Some(slot) = state.lookup.get(&self.sequence) { + let _ = slot.group.poll_timestamp(waiter); + } + if let Some(edge) = edge.and_then(|edge| edge.presentation) + && let Some(slot) = state.lookup.get(&edge.sequence) + { + let _ = slot.group.poll_timestamp(waiter); + } + } + + // Register on track changes even though the current answer is known: a + // newer group can move either live edge while this group read is pending. + Poll::<()>::Pending + }); + + expired + } +} + /// The group a poll's drift is measured against, identified well enough to tell it apart /// from whatever may occupy its sequence by the time a candidate is judged. #[derive(Clone, Copy)] @@ -2648,6 +2703,8 @@ struct PlainSubscriber { /// segment), folded into the drift anchor only. Delivery is still bounded by /// `end_sequence`, which stays unset on a segment so its completion is visible. stale_cap: Option, + /// Shared effective cap used by groups after this cursor hands them out. + drift_cap: kio::Producer>, /// Groups the drift budget skipped since the count was last drained. Accumulated /// here rather than metered in place because the handle that owns the stats scope /// is the outer [`Subscriber`], which may be reading this cursor through a @@ -2657,6 +2714,12 @@ struct PlainSubscriber { } impl PlainSubscriber { + fn update_drift_cap(&mut self) { + if let Ok(mut cap) = self.drift_cap.write() { + *cap = servable_cap(self.end_sequence, self.stale_cap); + } + } + // A helper to automatically apply Dropped if the state is closed without an error. fn poll(&self, waiter: &kio::Waiter, f: F) -> Poll> where @@ -2707,6 +2770,16 @@ impl PlainSubscriber { }) } + fn with_expiry(&self, group: group::Consumer) -> group::Consumer { + let sequence = group.sequence; + group.with_expiry(Arc::new(GroupExpiry { + state: self.state.clone(), + subscription: self.subscription.consume(), + cap: self.drift_cap.consume(), + sequence, + })) + } + fn poll_recv_group(&mut self, waiter: &kio::Waiter) -> Poll>> { // An eviction aborts a parked group without touching any cursor this // subscriber polls, so each entry needs a waiter or this poll would never @@ -2772,7 +2845,7 @@ impl PlainSubscriber { self.stale.add(consumer.content()); continue; } - return Poll::Ready(Ok(Some(consumer))); + return Poll::Ready(Ok(Some(self.with_expiry(consumer)))); } } @@ -2803,7 +2876,7 @@ impl PlainSubscriber { self.stale.add(group.content()); continue; } - return Poll::Ready(Ok(Some(group))); + return Poll::Ready(Ok(Some(self.with_expiry(group)))); } } @@ -2906,6 +2979,7 @@ impl Subscriber { pub(crate) fn set_stale_cap(&mut self, cap: Option) { if let SubscriberKind::Plain(plain) = &mut self.inner { plain.stale_cap = cap; + plain.update_drift_cap(); } } @@ -2949,6 +3023,8 @@ impl Subscriber { /// edge and writes the rest off; raise it to read history. [`Self::start_at`] and /// [`Subscription::start`] are filters, not exemptions: backfill needs a budget that /// covers it. [`Consumer::fetch_group`] is the way to ask for one old group outright. + /// The budget remains attached to a returned group: if it stalls while newer data + /// advances, its pending frame read ends with [`Error::Old`]. /// /// Honors the floor set by [`Self::start_at`] and the cap set by [`Self::end_at`]: /// a group beyond the cap is parked (not dropped) and re-offered once the cap rises @@ -3127,7 +3203,10 @@ impl Subscriber { /// consumer's current cursor parks the consumer until the cap is raised. pub fn end_at(&mut self, sequence: impl Into>) { match &mut self.inner { - SubscriberKind::Plain(plain) => plain.end_sequence = sequence.into(), + SubscriberKind::Plain(plain) => { + plain.end_sequence = sequence.into(); + plain.update_drift_cap(); + } SubscriberKind::Spliced(spliced) => spliced.end_at(sequence), } } @@ -4012,6 +4091,29 @@ mod test { assert_eq!(drain(&mut subscriber), vec![1]); } + #[tokio::test] + async fn a_handed_out_group_expires_while_its_first_frame_is_stalled() { + tokio::time::pause(); + + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + producer.append_group().unwrap(); + + let mut stalled = subscriber.recv_group().await.unwrap().expect("stalled group"); + let pending = tokio::spawn(async move { stalled.read_frame().await }); + tokio::task::yield_now().await; + assert!( + !pending.is_finished(), + "the empty live edge still waits for its first frame" + ); + + tokio::time::advance(Duration::from_secs(1)).await; + append_at(&mut producer, 1000); + + let result = pending.await.unwrap(); + assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); + } + #[test] fn latency_max_bounds_the_budget() { // The publisher only keeps a group around for 500ms, so a subscriber asking to From 9ac40859478f45c24d344230515f33cb1caa58f4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 20:19:57 -0700 Subject: [PATCH 08/32] fix(moq-net): preserve latency across route changes --- rs/libmoq/src/test.rs | 13 ++- rs/moq-net/src/model/group.rs | 5 + rs/moq-net/src/model/resume.rs | 142 ++++++++++++++++++++++---- rs/moq-net/src/model/track.rs | 58 ++++++++++- rs/moq-relay/tests/cluster_unknown.rs | 16 ++- 5 files changed, 210 insertions(+), 24 deletions(-) diff --git a/rs/libmoq/src/test.rs b/rs/libmoq/src/test.rs index fa0ada4c863..773a6602cc1 100644 --- a/rs/libmoq/src/test.rs +++ b/rs/libmoq/src/test.rs @@ -1144,12 +1144,23 @@ fn raw_track_publish_consume() { let consume = request_broadcast(origin, path); let frame_cb = Callback::new(); + // This round trip verifies every published frame. Allow the first group to + // finish draining if the second becomes visible while the callback runs. + let subscription = moq_subscription { + priority: 0, + ordered: false, + latency_max_ms: 1_000, + group_start: 0, + group_start_valid: false, + group_end: 0, + group_end_valid: false, + }; let consumer = id(unsafe { moq_consume_track( consume, track_name.as_ptr() as *const c_char, track_name.len(), - std::ptr::null(), + &subscription, Some(channel_callback), frame_cb.ptr, ) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 8678558e177..d5e0fd8ca4c 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -964,6 +964,11 @@ impl Consumer { self.expired } + /// Whether this cursor failed because its subscription latency budget expired. + pub(crate) fn latency_expired(&self) -> bool { + self.expired + } + /// Whether the group has been aborted (including pool eviction); the abort /// dropped the cached frames, so a held consumer has nothing left to read. /// diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 32dd70b4c96..44a1371817d 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -468,6 +468,7 @@ impl Consumer { next_sequence: 0, min_sequence: 0, end_sequence: None, + drift_cap: kio::Producer::new(None), reading: None, } } @@ -667,6 +668,11 @@ fn frames(start: Option, end: Option, sequence: u64) -> Opti Some((start, end)) } +/// The last group a half-open segment can serve, or no bound for the newest segment. +fn last_group(end: Option) -> Option { + end?.before().map(|position| position.group) +} + /// A group assembled from several routes' copies, joined at frame boundaries. /// /// Rather than being fed, it pulls: on every frame it asks the segment list which route @@ -679,6 +685,10 @@ fn frames(start: Option, end: Option, sequence: u64) -> Opti /// replacement can arrive. pub(crate) struct Group { state: kio::Consumer, + /// The logical subscription's live latency budget. + subscription: kio::Consumer, + /// The logical reader's group cap, used only to bound each route's drift anchor. + cap: kio::Consumer>, /// The logical group being assembled. sequence: u64, @@ -700,13 +710,14 @@ pub(crate) struct Group { struct Current { segment: u64, cap: Option, + bound: Option, group: group::Consumer, } /// A route covering one position: the segment id, its track, and the segment's /// exclusive frame bound on the group (outer `None` when the group escapes the /// segment's range entirely, from [`frames`]). -type Covering = (u64, track::Consumer, Option>); +type Covering = (u64, track::Consumer, Option>, Option); impl Clone for Group { fn clone(&self) -> Self { @@ -722,11 +733,14 @@ impl Clone for Group { (group.index() == self.index).then_some(Current { segment: current.segment, cap: current.cap, + bound: current.bound, group, }) }); Self { state: self.state.clone(), + subscription: self.subscription.clone(), + cap: self.cap.clone(), sequence: self.sequence, index: self.index, end: self.end, @@ -737,9 +751,17 @@ impl Clone for Group { } impl Group { - fn new(state: kio::Consumer, sequence: u64, index: u64) -> Self { + fn new( + state: kio::Consumer, + subscription: kio::Consumer, + cap: kio::Consumer>, + sequence: u64, + index: u64, + ) -> Self { Self { state, + subscription, + cap, sequence, index, end: None, @@ -758,12 +780,17 @@ impl Group { /// latched, since the reuse path trusts the latch's alignment and would /// misnumber its frames. The reader re-resolves instead, and the peek path's /// own check buries the copy as lagged. - fn latched(mut self, segment: u64, cap: Option, mut group: group::Consumer) -> Self { + fn latched(mut self, segment: u64, cap: Option, bound: Option, mut group: group::Consumer) -> Self { // The segment bound is exclusive; a group consumer's cap is inclusive. group.end_at(cap.map(|cap| cap.saturating_sub(1))); group.start_at(self.index); if group.index() == self.index { - self.current = Some(Current { segment, cap, group }); + self.current = Some(Current { + segment, + cap, + bound, + group, + }); } self } @@ -833,6 +860,7 @@ impl Group { segment.id, segment.track.clone(), frames(segment.start, segment.end, sequence).map(|(_, end)| end), + last_group(segment.end), ))), Poll::Pending => match terminal { true => Poll::Ready(None), @@ -844,6 +872,7 @@ impl Group { segment.id, segment.track.clone(), frames(segment.start, segment.end, sequence).map(|(_, end)| end), + last_group(segment.end), ))), // No route owns them yet; park unless none is coming. None if terminal => Poll::Ready(None), @@ -873,7 +902,7 @@ impl Group { let sequence = self.sequence; let found = ready!(self.poll_covering(position, dead, waiter)); - let Some((segment, track, Some(cap))) = found else { + let Some((segment, track, Some(cap), bound)) = found else { // No segment covers the position, but a latched copy still drains: // a pruned segment's cursor holds exactly the frames it owned (its // cap was its produced edge), so read it dry before giving up. Once @@ -889,13 +918,13 @@ impl Group { if self .current .as_ref() - .is_some_and(|current| current.segment == segment && current.cap == cap) + .is_some_and(|current| current.segment == segment && current.cap == cap && current.bound == bound) { return Poll::Ready(Ok(true)); } // The route may not have delivered this group yet, so wait on its cache. - let Some(mut group) = ready!(track.poll_peek_group(sequence, waiter)) else { + let Some(group) = ready!(track.poll_peek_group(sequence, waiter)) else { // This route will never have it; fall back to whichever segment replaces it. self.dead = Some((segment, Error::NotFound)); continue; @@ -904,6 +933,7 @@ impl Group { // `start_at` clamps up to the first frame the copy still holds, so landing // higher than asked means this route can't cover the seam after all. Treat it // like a dead copy and wait for one that can. + let mut group = track.guard_group(group, self.subscription.clone(), self.cap.clone(), bound); group.start_at(self.index); if group.index() != self.index { self.dead = Some((segment, Error::Lagged)); @@ -912,7 +942,12 @@ impl Group { // The segment bound is exclusive; a group consumer's cap is inclusive. group.end_at(cap.map(|cap| cap.saturating_sub(1))); - self.current = Some(Current { segment, cap, group }); + self.current = Some(Current { + segment, + cap, + bound, + group, + }); self.dead = None; return Poll::Ready(Ok(true)); } @@ -970,14 +1005,22 @@ impl Group { if !ready!(self.poll_current(waiter))? { return Poll::Ready(Ok(None)); } - let current = self.current.as_mut().expect("resolved above"); - match ready!(current.group.poll_read_frame(waiter)) { + let result = { + let current = self.current.as_mut().expect("resolved above"); + ready!(current.group.poll_read_frame(waiter)) + }; + let latency_expired = self + .current + .as_ref() + .is_some_and(|current| current.group.latency_expired()); + match result { Ok(Some(frame)) => { self.index += 1; return Poll::Ready(Ok(Some(frame))); } Ok(None) if self.roll() => continue, Ok(None) => return Poll::Ready(Ok(None)), + Err(err) if latency_expired => return Poll::Ready(Err(err)), Err(err) => self.bury(err), } } @@ -991,14 +1034,22 @@ impl Group { if !ready!(self.poll_current(waiter))? { return Poll::Ready(Ok(None)); } - let current = self.current.as_mut().expect("resolved above"); - match ready!(current.group.poll_next_frame(waiter)) { + let result = { + let current = self.current.as_mut().expect("resolved above"); + ready!(current.group.poll_next_frame(waiter)) + }; + let latency_expired = self + .current + .as_ref() + .is_some_and(|current| current.group.latency_expired()); + match result { Ok(Some(frame)) => { self.index += 1; return Poll::Ready(Ok(Some(frame))); } Ok(None) if self.roll() => continue, Ok(None) => return Poll::Ready(Ok(None)), + Err(err) if latency_expired => return Poll::Ready(Err(err)), Err(err) => self.bury(err), } } @@ -1029,13 +1080,17 @@ impl Group { }; loop { let dead = self.dead.as_ref().map(|(segment, _)| *segment); - let Some((segment, track, _)) = ready!(self.poll_covering(seam, dead, waiter)) else { + let Some((segment, track, _, bound)) = ready!(self.poll_covering(seam, dead, waiter)) else { return Poll::Ready(Ok(cap)); }; match ready!(track.poll_peek_group(self.sequence, waiter)) { // The continuation's copy declares the count: its own count // already includes the frames it skipped. - Some(mut continuation) => return continuation.poll_finished(waiter), + Some(continuation) => { + let mut continuation = + track.guard_group(continuation, self.subscription.clone(), self.cap.clone(), bound); + return continuation.poll_finished(waiter); + } // This route will never have it; wait for whatever replaces it. None => self.dead = Some((segment, Error::NotFound)), } @@ -1090,17 +1145,16 @@ impl SegmentSub { /// The last group this segment can serve (inclusive), for the underlying read /// cursor. `None` while it is the newest segment. fn last_group(&self) -> Option { - let end = self.end?; // An empty segment would serve nothing, and no switch produces one: every // boundary comes from `resume_position`, which sits at or above the first frame. // `None` here reads as "no cap", which is why it is asserted rather than relied // on; the authoritative filter is `Segment::covers`, which uses the exclusive // bound directly. debug_assert!( - end != Position::default(), + self.end != Some(Position::default()), "a segment cannot end at the start of the track" ); - Some(end.before()?.group) + last_group(self.end) } /// Whether a producer-dropped segment is spent and can be removed. A capped @@ -1161,6 +1215,8 @@ pub struct Subscriber { min_sequence: u64, /// Inclusive cap for [`Self::next_group`], set by [`Self::end_at`]. end_sequence: Option, + /// Shared copy of [`Self::end_sequence`] for groups that outlive this cursor poll. + drift_cap: kio::Producer>, /// The group currently being drained by [`Self::read_frame`]. reading: Option, @@ -1347,7 +1403,14 @@ impl Subscriber { // through the segment list, which forgets this route the moment its // segment is pruned, turning a group the cursor already delivered into an // empty husk. - let spliced = Group::new(self.state.clone(), sequence, 0).latched(seg.id, end, group.clone()); + let spliced = Group::new( + self.state.clone(), + self.prefs.consume(), + self.drift_cap.consume(), + sequence, + 0, + ) + .latched(seg.id, end, seg.last_group(), group.clone()); Some(group.into_spliced(spliced)) } @@ -1745,6 +1808,9 @@ impl Subscriber { /// re-offers it. pub fn end_at(&mut self, sequence: impl Into>) { self.end_sequence = sequence.into(); + if let Ok(mut cap) = self.drift_cap.write() { + *cap = self.end_sequence; + } // The cap bounds each segment's drift anchor as well as this reader's own // delivery: a segment must not measure against groups this cap hides. let end_sequence = self.end_sequence; @@ -2485,6 +2551,46 @@ mod test { recv_pending(&mut sub); } + #[tokio::test] + async fn a_replacement_copy_keeps_the_handed_out_group_latency_budget() { + tokio::time::pause(); + + let (mut track_a, consumer_a) = track_pair("a"); + let (mut track_b, consumer_b) = track_pair("b"); + + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + + let mut head = track_a.create_group(group::Info { sequence: 0 }).unwrap(); + head.write_frame(Timestamp::ZERO, b"a0".to_vec()).unwrap(); + let mut reading = sub.recv_group().now_or_never().unwrap().unwrap().expect("head group"); + assert_eq!(read(&mut reading), b"a0"); + + producer.takeover(&consumer_b).unwrap(); + track_a.abort(Error::Dropped).unwrap(); + recv_pending(&mut sub); + + let mut continuation = track_b.create_group(group::Info { sequence: 0 }).unwrap(); + continuation.start_at(1).unwrap(); + continuation.write_frame(Timestamp::ZERO, b"b1".to_vec()).unwrap(); + assert_eq!(read(&mut reading), b"b1"); + + assert!( + reading.read_frame().now_or_never().is_none(), + "the replacement group is still the live edge" + ); + + tokio::time::advance(Duration::from_secs(1)).await; + write_group_at(&mut track_b, 1, "edge", Duration::from_secs(1)); + + let result = reading + .read_frame() + .now_or_never() + .expect("the newer group changes the verdict"); + assert!(matches!(result, Err(Error::Old)), "the replacement expires: {result:?}"); + } + /// A replacement that serves the whole group instead of the requested tail still /// splices cleanly: the reader picks up at the seam and never sees the frames it /// already has. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4b1749de75f..b804da81bff 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2025,6 +2025,27 @@ impl Consumer { } } + /// Attach a subscription's drift policy to a cached group resolved outside its cursor. + pub(crate) fn guard_group( + &self, + group: group::Consumer, + subscription: kio::Consumer, + cap: kio::Consumer>, + bound: Option, + ) -> group::Consumer { + let ConsumerKind::Plain(state) = &self.inner else { + return group; + }; + let sequence = group.sequence; + group.with_expiry(Arc::new(GroupExpiry { + state: state.clone(), + subscription, + cap, + bound, + sequence, + })) + } + /// Poll for a cached group by sequence, parking `waiter` until it lands. /// /// `Ready(None)` once it can never arrive: the track ended below the sequence, or @@ -2601,6 +2622,7 @@ struct GroupExpiry { state: kio::Consumer, subscription: kio::Consumer, cap: kio::Consumer>, + bound: Option, sequence: u64, } @@ -2617,6 +2639,7 @@ impl group::Expiry for GroupExpiry { cap = **current; Poll::<()>::Pending }); + let cap = super::subscription::min_some(cap, self.bound); let mut expired = false; let _ = self.state.poll(waiter, |state| { @@ -2752,7 +2775,11 @@ impl PlainSubscriber { /// [`Poll::Ready`]; the track ending surfaces as the error the caller was going to /// get anyway. fn poll_drift(&self, waiter: &kio::Waiter) -> Poll> { - let latency = self.subscription.read().latency; + let mut latency = Latency::default(); + let _ = self.subscription.poll(waiter, |subscription| { + latency = subscription.latency; + Poll::<()>::Pending + }); let cap = servable_cap(self.end_sequence, self.stale_cap); self.poll(waiter, move |state| { Poll::Ready(Ok(Drift { @@ -2776,6 +2803,7 @@ impl PlainSubscriber { state: self.state.clone(), subscription: self.subscription.consume(), cap: self.drift_cap.consume(), + bound: None, sequence, })) } @@ -4114,6 +4142,34 @@ mod test { assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); } + #[tokio::test] + async fn widening_latency_wakes_a_pending_frame_read() { + tokio::time::pause(); + + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + let control = subscriber.control(); + + append_at(&mut producer, 0); + tokio::time::advance(Duration::from_secs(1)).await; + let _edge = producer.append_group().unwrap(); + + let pending = tokio::spawn(async move { subscriber.read_frame().await }); + tokio::task::yield_now().await; + assert!(!pending.is_finished(), "the real-time budget skips the old frame"); + + control + .update(Subscription::default().with_latency(Latency::max(Duration::from_secs(2)))) + .unwrap(); + + let frame = pending + .await + .unwrap() + .unwrap() + .expect("the wider budget admits the frame"); + assert_eq!(frame.payload, bytes::Bytes::from_static(b"x")); + } + #[test] fn latency_max_bounds_the_budget() { // The publisher only keeps a group around for 500ms, so a subscriber asking to diff --git a/rs/moq-relay/tests/cluster_unknown.rs b/rs/moq-relay/tests/cluster_unknown.rs index 69634f38329..e20c5ef7fe4 100644 --- a/rs/moq-relay/tests/cluster_unknown.rs +++ b/rs/moq-relay/tests/cluster_unknown.rs @@ -148,10 +148,18 @@ async fn read_first_frame(port: u16) -> Result, String> { .map_err(|_| "announced_broadcast timed out".to_string())? .ok_or_else(|| "origin closed before the broadcast was announced".to_string())?; - let mut track = tokio::time::timeout(TIMEOUT, broadcast.track("video").expect("track handle").subscribe(None)) - .await - .map_err(|_| "subscribe timed out".to_string())? - .map_err(|err| format!("subscribe failed: {err}"))?; + // This verifies route propagation rather than real-time backlog skipping. Give + // every hop enough tolerance for the next 100ms group to arrive while the + // selected group's frame is still crossing the redundant mesh. + let subscription = + moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(Duration::from_secs(1))); + let mut track = tokio::time::timeout( + TIMEOUT, + broadcast.track("video").expect("track handle").subscribe(subscription), + ) + .await + .map_err(|_| "subscribe timed out".to_string())? + .map_err(|err| format!("subscribe failed: {err}"))?; let mut group = tokio::time::timeout(TIMEOUT, track.recv_group()) .await From 65e228033fabbe8ff1b1eebe0c34790040344495 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 20:59:12 -0700 Subject: [PATCH 09/32] fix(moq-net): settle handed-out group expiry --- rs/moq-net/src/model/group.rs | 78 ++++++++++++++++++++++++++++++++-- rs/moq-net/src/model/origin.rs | 59 +++++++++++++++++++++++++ rs/moq-net/src/model/resume.rs | 15 +++++++ 3 files changed, 149 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index d5e0fd8ca4c..99dff4d0588 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -21,6 +21,7 @@ use crate::{Timescale, stats, track}; use std::collections::VecDeque; use std::mem::MaybeUninit; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::task::{Poll, ready}; use crate::{Error, IntoBytes, Result, Timestamp}; @@ -150,6 +151,36 @@ impl GroupState { } } + /// Content in the half-open frame range that is still cached here. + fn content_range(&self, start: usize, end: usize) -> stats::Content { + let start = start.max(self.offset); + let end = end.min(self.next_index); + if start >= end { + return stats::Content::default(); + } + + let local_start = start.saturating_sub(self.offset).min(self.frames.len()); + let local_end = end.saturating_sub(self.offset).min(self.frames.len()); + let mut bytes = self + .frames + .range(local_start..local_end) + .map(|frame| frame.payload.len() as u64) + .sum(); + if start <= self.committed + && self.committed < end + && let Some(partial) = &self.partial + { + bytes += partial.buf.capacity() as u64; + } + + stats::Content { + bytes, + frames: (end - start) as u64, + groups: 0, + datagrams: 0, + } + } + /// Resolve the source for the frame at `index`: a completed frame (whole) or the /// in-flight tail (streamed). Used by [`Consumer::poll_next_frame`]. fn poll_frame_source(&self, index: usize) -> Poll>> { @@ -693,8 +724,10 @@ impl Producer { // Untagged: a tagged track attaches the egress meter via `with_meter` // when it hands the consumer to a subscriber/fetch. stats: stats::Meter::default(), + stale_stats: stats::Meter::default(), expiry: None, expired: false, + stale_counted: Arc::default(), } } @@ -834,11 +867,17 @@ pub struct Consumer { // Egress payload meter, set by a tagged track via [`Self::with_meter`]. Empty // (no-op) for an untagged group. stats: stats::Meter, + // The meter that owns unread content discarded by expiry. Route-specific + // cursors inside a spliced group inherit this without metering delivery twice. + stale_stats: stats::Meter, // Subscriber-specific drift policy. A group can become stale after the track // hands it out, while its reader is waiting for the first or next frame. expiry: Option>, expired: bool, + // Cloned cursors are parallel views of one handed-out delivery. Whichever + // observes expiry first records its unread tail; the others must not repeat it. + stale_counted: Arc, } /// Subscriber-specific policy for expiring a group after it was handed out. @@ -898,8 +937,10 @@ impl Clone for Consumer { // Inherit the meter without re-counting the group: the original already // counted it when the track handed it out. stats: self.stats.clone(), + stale_stats: self.stale_stats.clone(), expiry: self.expiry.clone(), expired: self.expired, + stale_counted: self.stale_counted.clone(), } } } @@ -926,19 +967,31 @@ impl Consumer { } } + /// Content not already attributed as delivered by this handed-out cursor. + fn unread_content(&self) -> stats::Content { + match &self.inner { + ConsumerKind::Plain(plain) => plain.unread_content(), + // Each route-specific plain cursor enforces expiry inside a spliced group. + ConsumerKind::Spliced(_) => stats::Content::default(), + } + } + /// Rebuild this consumer as the head of a group assembled across route changes, /// keeping the group's identity and its track's properties. See [`super::resume`]. - pub(crate) fn into_spliced(self, spliced: super::resume::Group) -> Self { + pub(crate) fn into_spliced(self, mut spliced: super::resume::Group) -> Self { + spliced.set_stale_meter(self.stale_stats.clone()); Self { inner: ConsumerKind::Spliced(Box::new(spliced)), info: self.info, track: self.track, stats: self.stats, + stale_stats: self.stale_stats, // Each segment keeps its own route-specific expiry policy. Applying the // head segment's policy to the assembled group would use the wrong edge // after a takeover. expiry: None, expired: false, + stale_counted: self.stale_counted, } } @@ -946,10 +999,19 @@ impl Consumer { /// Called by a tagged track when it hands the consumer to a subscriber or fetch. pub(crate) fn with_meter(mut self, meter: stats::Meter) -> Self { meter.group(); - self.stats = meter; + self.stats = meter.clone(); + self.set_stale_meter(meter); self } + /// Attach only the meter that owns content discarded by expiry. + pub(crate) fn set_stale_meter(&mut self, meter: stats::Meter) { + if let ConsumerKind::Spliced(spliced) = &mut self.inner { + spliced.set_stale_meter(meter.clone()); + } + self.stale_stats = meter; + } + /// Keep applying this subscription's drift budget while the group is read. pub(crate) fn with_expiry(mut self, expiry: Arc) -> Self { self.expiry = Some(expiry); @@ -959,7 +1021,9 @@ impl Consumer { fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { if !self.expired && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) { self.expired = true; - self.stats.stale(self.content()); + if !self.stale_counted.swap(true, Ordering::Relaxed) { + self.stale_stats.stale(self.unread_content()); + } } self.expired } @@ -1123,6 +1187,14 @@ impl Consumer { } impl Plain { + /// Content this cursor has neither returned nor already counted in a prefetch batch. + fn unread_content(&self) -> stats::Content { + let prefetched = self.prefetch.buffered().0 as usize; + let start = self.index.saturating_add(prefetched); + let end = self.end.map_or(usize::MAX, |end| end.saturating_add(1)); + self.state.read().content_range(start, end) + } + // A helper to automatically apply Dropped if the state is closed without an error. fn poll(&self, waiter: &kio::Waiter, f: F) -> Poll> where diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ba8112b3833..d9f892694c1 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3902,6 +3902,65 @@ mod tests { assert_eq!(entry.subscriber.groups, 3); } + /// Expiry after handoff writes off only the unread tail. The group itself and + /// the frame already returned stay solely in the delivered counters, and a + /// cloned cursor cannot report the same tail again. + #[tokio::test] + async fn test_stats_handed_out_expiry_counts_the_unread_tail_once() { + use crate::Timestamp; + use crate::stats::{Config, Registry, Tier}; + use bytes::Bytes; + + tokio::time::pause(); + + let registry = Registry::new(Config::new()); + let ctx = registry.tier(Tier::default()).session("acme"); + + let origin = Origin::random().produce(); + let ingress = origin.clone().with_stats(ctx.clone()); + let egress = origin.consume().with_stats(ctx.clone()); + + let mut announced = egress.announced(); + let source = ingress.create_broadcast("demo", announce()).unwrap(); + let mut dynamic = source.dynamic(); + settle().await; + settle().await; + + let broadcast = announced.next().await.unwrap().broadcast.unwrap(); + let subscribing = broadcast.track("video").unwrap().subscribe(None); + let mut producer = accept_track(&mut dynamic, "video").await; + settle().await; + let mut sub = subscribing.await.unwrap(); + + let mut group = producer.append_group().unwrap(); + for payload in [b"aa", b"bb", b"cc"] { + group.write_frame(Timestamp::ZERO, Bytes::from_static(payload)).unwrap(); + } + group.finish().unwrap(); + + let mut reading = sub.recv_group().await.unwrap().expect("first group"); + let mut first = reading.next_frame().await.unwrap().expect("first frame"); + assert_eq!(first.read_all().await.unwrap(), Bytes::from_static(b"aa")); + let mut clone = reading.clone(); + + let mut edge = producer.append_group().unwrap(); + edge.write_frame(Timestamp::from_millis(1000).unwrap(), Bytes::from_static(b"edge")) + .unwrap(); + edge.finish().unwrap(); + + assert!(matches!(reading.next_frame().await, Err(crate::Error::Old))); + assert!(matches!(clone.next_frame().await, Err(crate::Error::Old))); + + let report = registry.report(); + let entry = report.traffic.iter().find(|e| e.path.as_str() == "demo").unwrap(); + assert_eq!(entry.publisher.groups, 1, "the handed-out group was delivered"); + assert_eq!(entry.publisher.frames, 1, "the returned frame was delivered"); + assert_eq!(entry.publisher.bytes, 2, "the returned payload was delivered"); + assert_eq!(entry.publisher.stale.groups, 0, "the group is not counted twice"); + assert_eq!(entry.publisher.stale.frames, 2, "only the unread frames are stale"); + assert_eq!(entry.publisher.stale.bytes, 4, "only the unread payload is stale"); + } + /// `Subscriber::read_frame` collapses a group to its first frame. The paths it /// delegates to (plain and spliced) build their own *unmetered* group consumers, /// so the wrapper is the only place that can attribute the read: exactly one diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 44a1371817d..df074a531e3 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -705,6 +705,9 @@ pub(crate) struct Group { /// The segment whose copy died under us, and why. dead: Option<(u64, Error)>, + + /// The tagged logical subscriber's meter for route-copy expiry only. + stale_stats: crate::stats::Meter, } struct Current { @@ -746,6 +749,7 @@ impl Clone for Group { end: self.end, current, dead: self.dead.clone(), + stale_stats: self.stale_stats.clone(), } } } @@ -767,7 +771,15 @@ impl Group { end: None, current: None, dead: None, + stale_stats: Default::default(), + } + } + + pub(crate) fn set_stale_meter(&mut self, meter: crate::stats::Meter) { + if let Some(current) = &mut self.current { + current.group.set_stale_meter(meter.clone()); } + self.stale_stats = meter; } /// Pre-latch the delivering route's own copy, so the payload it already @@ -784,6 +796,7 @@ impl Group { // The segment bound is exclusive; a group consumer's cap is inclusive. group.end_at(cap.map(|cap| cap.saturating_sub(1))); group.start_at(self.index); + group.set_stale_meter(self.stale_stats.clone()); if group.index() == self.index { self.current = Some(Current { segment, @@ -934,6 +947,7 @@ impl Group { // higher than asked means this route can't cover the seam after all. Treat it // like a dead copy and wait for one that can. let mut group = track.guard_group(group, self.subscription.clone(), self.cap.clone(), bound); + group.set_stale_meter(self.stale_stats.clone()); group.start_at(self.index); if group.index() != self.index { self.dead = Some((segment, Error::Lagged)); @@ -1089,6 +1103,7 @@ impl Group { Some(continuation) => { let mut continuation = track.guard_group(continuation, self.subscription.clone(), self.cap.clone(), bound); + continuation.set_stale_meter(self.stale_stats.clone()); return continuation.poll_finished(waiter); } // This route will never have it; wait for whatever replaces it. From 30f63bbb2f5894136524c1db2199a352cdd2346c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 22:11:35 -0700 Subject: [PATCH 10/32] fix(moq-net): close zero-latency gaps --- rs/moq-net/src/lite/publisher.rs | 3 ++- rs/moq-net/src/model/origin.rs | 10 ++++++++-- rs/moq-net/src/model/resume.rs | 4 ++-- rs/moq-net/src/model/track.rs | 27 ++++++++++++++++++++------- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index e838518cb19..8fa770f671e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1718,7 +1718,8 @@ mod test { use futures::FutureExt; let mut producer = track_producer("test"); - let mut subscriber = producer.subscribe(None); + let mut subscriber = producer + .subscribe(track::Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(5)))); producer.create_group(group::Info { sequence: 2 }).unwrap(); match recv_next(&mut subscriber, false, false).await.unwrap() { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index d9f892694c1..de0cfcebc0f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4403,7 +4403,10 @@ mod tests { announced.assert_next_wait(); // Subscribing dispatches the track to the best source (A). - let subscribing = broadcast.track("video").unwrap().subscribe(None); + let subscribing = broadcast + .track("video") + .unwrap() + .subscribe(track::Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(5)))); let mut producer = accept_track(&mut dynamic_a, "video").await; settle().await; dynamic_b.assert_no_request(); @@ -4786,7 +4789,10 @@ mod tests { let broadcast = consumer.request_broadcast("test").await.unwrap(); announced.assert_next_some("test"); - let subscribing = broadcast.track("video").unwrap().subscribe(None); + let subscribing = broadcast + .track("video") + .unwrap() + .subscribe(track::Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(5)))); let mut producer_a = accept_track(&mut dynamic_a, "video").await; settle().await; let mut sub = subscribing.await.unwrap(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index df074a531e3..ea70b1a31ad 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2856,7 +2856,7 @@ mod test { let mut producer = Producer::new(); producer.takeover(&consumer_a).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); let mut group = track_a.create_group(group::Info { sequence: 0 }).unwrap(); group.write_frame(Timestamp::ZERO, b"a0".to_vec()).unwrap(); @@ -3431,7 +3431,7 @@ mod test { #[tokio::test] async fn late_group_drains_from_a_pruned_cursor() { let mut producer = Producer::new(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); // A delivers group 1 first; group 0 is still in flight when A is outranked // and enough failovers prune its segment. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index b804da81bff..51296765413 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -592,11 +592,12 @@ impl TrackState { .lookup .get(&edge.wall.sequence) .is_some_and(|live| live.stamp == edge.wall.stamp && !live.group.is_aborted()) - && edge - .wall - .arrived - .checked_duration_since(slot.arrived) - .is_some_and(|age| age > budget); + && (budget.is_zero() + || edge + .wall + .arrived + .checked_duration_since(slot.arrived) + .is_some_and(|age| age > budget)); let presentation_stale = edge.presentation.is_some_and(|edge| { edge.sequence > sequence @@ -4221,6 +4222,18 @@ mod test { assert!(subscriber.next_group().now_or_never().is_none()); } + #[tokio::test] + async fn real_time_skips_older_sequences_with_equal_ages() { + tokio::time::pause(); + + let mut producer = track_producer("test", None); + append_at(&mut producer, 0); + append_at(&mut producer, 0); + + let mut subscriber = producer.subscribe(None); + assert_eq!(drain(&mut subscriber), vec![1]); + } + #[test] fn fetch_ignores_the_budget() { let mut producer = track_producer("test", None); @@ -4342,7 +4355,7 @@ mod test { .unwrap(); rewound.finish().unwrap(); - let mut subscriber = producer.subscribe(None); + let mut subscriber = producer.subscribe(replay()); assert_eq!(drain(&mut subscriber), vec![0, 1]); } @@ -4828,7 +4841,7 @@ mod test { #[tokio::test] async fn next_group_and_recv_group_use_independent_cursors() { let mut producer = track_producer("test", None); - let mut consumer = producer.subscribe(None); + let mut consumer = producer.subscribe(replay()); // Out-of-order arrivals: seq 5 first, then seq 3. producer.create_group(group::Info { sequence: 5 }).unwrap(); From a069d9587b256668e0579431e7b45feb749ad859 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 22:58:45 -0700 Subject: [PATCH 11/32] fix(moq-net): expire in-flight frame reads --- rs/moq-net/src/model/frame.rs | 80 ++++++++++++++++++++++++++++++++++- rs/moq-net/src/model/group.rs | 31 +++++++++++--- rs/moq-net/src/model/track.rs | 36 ++++++++++++++++ 3 files changed, 139 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index ea4d8a36c7e..ad4f54ac4c4 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -5,9 +5,10 @@ //! through [`Producer`], which borrows its parent [`group::Producer`] exclusively so //! the borrow checker enforces that only one frame is open at a time. A [`Consumer`] //! reads one frame, sharing the group's channel rather than a per-frame one. +use std::ops::Range; use std::sync::Arc; use std::sync::OnceLock; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::task::{Poll, ready}; use bytes::Bytes; @@ -414,6 +415,38 @@ pub(crate) enum Source { Partial(FrameBuf), } +/// Subscriber expiry state carried across the group-to-frame handoff. +#[derive(Clone)] +pub(crate) struct Expiry { + policy: Arc, + stale_stats: stats::Meter, + stale_counted: Arc, + tail: Range, + count_payload: bool, +} + +impl Expiry { + pub(crate) fn new( + policy: Arc, + stale_stats: stats::Meter, + stale_counted: Arc, + ) -> Self { + Self { + policy, + stale_stats, + stale_counted, + tail: 0..0, + count_payload: false, + } + } + + pub(crate) fn for_frame(mut self, tail: Range, count_payload: bool) -> Self { + self.tail = tail; + self.count_payload = count_payload; + self + } +} + /// Reads one frame's payload, streaming as bytes arrive for the in-flight tail. /// /// Owns a handle to the parent group's channel (not a per-frame one), so a group with @@ -431,6 +464,9 @@ pub struct Consumer { // from the prefetch batch, whose bytes were already counted at fill), so chunks // bump `bytes` exactly once. Empty (no-op) otherwise. stats: stats::Meter, + // The parent subscription can expire after this frame handle is returned. + expiry: Option, + expired: bool, } impl std::ops::Deref for Consumer { @@ -449,6 +485,8 @@ impl Consumer { source, read_idx: 0, stats: stats::Meter::default(), + expiry: None, + expired: false, } } @@ -459,10 +497,47 @@ impl Consumer { self } + pub(crate) fn with_expiry(mut self, expiry: Expiry) -> Self { + self.expiry = Some(expiry); + self + } + + fn size(&self) -> usize { + match &self.source { + Source::Complete(bytes) => bytes.len(), + Source::Partial(_) => self.info.size as usize, + } + } + + fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { + if self.expired || self.read_idx >= self.size() { + return self.expired; + } + let Some(expiry) = &self.expiry else { + return false; + }; + if !expiry.policy.is_expired(waiter) { + return false; + } + + self.expired = true; + if !expiry.stale_counted.swap(true, Ordering::Relaxed) { + let mut stale = self.state.read().content_range(expiry.tail.start, expiry.tail.end); + if expiry.count_payload { + stale.bytes += self.size().saturating_sub(self.read_idx) as u64; + } + expiry.stale_stats.stale(stale); + } + true + } + /// Poll for the next chunk of bytes since the last read. /// /// Returns `None` once the frame is finished and all bytes have been consumed. pub fn poll_read_chunk(&mut self, waiter: &kio::Waiter) -> Poll>> { + if self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } match &self.source { Source::Complete(bytes) => { if self.read_idx >= bytes.len() { @@ -513,6 +588,9 @@ impl Consumer { /// Poll for all remaining bytes, resolving once the frame is finished. pub fn poll_read_all(&mut self, waiter: &kio::Waiter) -> Poll> { + if self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } match &self.source { Source::Complete(bytes) => { let out = bytes.slice(self.read_idx..); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 99dff4d0588..2e1aa840533 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -152,7 +152,7 @@ impl GroupState { } /// Content in the half-open frame range that is still cached here. - fn content_range(&self, start: usize, end: usize) -> stats::Content { + pub(crate) fn content_range(&self, start: usize, end: usize) -> stats::Content { let start = start.max(self.offset); let end = end.min(self.next_index); if start >= end { @@ -1120,8 +1120,12 @@ impl Consumer { return Poll::Ready(Err(Error::Old)); } let stats = self.stats.clone(); + let expiry = self + .expiry + .as_ref() + .map(|policy| frame::Expiry::new(policy.clone(), self.stale_stats.clone(), self.stale_counted.clone())); match &mut self.inner { - ConsumerKind::Plain(plain) => plain.poll_next_frame(waiter, &stats), + ConsumerKind::Plain(plain) => plain.poll_next_frame(waiter, &stats, expiry), ConsumerKind::Spliced(spliced) => { // The per-route copies underneath are untagged, so meter the spliced // stream here: it is the one the subscriber actually reads. @@ -1223,22 +1227,33 @@ impl Plain { self.prefetch = Prefetch::default(); } - fn poll_next_frame(&mut self, waiter: &kio::Waiter, stats: &stats::Meter) -> Poll>> { + fn poll_next_frame( + &mut self, + waiter: &kio::Waiter, + stats: &stats::Meter, + expiry: Option, + ) -> Poll>> { if self.capped() { return Poll::Ready(Ok(None)); } + let end = self.end.map_or(usize::MAX, |end| end.saturating_add(1)); // Hand out any frames a prior read_frame prefetched before touching the tail. // Their bytes were already counted at the batch fill, so the frame::Consumer // carries no meter. if let Some(frame) = self.prefetch.pop() { self.index += 1; + let tail = self.index.saturating_add(self.prefetch.buffered().0 as usize)..end; let info = frame::Info { size: frame.payload.len() as u64, timestamp: frame.timestamp, }; let source = frame::Source::Complete(frame.payload); - return Poll::Ready(Ok(Some(frame::Consumer::new(self.state.clone(), info, source)))); + let frame = frame::Consumer::new(self.state.clone(), info, source); + return Poll::Ready(Ok(Some(match expiry { + Some(expiry) => frame.with_expiry(expiry.for_frame(tail, false)), + None => frame, + }))); } let index = self.index; @@ -1250,9 +1265,11 @@ impl Plain { // A direct read (not prefetched): count the frame here; the frame::Consumer // counts its bytes per chunk as they're read out. stats.frames(1); - Poll::Ready(Ok(Some( - frame::Consumer::new(self.state.clone(), info, source).with_meter(stats.clone()), - ))) + let frame = frame::Consumer::new(self.state.clone(), info, source).with_meter(stats.clone()); + Poll::Ready(Ok(Some(match expiry { + Some(expiry) => frame.with_expiry(expiry.for_frame(self.index..end, true)), + None => frame, + }))) } fn poll_read_frame(&mut self, waiter: &kio::Waiter, stats: &stats::Meter) -> Poll>> { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 51296765413..2ec0933cc6c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4143,6 +4143,42 @@ mod test { assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); } + #[tokio::test] + async fn a_handed_out_partial_frame_expires_while_its_payload_is_stalled() { + tokio::time::pause(); + + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + let mut source = producer.append_group().unwrap(); + let mut writing = source + .create_frame(frame::Info { + size: 6, + timestamp: Timestamp::ZERO, + }) + .unwrap(); + writing.write(bytes::Bytes::from_static(b"old")).unwrap(); + + let mut group = subscriber.recv_group().await.unwrap().expect("partial group"); + let mut frame = group.next_frame().await.unwrap().expect("partial frame"); + assert_eq!( + frame.read_chunk().await.unwrap(), + Some(bytes::Bytes::from_static(b"old")) + ); + let pending = tokio::spawn(async move { frame.read_chunk().await }); + tokio::task::yield_now().await; + assert!(!pending.is_finished(), "the partial payload is still stalled"); + + tokio::time::advance(Duration::from_secs(1)).await; + append_at(&mut producer, 1000); + + let result = pending.await.unwrap(); + assert!( + matches!(result, Err(Error::Old)), + "the in-flight frame expires: {result:?}" + ); + writing.abort(Error::Cancel).unwrap(); + } + #[tokio::test] async fn widening_latency_wakes_a_pending_frame_read() { tokio::time::pause(); From 16a9bf431b223338dadbe1ca3608a22384c5bdde Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 23:37:52 -0700 Subject: [PATCH 12/32] fix(moq-net): preserve protocol latency semantics --- rs/moq-net/src/ietf/publisher.rs | 46 ++++++++++++++++++--- rs/moq-net/src/model/origin.rs | 70 ++++++++++++++++++++++++++++++++ rs/moq-net/src/model/resume.rs | 17 +++++++- rs/moq-net/src/model/track.rs | 5 ++- 4 files changed, 131 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 5be62aba061..5f1d2dcf2a4 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -1,7 +1,8 @@ -use crate::{frame, group, origin, track}; +use crate::{Latency, frame, group, origin, track}; use std::{ collections::HashMap, task::{Poll, ready}, + time::Duration, }; use web_transport_trait::poll::SendStream as _; @@ -16,6 +17,16 @@ use crate::{ use super::{Message, Version, cluster, peer}; +/// Build the serving-side subscription for a peer whose wire protocol carries no +/// latency preference. The receiver applies its own budget after the transfer. +fn serving_subscription(subscriber_priority: u8) -> Subscription { + Subscription { + priority: super::priority::from_wire(subscriber_priority), + latency: Latency::max(Duration::MAX), + ..Default::default() + } +} + /// A broadcast whose route table is watched for changes in what we advertise: the /// namespace becoming (un)advertisable, or its path or cost moving. struct Watched { @@ -537,10 +548,9 @@ impl Publisher { } }; - let subscription = Subscription { - priority: super::priority::from_wire(msg.subscriber_priority), - ..Default::default() - }; + // moq-transport has no subscriber latency parameter. Keep everything the + // producer retained and let the receiving subscriber enforce its own budget. + let subscription = serving_subscription(msg.subscriber_priority); let track = match async { broadcast.track(&msg.track_name)?.subscribe(subscription).await }.await { Ok(track) => track, @@ -1736,6 +1746,7 @@ mod tests { use super::*; use crate::lite::test_transport::SinkSession; use crate::model::ProduceTest; + use futures::FutureExt; async fn settle() { tokio::time::sleep(std::time::Duration::from_millis(1)).await; @@ -1757,6 +1768,31 @@ mod tests { slot } + /// moq-transport cannot carry the receiver's latency budget, so the serving + /// subscription must preserve everything the producer still retains. + #[test] + fn serving_subscription_keeps_retained_backlog() { + let mut producer = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", None); + for millis in [0, 1000] { + let mut group = producer.append_group().unwrap(); + group + .write_frame(crate::Timestamp::from_millis(millis).unwrap(), b"frame".as_slice()) + .unwrap(); + group.finish().unwrap(); + } + + let mut subscriber = producer.subscribe(serving_subscription(128)); + for sequence in [0, 1] { + let group = subscriber + .recv_group() + .now_or_never() + .expect("retained group should be ready") + .unwrap() + .expect("track should remain open"); + assert_eq!(group.sequence, sequence); + } + } + /// A peer that requires solicitation, which is what hands the advertisements to the /// SUBSCRIBE_NAMESPACE stream. fn requires_solicitation() -> peer::PeerSetup { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index de0cfcebc0f..14403c7f932 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4015,6 +4015,76 @@ mod tests { ); } + /// A spliced `read_frame` holds its group internally while a partial frame fills. + /// If that group expires, its unread content still belongs to the logical + /// subscriber's stale counters even though the helper skips to the live edge. + #[tokio::test] + async fn test_stats_spliced_read_frame_counts_internal_expiry() { + use crate::Timestamp; + use crate::frame; + use crate::stats::{Config, Registry, Tier}; + use bytes::Bytes; + + tokio::time::pause(); + + let registry = Registry::new(Config::new()); + let ctx = registry.tier(Tier::default()).session("acme"); + + let origin = Origin::random().produce(); + let ingress = origin.clone().with_stats(ctx.clone()); + let egress = origin.consume().with_stats(ctx.clone()); + + let mut announced = egress.announced(); + let source = ingress.create_broadcast("demo", announce()).unwrap(); + let mut dynamic = source.dynamic(); + settle().await; + settle().await; + + let broadcast = announced.next().await.unwrap().broadcast.unwrap(); + let subscribing = broadcast.track("video").unwrap().subscribe(None); + let mut producer = accept_track(&mut dynamic, "video").await; + settle().await; + let mut subscriber = subscribing.await.unwrap(); + + let mut old = producer.append_group().unwrap(); + let mut writing = old + .create_frame(frame::Info { + size: 6, + timestamp: Timestamp::ZERO, + }) + .unwrap(); + writing.write(Bytes::from_static(b"old")).unwrap(); + + let pending = tokio::spawn(async move { subscriber.read_frame().await }); + tokio::task::yield_now().await; + assert!(!pending.is_finished(), "the partial frame should still be pending"); + + tokio::time::advance(std::time::Duration::from_secs(1)).await; + let mut edge = producer.append_group().unwrap(); + edge.write_frame(Timestamp::from_millis(1000).unwrap(), Bytes::from_static(b"edge")) + .unwrap(); + edge.finish().unwrap(); + + let frame = pending.await.unwrap().unwrap().expect("live edge frame"); + assert_eq!(frame.payload, Bytes::from_static(b"edge")); + writing.abort(crate::Error::Cancel).unwrap(); + settle().await; + + let report = registry.report(); + let traffic = &report + .traffic + .iter() + .find(|entry| entry.path.as_str() == "demo") + .expect("demo tracked") + .publisher; + assert_eq!(traffic.groups, 1, "only the live edge group was delivered"); + assert_eq!(traffic.frames, 1, "only the live edge frame was delivered"); + assert_eq!(traffic.bytes, 4, "only the live edge payload was delivered"); + assert_eq!(traffic.stale.groups, 0, "the internal group was never handed out"); + assert_eq!(traffic.stale.frames, 1, "the expired partial frame was discarded"); + assert_eq!(traffic.stale.bytes, 6, "the declared partial payload was discarded"); + } + /// Datagrams bypass the group/frame handles entirely, so they're metered at the /// producer (ingress write) and the subscriber (egress read). Each one counts as /// the single-frame group it stands in for, plus the `datagrams` breakout. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index ea70b1a31ad..e46fea8ad53 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -469,6 +469,7 @@ impl Consumer { min_sequence: 0, end_sequence: None, drift_cap: kio::Producer::new(None), + stale_stats: Default::default(), reading: None, } } @@ -775,6 +776,7 @@ impl Group { } } + /// Attribute expiry for route-specific copies behind this logical group. pub(crate) fn set_stale_meter(&mut self, meter: crate::stats::Meter) { if let Some(current) = &mut self.current { current.group.set_stale_meter(meter.clone()); @@ -1232,6 +1234,8 @@ pub struct Subscriber { end_sequence: Option, /// Shared copy of [`Self::end_sequence`] for groups that outlive this cursor poll. drift_cap: kio::Producer>, + /// Logical subscriber meter for groups drained internally by [`Self::poll_read_frame`]. + stale_stats: crate::stats::Meter, /// The group currently being drained by [`Self::read_frame`]. reading: Option, @@ -1245,6 +1249,14 @@ impl Subscriber { self.reap(); } + /// Attribute expiry for the group this frame helper drains internally. + pub(crate) fn set_stale_meter(&mut self, meter: crate::stats::Meter) { + if let Some(reading) = &mut self.reading { + reading.set_stale_meter(meter.clone()); + } + self.stale_stats = meter; + } + /// Reap retired cursors, then bound the live stragglers: a pruned segment's /// cursor keeps draining (groups below its cap may still arrive out of /// order, and its demand keeps the upstream serving them), but only the @@ -1699,7 +1711,10 @@ impl Subscriber { } match ready!(self.poll_next_group(waiter))? { - Some(group) => self.reading = Some(group), + Some(mut group) => { + group.set_stale_meter(self.stale_stats.clone()); + self.reading = Some(group); + } None => return Poll::Ready(Ok(None)), } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 2ec0933cc6c..4befee91a02 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3156,7 +3156,10 @@ impl Subscriber { let meter = self.stats.meter(); let res = match &mut self.inner { SubscriberKind::Plain(plain) => plain.poll_read_frame(waiter), - SubscriberKind::Spliced(spliced) => spliced.poll_read_frame(waiter), + SubscriberKind::Spliced(spliced) => { + spliced.set_stale_meter(meter.clone()); + spliced.poll_read_frame(waiter) + } }; self.count_stale(&meter); // This helper collapses a group to its first frame: count the group, the one From 25873f3c940f56a47a98c00b637de4d7c875db69 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 23:49:06 -0700 Subject: [PATCH 13/32] fix(moq-net): bound IETF latency fallback --- rs/moq-net/src/ietf/publisher.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 5f1d2dcf2a4..59e5d0d3045 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -22,7 +22,9 @@ use super::{Message, Version, cluster, peer}; fn serving_subscription(subscriber_priority: u8) -> Subscription { Subscription { priority: super::priority::from_wire(subscriber_priority), - latency: Latency::max(Duration::MAX), + // Demand can cross a Lite hop before the producer's retention bound is + // known, so use the largest duration that remains wire-encodable. + latency: Latency::max(Duration::from_millis(crate::coding::VarInt::MAX.into_inner())), ..Default::default() } } @@ -1781,7 +1783,12 @@ mod tests { group.finish().unwrap(); } - let mut subscriber = producer.subscribe(serving_subscription(128)); + let subscription = serving_subscription(128); + assert!( + crate::coding::VarInt::try_from(subscription.latency.max.as_millis()).is_ok(), + "fallback must remain encodable across a Lite relay hop" + ); + let mut subscriber = producer.subscribe(subscription); for sequence in [0, 1] { let group = subscriber .recv_group() From f7f95ead1b4520fbec57975f196c74e4ea60ef6a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 23:52:39 -0700 Subject: [PATCH 14/32] fix(moq-net): align IETF fallback across languages --- rs/moq-net/src/ietf/publisher.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 59e5d0d3045..30915ca3166 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -17,6 +17,9 @@ use crate::{ use super::{Message, Version, cluster, peer}; +/// Largest millisecond duration every implementation can carry losslessly. +const MAX_SAFE_LATENCY_MS: u64 = (1_u64 << 53) - 1; + /// Build the serving-side subscription for a peer whose wire protocol carries no /// latency preference. The receiver applies its own budget after the transfer. fn serving_subscription(subscriber_priority: u8) -> Subscription { @@ -24,7 +27,7 @@ fn serving_subscription(subscriber_priority: u8) -> Subscription { priority: super::priority::from_wire(subscriber_priority), // Demand can cross a Lite hop before the producer's retention bound is // known, so use the largest duration that remains wire-encodable. - latency: Latency::max(Duration::from_millis(crate::coding::VarInt::MAX.into_inner())), + latency: Latency::max(Duration::from_millis(MAX_SAFE_LATENCY_MS)), ..Default::default() } } @@ -1784,10 +1787,7 @@ mod tests { } let subscription = serving_subscription(128); - assert!( - crate::coding::VarInt::try_from(subscription.latency.max.as_millis()).is_ok(), - "fallback must remain encodable across a Lite relay hop" - ); + assert_eq!(subscription.latency.max.as_millis(), MAX_SAFE_LATENCY_MS as u128); let mut subscriber = producer.subscribe(subscription); for sequence in [0, 1] { let group = subscriber From 5dc992ce1ebc6af51914f98c51987b11d2e1425e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 00:19:46 -0700 Subject: [PATCH 15/32] fix(moq-net): expire transport-blocked group streams --- rs/moq-net/src/ietf/publisher.rs | 6 ++++ rs/moq-net/src/lite/publisher.rs | 51 +++++++++++++++++++++++++++ rs/moq-net/src/lite/test_transport.rs | 19 +++++++++- rs/moq-net/src/model/group.rs | 3 +- 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 30915ca3166..9a6f5b52670 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -1552,6 +1552,12 @@ impl GroupServe { }; } GroupState::Serve { writer, frame, chunk } => { + // Keep the group guard live while the transport owns a detached + // frame chunk and may be blocked on flow control. + if self.group.poll_expired(waiter) { + self.state = GroupState::Done; + return Poll::Ready(Err(Error::Old)); + } // The peer closing first cancels the group. if writer.poll_closed(&mut cx).is_ready() { self.state = GroupState::Done; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 8fa770f671e..59947021334 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2962,6 +2962,11 @@ impl GroupServe { } let outcome = 'serve: { + // Keep the group guard live while the transport owns a detached + // frame chunk and may be blocked on flow control. + if self.group.poll_expired(waiter) { + break 'serve Err(Error::Old); + } // The peer closing first cancels the group. if writer.poll_closed(&mut cx).is_ready() { break 'serve Err(Error::Cancel); @@ -3210,6 +3215,52 @@ mod serve_group_test { assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); } + /// A subscription group keeps checking latency while a transport write is + /// flow-control blocked, so a stalled send cannot pin the stream indefinitely. + #[tokio::test] + async fn blocked_transport_write_expires_with_the_group() { + tokio::time::pause(); + + let gate = kio::Producer::new(false); + let session = SinkSession::gated_uni(gate.consume()); + let log = session.log.clone(); + let track_priority = kio::Producer::new(0u8); + let subscription = Subscription { + session, + id: 0, + track_name: "test".into(), + priority: PriorityQueue::default(), + track_priority: track_priority.consume(), + track_priority_seen: 0, + ordered: false, + version: Version::Lite06Wip, + timescale: Some(crate::Timescale::default()), + }; + + let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); + let mut subscriber = track.subscribe(None); + let mut old = track.append_group().unwrap(); + old.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); + old.finish().unwrap(); + let group = subscriber.recv_group().await.unwrap().expect("old group"); + + let handle = subscription.priority.insert(Priority::new(0, 0, 0)); + let mut serve = std::pin::pin!(subscription.serve_group(0, 0, handle, group)); + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "transport write is blocked" + ); + + tokio::time::advance(std::time::Duration::from_secs(1)).await; + let mut edge = track.append_group().unwrap(); + edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) + .unwrap(); + edge.finish().unwrap(); + + assert!(matches!(serve.await, Err(Error::Old))); + assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); + } + /// A group that completes cleanly must not reset at all. The completion path /// releases the stream via `poll_close`; leaving the writer to drop after /// `finish()` would fire the Drop fallback and tack a spurious Cancel reset diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 6b3dd8c76fc..b8c9a6e1f0b 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -313,6 +313,8 @@ pub struct SinkSession { /// Set by [`Self::gated_bi`]. `None` parks `open_bi` itself forever, which is all /// a test driving only uni streams needs. bi_gate: Option>, + /// Set by [`Self::gated_uni`] to hold unidirectional stream writes. + uni_gate: Option>, /// The ALPN to report, for a test that needs a specific negotiated version rather /// than the SETUP-negotiated fallback an absent one selects. protocol: Option<&'static str>, @@ -323,6 +325,7 @@ impl SinkSession { Self { log, bi_gate: None, + uni_gate: None, protocol: None, } } @@ -342,6 +345,17 @@ impl SinkSession { Self { log: Log::default(), bi_gate: Some(gate), + uni_gate: None, + protocol: None, + } + } + + /// Open unidirectional streams immediately, holding their writes until `gate` opens. + pub fn gated_uni(gate: kio::Consumer) -> Self { + Self { + log: Log::default(), + bi_gate: None, + uni_gate: Some(gate), protocol: None, } } @@ -376,7 +390,10 @@ impl poll::Session for SinkSession { } fn poll_open_uni(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(SinkSend::new(self.log.clone()))) + Poll::Ready(Ok(match &self.uni_gate { + Some(gate) => SinkSend::gated(self.log.clone(), gate.clone()), + None => SinkSend::new(self.log.clone()), + })) } fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 2e1aa840533..debd983cdea 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1018,7 +1018,8 @@ impl Consumer { self } - fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { + /// Check the parent subscription while a wire publisher drains detached payload. + pub(crate) fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { if !self.expired && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) { self.expired = true; if !self.stale_counted.swap(true, Ordering::Relaxed) { From aaa68014487d89a2c9badbbf8137b453435fcba9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 00:49:20 -0700 Subject: [PATCH 16/32] fix(moq-net): enforce latency during group startup --- rs/hang/examples/subscribe.rs | 9 +++-- rs/moq-net/src/ietf/publisher.rs | 48 +++++++++++++++++++++++++++ rs/moq-net/src/lite/publisher.rs | 48 +++++++++++++++++++++++++++ rs/moq-net/src/lite/test_transport.rs | 30 ++++++++++++++++- 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index 2b2365adb33..5881bba75ce 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -74,12 +74,17 @@ async fn run_subscribe(consumer: moq_net::origin::Consumer) -> anyhow::Result<() ); // Subscribe to the video track. + let latency = moq_mux::Latency::max(Duration::from_millis(500)); let track_consumer = broadcast .track(name)? - .subscribe(moq_net::track::Subscription::default().with_priority(1)) + .subscribe( + moq_net::track::Subscription::default() + .with_priority(1) + .with_latency(latency), + ) .await?; let mut ordered = moq_mux::container::Consumer::new(track_consumer, moq_mux::catalog::hang::Container::Legacy) - .with_latency(moq_mux::Latency::max(Duration::from_millis(500))); + .with_latency(latency); // Read frames in latency-bounded presentation order. while let Some(frame) = ordered.read().await? { diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 9a6f5b52670..4b93444f487 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -1530,6 +1530,10 @@ impl GroupServe { loop { match &mut self.state { GroupState::Open => { + if self.group.poll_expired(waiter) { + self.state = GroupState::Done; + return Poll::Ready(Err(Error::Old)); + } let stream = match ready!(self.session.poll_open_uni(&mut cx)) { Ok(stream) => stream, Err(err) => { @@ -1717,6 +1721,50 @@ mod group_priority_test { "model priority must pass through unchanged" ); } + + /// A subgroup waiting for stream credit keeps its subscription expiry armed. + #[tokio::test] + async fn group_waiting_for_stream_credit_expires() { + tokio::time::pause(); + + let gate = kio::Producer::new(false); + let session = SinkSession::gated_open_uni(gate.consume()); + let mut track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "test", None); + let mut subscriber = track.subscribe(None); + let mut old = track.append_group().unwrap(); + old.write_frame(crate::Timestamp::ZERO, b"old".as_slice()).unwrap(); + old.finish().unwrap(); + let group = subscriber.recv_group().await.unwrap().expect("old group"); + + let mut serve = GroupServe { + session, + msg: ietf::GroupHeader { + track_alias: 0, + group_id: 0, + sub_group_id: 0, + publisher_priority: 0, + flags: Default::default(), + }, + priority: 0, + group, + timescale: Timescale::default(), + version: Version::Draft19, + state: GroupState::Open, + }; + let mut serving = std::pin::pin!(kio::wait(|waiter| serve.poll_serve(waiter))); + assert!( + futures::poll!(serving.as_mut()).is_pending(), + "stream credit is exhausted" + ); + + tokio::time::advance(std::time::Duration::from_secs(1)).await; + let mut edge = track.append_group().unwrap(); + edge.write_frame(crate::Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) + .unwrap(); + edge.finish().unwrap(); + + assert!(matches!(serving.await, Err(Error::Old))); + } } #[cfg(test)] diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 59947021334..4a36212c121 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2911,6 +2911,10 @@ impl GroupServe { loop { match &mut self.state { GroupState::Open => { + if self.group.poll_expired(waiter) { + self.state = GroupState::Done; + return Poll::Ready(Err(Error::Old)); + } let mut cx = Context::from_waker(waiter.waker()); let stream = match ready!(self.ctx.session.poll_open_uni(&mut cx)) { Ok(stream) => stream, @@ -3261,6 +3265,50 @@ mod serve_group_test { assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); } + /// A subscription group keeps checking latency while transport stream credit is + /// exhausted, so returning credit is reserved for content that is still live. + #[tokio::test] + async fn blocked_transport_open_expires_with_the_group() { + tokio::time::pause(); + + let gate = kio::Producer::new(false); + let session = SinkSession::gated_open_uni(gate.consume()); + let track_priority = kio::Producer::new(0u8); + let subscription = Subscription { + session, + id: 0, + track_name: "test".into(), + priority: PriorityQueue::default(), + track_priority: track_priority.consume(), + track_priority_seen: 0, + ordered: false, + version: Version::Lite06Wip, + timescale: Some(crate::Timescale::default()), + }; + + let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); + let mut subscriber = track.subscribe(None); + let mut old = track.append_group().unwrap(); + old.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); + old.finish().unwrap(); + let group = subscriber.recv_group().await.unwrap().expect("old group"); + + let handle = subscription.priority.insert(Priority::new(0, 0, 0)); + let mut serve = std::pin::pin!(subscription.serve_group(0, 0, handle, group)); + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "stream credit is exhausted" + ); + + tokio::time::advance(std::time::Duration::from_secs(1)).await; + let mut edge = track.append_group().unwrap(); + edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) + .unwrap(); + edge.finish().unwrap(); + + assert!(matches!(serve.await, Err(Error::Old))); + } + /// A group that completes cleanly must not reset at all. The completion path /// releases the stream via `poll_close`; leaving the writer to drop after /// `finish()` would fire the Drop fallback and tack a spurious Cancel reset diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index b8c9a6e1f0b..3b56ade9d4b 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -315,6 +315,9 @@ pub struct SinkSession { bi_gate: Option>, /// Set by [`Self::gated_uni`] to hold unidirectional stream writes. uni_gate: Option>, + /// Set by [`Self::gated_open_uni`] to withhold unidirectional stream credit. + uni_open_gate: Option>, + uni_open_park: kio::Park, /// The ALPN to report, for a test that needs a specific negotiated version rather /// than the SETUP-negotiated fallback an absent one selects. protocol: Option<&'static str>, @@ -326,6 +329,8 @@ impl SinkSession { log, bi_gate: None, uni_gate: None, + uni_open_gate: None, + uni_open_park: kio::Park::default(), protocol: None, } } @@ -346,6 +351,8 @@ impl SinkSession { log: Log::default(), bi_gate: Some(gate), uni_gate: None, + uni_open_gate: None, + uni_open_park: kio::Park::default(), protocol: None, } } @@ -356,6 +363,20 @@ impl SinkSession { log: Log::default(), bi_gate: None, uni_gate: Some(gate), + uni_open_gate: None, + uni_open_park: kio::Park::default(), + protocol: None, + } + } + + /// Hold unidirectional stream opens until `gate` grants stream credit. + pub fn gated_open_uni(gate: kio::Consumer) -> Self { + Self { + log: Log::default(), + bi_gate: None, + uni_gate: None, + uni_open_gate: Some(gate), + uni_open_park: kio::Park::default(), protocol: None, } } @@ -389,7 +410,14 @@ impl poll::Session for SinkSession { Poll::Ready(Ok((send, PendingRecv))) } - fn poll_open_uni(&mut self, _cx: &mut Context<'_>) -> Poll> { + fn poll_open_uni(&mut self, cx: &mut Context<'_>) -> Poll> { + if let Some(gate) = &self.uni_open_gate { + let waiter = self.uni_open_park.hold(cx); + match gate.poll(waiter, |open| (**open).then_some(()).map_or(Poll::Pending, Poll::Ready)) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(_)) | Poll::Pending => return Poll::Pending, + } + } Poll::Ready(Ok(match &self.uni_gate { Some(gate) => SinkSend::gated(self.log.clone(), gate.clone()), None => SinkSend::new(self.log.clone()), From a9fc6bb86099f6298785d451706167ea6db05b49 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 01:07:06 -0700 Subject: [PATCH 17/32] fix(moq-net): exclude fetches from live drift --- rs/moq-net/src/model/track.rs | 50 ++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4befee91a02..febdceeb930 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -538,10 +538,18 @@ impl TrackState { /// Capped because a group can only be late relative to data that would actually be /// served in its place: a subscription ending at a takeover boundary can't jump past /// it, so the groups it still wants aren't stale just because the route ran on. + /// Fetched backfill is absent from `arrival`, so it cannot age subscription content + /// as though it were a live replacement. fn live_edge(&self, cap: Option) -> Option { let mut wall: Option = None; let mut presentation: Option = None; - for slot in self.lookup.values() { + for (sequence, stamp) in &self.arrival { + let Some(slot) = self.lookup.get(sequence) else { + continue; + }; + if slot.stamp != *stamp { + continue; + } let group = &slot.group; if cap.is_some_and(|cap| group.sequence > cap) || group.is_aborted() { continue; @@ -4306,6 +4314,46 @@ mod test { ); } + /// A one-shot fetch populates the shared cache but never the live arrival cursor, + /// so it cannot make content available to a subscription look stale. + #[tokio::test] + async fn fetched_group_is_not_a_live_drift_edge() { + let mut producer = track_producer("test", None); + let dynamic = producer.dynamic(); + let consumer = producer.consume(); + append_at(&mut producer, 0); + + let pending = consumer.fetch_group(100, None); + let req = dynamic + .requested_group() + .now_or_never() + .expect("fetch request is ready") + .unwrap(); + let mut fetched = req.accept(None).unwrap(); + fetched + .write_frame( + Timestamp::from_millis(100_000).unwrap(), + bytes::Bytes::from_static(b"fetched"), + ) + .unwrap(); + fetched.finish().unwrap(); + pending.await.unwrap(); + + let mut groups = producer.subscribe(None); + assert_eq!(groups.assert_group().sequence, 0); + groups.assert_no_group(); + + let mut frames = producer.subscribe(None); + let frame = frames + .read_frame() + .now_or_never() + .expect("live frame is ready") + .unwrap() + .expect("live frame"); + assert_eq!(frame.timestamp.as_millis(), 0); + assert!(frames.read_frame().now_or_never().is_none()); + } + /// The live edge is resolved once per poll and applied to every group that poll /// walks off, so it has to be revalidated before it convicts anything: an eviction /// in between would otherwise discard a group on the strength of content that is no From c8b3c04ec6dec5606c77129cfbb01411ca37666c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 01:15:19 -0700 Subject: [PATCH 18/32] refactor(moq-net): store group visibility --- rs/moq-net/src/model/track.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index febdceeb930..8af15e352c6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -256,6 +256,10 @@ struct Slot { // backfill) gets a fresh stamp, so a historical arrival entry can't resolve to // the replacement and deliver it twice or at the wrong position. stamp: u32, + + // Whether this incarnation came from the live publisher and can replace older + // subscription content. Fetch-only backfill stays cached but never anchors drift. + visible: bool, } /// The registered subscriptions, aggregated by the producer. @@ -543,11 +547,8 @@ impl TrackState { fn live_edge(&self, cap: Option) -> Option { let mut wall: Option = None; let mut presentation: Option = None; - for (sequence, stamp) in &self.arrival { - let Some(slot) = self.lookup.get(sequence) else { - continue; - }; - if slot.stamp != *stamp { + for slot in self.lookup.values() { + if !slot.visible { continue; } let group = &slot.group; @@ -806,6 +807,7 @@ impl TrackState { group: group.clone(), arrived: web_async::time::Instant::now(), stamp, + visible, }, ); if visible { From 62ccb8378c20fb1c93a6b0c2e21978c4952c9d23 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 01:44:26 -0700 Subject: [PATCH 19/32] fix(moq-relay): expose stale Prometheus counters --- doc/bin/relay/http.md | 5 +++- rs/moq-relay/src/internal.rs | 53 ++++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 566d1493e1e..874d8e74f55 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -109,7 +109,10 @@ cluster topology endpoint below. ### GET /metrics Prometheus text exposition of this node's own traffic counters (bytes, frames, -groups, subscriptions, viewers, sessions), split by `tier` and `role`. +groups, subscriptions, viewers, sessions), split by `tier` and `role`. Content +skipped after drifting beyond a subscriber's latency budget is exposed through +`moq_relay_stale_bytes_total`, `moq_relay_stale_frames_total`, +`moq_relay_stale_groups_total`, and `moq_relay_stale_datagrams_total`. Alongside them are the TCP listeners' accept-loop counters, which are how a node that has stopped accepting connections says so: diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index b48812ddf18..bbf03181452 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -263,6 +263,30 @@ fn render_metrics(snap: &moq_net::stats::Snapshot, listeners: &[moq_tokio::accep "Single-frame groups carried over unreliable QUIC datagrams; a subset of groups.", |c| c.datagrams, ); + counter( + &mut out, + "moq_relay_stale_bytes_total", + "Media payload bytes skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.bytes, + ); + counter( + &mut out, + "moq_relay_stale_frames_total", + "Media frames skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.frames, + ); + counter( + &mut out, + "moq_relay_stale_groups_total", + "Media groups skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.groups, + ); + counter( + &mut out, + "moq_relay_stale_datagrams_total", + "Media datagrams skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.datagrams, + ); counter( &mut out, "moq_relay_fetches_total", @@ -580,12 +604,21 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(1)).await; tokio::time::sleep(std::time::Duration::from_millis(1)).await; - // Read 1234 egress bytes out of the default-tier broadcast. + // Leave 46 bytes across two frames behind the live edge, then read 1234 + // egress bytes out of the default-tier broadcast. let bc = announced.next().await.unwrap().broadcast.unwrap(); let mut egress_sub = bc.track("video").unwrap().subscribe(None).await.unwrap(); { let mut group = pub_track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, vec![0u8; 1234]).unwrap(); + group.write_frame(Timestamp::ZERO, vec![0u8; 12]).unwrap(); + group.write_frame(Timestamp::ZERO, vec![0u8; 34]).unwrap(); + group.finish().unwrap(); + } + { + let mut group = pub_track.append_group().unwrap(); + group + .write_frame(Timestamp::from_millis(10_000).unwrap(), vec![0u8; 1234]) + .unwrap(); group.finish().unwrap(); } let mut group = egress_sub.recv_group().await.unwrap().unwrap(); @@ -612,6 +645,22 @@ mod tests { body.contains("moq_relay_bytes_total{tier=\"region/sjc\",role=\"subscriber\"} 56"), "named tier gets its own row:\n{body}" ); + assert!( + body.contains("moq_relay_stale_bytes_total{tier=\"\",role=\"publisher\"} 46"), + "stale egress bytes:\n{body}" + ); + assert!( + body.contains("moq_relay_stale_frames_total{tier=\"\",role=\"publisher\"} 2"), + "stale egress frames:\n{body}" + ); + assert!( + body.contains("moq_relay_stale_groups_total{tier=\"\",role=\"publisher\"} 1"), + "stale egress groups:\n{body}" + ); + assert!( + body.contains("moq_relay_stale_datagrams_total{tier=\"\",role=\"publisher\"} 0"), + "stale egress datagrams:\n{body}" + ); assert!( body.contains("moq_relay_sessions_opened_total{tier=\"\"} 1"), "session presence:\n{body}" From cfcc1a5e12a12fc7f63587509792a1d9f8b4188b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 02:37:34 -0700 Subject: [PATCH 20/32] fix(moq-net): preserve clean group completion --- py/moq-rs/tests/test_local.py | 15 ++++++++------- rs/moq-net/src/model/group.rs | 24 +++++++++++++++++++++++- rs/moq-net/src/model/track.rs | 15 +++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index c3d57ad1bf8..211e83104de 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -621,7 +621,7 @@ async def test_raw_multiple_frames(): consumer = origin.consume() async for announcement in consumer.announced(): - raw_consumer = await announcement.broadcast.subscribe_track("commands") + raw_consumer = await announcement.broadcast.subscribe_track("commands", moq.Subscription(latency_max_ms=1_000)) messages = [ b'{"cmd": "led", "arm": "left", "led": "THUMB", "state": 1}', @@ -646,7 +646,7 @@ async def test_raw_producer_consume_direct(): """Consume a raw track directly from the producer, no origin/broadcast plumbing.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("direct") - consumer = track.consume() + consumer = track.consume(moq.Subscription(latency_max_ms=1_000)) track.write_frame(b"hello", 0) track.write_frame(b"world", 0) @@ -702,7 +702,7 @@ async def test_raw_group_sequence(): consumer = origin.consume() async for announcement in consumer.announced(): - raw_consumer = await announcement.broadcast.subscribe_track("seq") + raw_consumer = await announcement.broadcast.subscribe_track("seq", moq.Subscription(latency_max_ms=1_000)) sent_sequences = [] for i in range(3): @@ -733,8 +733,9 @@ async def test_default_iteration_is_sequence_order(): broadcast = origin.create_broadcast("track/ordering") raw = broadcast.publish_track("ordering") - seq_consumer = raw.consume() - arr_consumer = raw.consume() + subscription = moq.Subscription(latency_max_ms=1_000) + seq_consumer = raw.consume(subscription) + arr_consumer = raw.consume(subscription) for sequence in (5, 3): group = raw.create_group(sequence) @@ -787,7 +788,7 @@ async def test_read_frame_one_per_group(): """read_frame() returns the first frame of each successive group.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("status") - consumer = track.consume() + consumer = track.consume(moq.Subscription(latency_max_ms=1_000)) track.write_frame(b"ready", 0) track.write_frame(b"running", 0) @@ -831,7 +832,7 @@ async def test_read_frame_skips_remaining_frames_in_group(): """read_frame() only returns the first frame of a multi-frame group.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("mixed") - consumer = track.consume() + consumer = track.consume(moq.Subscription(latency_max_ms=1_000)) group = track.append_group() group.write_frame(b"first", 0) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index debd983cdea..27e3bd55749 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1020,7 +1020,10 @@ impl Consumer { /// Check the parent subscription while a wire publisher drains detached payload. pub(crate) fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { - if !self.expired && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) { + if !self.expired + && self.expiry_pending() + && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) + { self.expired = true; if !self.stale_counted.swap(true, Ordering::Relaxed) { self.stale_stats.stale(self.unread_content()); @@ -1029,6 +1032,15 @@ impl Consumer { self.expired } + /// Whether expiry can still discard content or unblock a group that may grow. + fn expiry_pending(&self) -> bool { + match &self.inner { + ConsumerKind::Plain(plain) => plain.expiry_pending(), + // The route-specific plain cursors own expiry for a spliced group. + ConsumerKind::Spliced(_) => false, + } + } + /// Whether this cursor failed because its subscription latency budget expired. pub(crate) fn latency_expired(&self) -> bool { self.expired @@ -1192,6 +1204,16 @@ impl Consumer { } impl Plain { + /// Whether this cursor still has unread content or may receive another frame. + fn expiry_pending(&self) -> bool { + if self.capped() { + return false; + } + + let state = self.state.read(); + state.abort.is_none() && state.fin.is_none_or(|fin| self.index < fin) + } + /// Content this cursor has neither returned nor already counted in a prefetch batch. fn unread_content(&self) -> stats::Content { let prefetched = self.prefetch.buffered().0 as usize; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8af15e352c6..adebeba2b10 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4156,6 +4156,21 @@ mod test { assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); } + #[tokio::test] + async fn a_drained_group_finishes_cleanly_after_the_live_edge_advances() { + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + append_at(&mut producer, 0); + + let mut group = subscriber.recv_group().await.unwrap().expect("first group"); + assert!(group.read_frame().await.unwrap().is_some()); + + append_at(&mut producer, 1000); + + assert!(group.read_frame().await.unwrap().is_none()); + assert!(!group.latency_expired()); + } + #[tokio::test] async fn a_handed_out_partial_frame_expires_while_its_payload_is_stalled() { tokio::time::pause(); From 86afbb8c4e87e760067ff27da90f30e18907af26 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 02:38:24 -0700 Subject: [PATCH 21/32] fix(moq-mux): inherit initial subscription latency --- rs/hang/examples/subscribe.rs | 3 +- rs/libmoq/src/consume.rs | 4 +- rs/moq-audio/src/decode/consumer.rs | 2 +- rs/moq-audio/src/decode/decoder.rs | 4 +- rs/moq-cli/src/publish.rs | 2 +- rs/moq-ffi/src/consumer.rs | 3 +- rs/moq-gst/src/source/imp.rs | 2 +- rs/moq-mux/src/codec/h264/export.rs | 4 +- rs/moq-mux/src/codec/h265/export.rs | 4 +- rs/moq-mux/src/container/consumer.rs | 158 +++++++++++--------- rs/moq-mux/src/container/flv/export.rs | 4 +- rs/moq-mux/src/container/flv/import_test.rs | 8 +- rs/moq-mux/src/container/fmp4/export.rs | 4 +- rs/moq-mux/src/container/mkv/export.rs | 4 +- rs/moq-mux/src/container/source.rs | 2 +- rs/moq-mux/src/container/ts/export.rs | 4 +- rs/moq-mux/src/container/ts/export_test.rs | 8 +- rs/moq-mux/src/container/ts/import.rs | 10 +- rs/moq-mux/src/container/ts/import_test.rs | 6 +- rs/moq-video/src/decode/consumer.rs | 2 +- rs/moq-video/src/decode/decoder.rs | 4 +- 21 files changed, 126 insertions(+), 116 deletions(-) diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index 5881bba75ce..194efa71626 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -83,8 +83,7 @@ async fn run_subscribe(consumer: moq_net::origin::Consumer) -> anyhow::Result<() .with_latency(latency), ) .await?; - let mut ordered = moq_mux::container::Consumer::new(track_consumer, moq_mux::catalog::hang::Container::Legacy) - .with_latency(latency); + let mut ordered = moq_mux::container::Consumer::new(track_consumer, moq_mux::catalog::hang::Container::Legacy); // Read frames in latency-bounded presentation order. while let Some(frame) = ordered.read().await? { diff --git a/rs/libmoq/src/consume.rs b/rs/libmoq/src/consume.rs index b945873c0e1..fa07d806701 100644 --- a/rs/libmoq/src/consume.rs +++ b/rs/libmoq/src/consume.rs @@ -385,7 +385,7 @@ impl Consume { .with_latency(latency), ) .await?; - let track = moq_mux::container::Consumer::new(track, container).with_latency(latency); + let track = moq_mux::container::Consumer::new(track, container); Self::run_track(on_frame, track, channel.1).await } .await; @@ -438,7 +438,7 @@ impl Consume { .with_latency(latency), ) .await?; - let track = moq_mux::container::Consumer::new(track, container).with_latency(latency); + let track = moq_mux::container::Consumer::new(track, container); Self::run_track(on_frame, track, channel.1).await } .await; diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 6ff14c48411..e67af9a6cbd 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -60,7 +60,7 @@ impl Consumer { // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(&catalog.container)?; - let track = moq_mux::container::Consumer::new(track, container).with_latency(config.latency); + let track = moq_mux::container::Consumer::new(track, container); Ok(Self { decoder, diff --git a/rs/moq-audio/src/decode/decoder.rs b/rs/moq-audio/src/decode/decoder.rs index 9506930e956..c2474235aef 100644 --- a/rs/moq-audio/src/decode/decoder.rs +++ b/rs/moq-audio/src/decode/decoder.rs @@ -33,8 +33,8 @@ pub struct Config { pub channels: Option, /// How far playback may drift from the live edge before skipping a stalled group. /// - /// Applied to the initial transport subscription and - /// [`moq_mux::container::Consumer::with_latency`]. Defaults to + /// Applied to the initial transport subscription and inherited by + /// [`moq_mux::container::Consumer`]. Defaults to /// [`Latency::REAL_TIME`](moq_mux::Latency::REAL_TIME), which skips aggressively. /// Set [`Latency::max`](moq_mux::Latency::max) to the playout buffer you can /// tolerate (typically tens to a few hundred ms) for the best diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index ea478e21ef6..856da08b088 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -663,7 +663,7 @@ mod tests { /// Read the first frame of a verbatim track back as raw bytes. async fn read_frame(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(moq_mux::Latency::REAL_TIME); + let mut reader = Consumer::new(track, Container::Legacy); let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) .await .expect("verbatim read timed out") diff --git a/rs/moq-ffi/src/consumer.rs b/rs/moq-ffi/src/consumer.rs index cb4113c8e56..e610b0b3360 100644 --- a/rs/moq-ffi/src/consumer.rs +++ b/rs/moq-ffi/src/consumer.rs @@ -289,9 +289,8 @@ impl MoqBroadcastConsumer { // subscription if init parsing fails. let media = media_container(container)?; let subscription = subscription.map(moq_net::track::Subscription::from).unwrap_or_default(); - let latency = subscription.latency; let track = self.inner.track(&name)?.subscribe(subscription).await?; - let consumer = moq_mux::container::Consumer::new(track, media).with_latency(latency); + let consumer = moq_mux::container::Consumer::new(track, media); Ok(Arc::new(MoqMediaConsumer { task: Task::new(Media { inner: consumer }), })) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index e91f0606585..d0e7ab6ac1d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -445,7 +445,7 @@ async fn reconcile( let latency = moq_mux::Latency::max(Duration::from_secs(1)); let subscription = moq_net::track::Subscription::default().with_latency(latency); let track_subscriber = broadcast.track(&name)?.subscribe(subscription).await?; - let track = moq_mux::container::Consumer::new(track_subscriber, container).with_latency(latency); + let track = moq_mux::container::Consumer::new(track_subscriber, container); let descriptor = TrackDescriptor { kind: d.kind, diff --git a/rs/moq-mux/src/codec/h264/export.rs b/rs/moq-mux/src/codec/h264/export.rs index 08d2bc36940..9d77642cd38 100644 --- a/rs/moq-mux/src/codec/h264/export.rs +++ b/rs/moq-mux/src/codec/h264/export.rs @@ -67,8 +67,8 @@ impl Export { /// Set the latency tolerance for the per-track source. /// - /// See [`Consumer::with_latency`](crate::container::Consumer::with_latency) for the - /// per-track skip behavior. Defaults to + /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. + /// Defaults to /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (skip aggressively). pub fn with_latency(mut self, latency: crate::Latency) -> Self { self.latency = latency; diff --git a/rs/moq-mux/src/codec/h265/export.rs b/rs/moq-mux/src/codec/h265/export.rs index 765057f995a..3b0f849bfbe 100644 --- a/rs/moq-mux/src/codec/h265/export.rs +++ b/rs/moq-mux/src/codec/h265/export.rs @@ -59,8 +59,8 @@ impl Export { /// Set the latency tolerance for the per-track source. /// - /// See [`Consumer::with_latency`](crate::container::Consumer::with_latency) for the - /// per-track skip behavior. Defaults to + /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. + /// Defaults to /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (skip aggressively). pub fn with_latency(mut self, latency: crate::Latency) -> Self { self.latency = latency; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index d3088a67235..5ec55b1d23e 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -29,8 +29,9 @@ use super::{Container, Frame}; /// group's first timestamp has nothing left worth waiting for. Containers without a /// duration report zero, which disables this check and falls back to the latency budget. /// -/// Set the latency with [`with_latency`](Self::with_latency) (builder) or -/// [`set_latency`](Self::set_latency) (mid-stream). +/// Put the initial latency on the [`moq_net::track::Subscription`] before +/// subscribing. [`new`](Self::new) inherits that budget, and +/// [`set_latency`](Self::set_latency) changes it mid-stream. /// /// ## Timeline rewinds /// @@ -128,26 +129,25 @@ impl Reset { impl Consumer { /// Create a Consumer wrapping the given moq-lite consumer, decoding `format`. /// - /// Skips aggressively by default; raise the tolerance with - /// [`with_latency`](Self::with_latency). + /// The ordering window inherits the subscriber's current latency budget. Put + /// that budget on the [`moq_net::track::Subscription`] before awaiting the + /// subscription, so the publisher preserves the same replay window. pub fn new(track: moq_net::track::Subscriber, format: F) -> Self { + let latency = track.subscription().latency; Self { track, format, current: 0, pending: VecDeque::new(), startup: true, - latency: crate::Latency::REAL_TIME, + latency, rewind: Rewind::default(), } } - /// Set the latency tolerance. - /// - /// Groups older than the newest timestamp minus [`Latency::max`](crate::Latency::max) are - /// skipped. [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (the default) skips - /// aggressively: any group with a newer alternative is dropped. - pub fn with_latency(mut self, latency: crate::Latency) -> Self { + /// Configure both layers while constructing an in-process test fixture. + #[cfg(test)] + pub(crate) fn with_test_latency(mut self, latency: crate::Latency) -> Self { self.set_latency(latency); self } @@ -775,7 +775,7 @@ mod tests { /// tests are the exception, since they are about the consumer's half of it. fn container_latency_only(track: moq_net::track::Subscriber, latency: Latency) -> Consumer { let control = track.control(); - let consumer = Consumer::new(track, Container::Legacy).with_latency(latency); + let consumer = Consumer::new(track, Container::Legacy).with_test_latency(latency); control .update(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(30)))) .unwrap(); @@ -784,12 +784,24 @@ mod tests { // ---- Basic Reading ---- + #[test] + fn new_inherits_the_initial_subscription_latency() { + let track = track_producer("test", hang::container::track_info()); + let latency = Latency::max(Duration::from_millis(250)); + let subscriber = track.subscribe(moq_net::track::Subscription::default().with_latency(latency)); + + let consumer = Consumer::new(subscriber, Container::Legacy); + + assert_eq!(consumer.latency, latency); + assert_eq!(consumer.track.subscription().latency, latency); + } + #[tokio::test] async fn read_single_group() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); @@ -807,8 +819,8 @@ mod tests { async fn read_multiple_frames_single_group() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0), ts(33_000), ts(66_000)]); track.finish().unwrap(); @@ -826,8 +838,8 @@ mod tests { async fn read_multiple_groups_within_latency() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); // 5 groups, 20ms spacing. Total span = 80ms, well within 500ms latency. for i in 0..5u64 { @@ -846,8 +858,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(100))); // Group 0: 5 frames, NOT finished (blocks consumer) let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -998,7 +1010,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); // Old epoch, played forward until the live edge passes the rewind point. write_group(&mut track, 0, &[ts(0)]); @@ -1038,7 +1050,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); // Old timeline, played to a live edge of 200 ms. write_group(&mut track, 0, &[ts(0)]); @@ -1075,7 +1087,7 @@ mod tests { let consumer_track = track.subscribe(None); // Large latency so the slow-group skip never fires; isolate the rewind path. let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); // Publisher runs ahead: groups 0-4 at 0, 100, 200, 300, 400 ms. for i in 0..5u64 { @@ -1101,7 +1113,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 1, &[ts(500_000)]); @@ -1136,8 +1148,8 @@ mod tests { async fn empty_payload_is_skipped() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); let media = |timestamp| Frame { @@ -1166,8 +1178,8 @@ mod tests { async fn consecutive_markers_do_not_stall() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1200,8 +1212,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1237,8 +1249,8 @@ mod tests { async fn adjacent_group_flushed_immediately() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 1, &[ts(30_000)]); @@ -1256,8 +1268,8 @@ mod tests { async fn bframes_within_group() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0), ts(66_000), ts(33_000)]); track.finish().unwrap(); @@ -1276,8 +1288,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); track.finish().unwrap(); @@ -1295,8 +1307,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0)]); track.abort(moq_net::Error::Cancel).unwrap(); @@ -1319,8 +1331,8 @@ mod tests { async fn gap_in_group_sequence_recovery() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(100))); write_group(&mut track, 0, &[ts(0), ts(20_000)]); write_group(&mut track, 1, &[ts(40_000), ts(60_000)]); @@ -1339,7 +1351,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(80))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_millis(80))); write_group(&mut track, 5, &[ts(0), ts(20_000)]); write_group(&mut track, 7, &[ts(80_000), ts(100_000)]); @@ -1362,8 +1374,8 @@ mod tests { async fn evicted_group_with_gap_skips_to_live() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(100))); // Group 0: a frame the consumer reads, positioning it there. let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1406,8 +1418,8 @@ mod tests { async fn missing_sequence_skips_on_live_track() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(100))); // Group 0, then group 2 -- sequence 1 is missing (evicted) and never arrives. // The track is NOT finished (live), the case that used to hang. @@ -1506,8 +1518,8 @@ mod tests { async fn frame_timestamp_and_index_decoding() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0), ts(33_333), ts(66_666)]); track.finish().unwrap(); @@ -1527,8 +1539,8 @@ mod tests { async fn frame_payload_preserved() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let payload_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05]; let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1567,7 +1579,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1614,7 +1626,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(3700))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(3700))); let one_hour = 3_600_000_000u64; write_group(&mut track, 0, &[ts(one_hour)]); @@ -1631,7 +1643,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); @@ -1651,8 +1663,8 @@ mod tests { let consumer_track = track.subscribe(None); // latency must exceed (group1_max - group0_min) = 100ms - 0ms = 100ms // to avoid the latency skip and test B-frame timestamp tracking. - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(110))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(110))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); for ×tamp in &[ts(0), ts(66_000), ts(33_000)] { @@ -1702,8 +1714,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(100))); write_group(&mut track, 3, &[ts(0)]); write_group(&mut track, 5, &[ts(150_000)]); @@ -1756,8 +1768,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let _group5 = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); write_group(&mut track, 7, &[ts(210_000)]); @@ -1780,8 +1792,8 @@ mod tests { async fn startup_single_group_mid_stream() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 100, &[ts(3_000_000)]); track.finish().unwrap(); @@ -1796,7 +1808,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(50))); + Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_millis(50))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1832,8 +1844,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(100))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(100))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1925,8 +1937,8 @@ mod tests { async fn group_error_skips_to_next() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); group0.abort(moq_net::Error::Cancel).unwrap(); @@ -1943,8 +1955,8 @@ mod tests { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); write_group(&mut track, 0, &[ts(0)]); @@ -1973,8 +1985,8 @@ mod tests { async fn empty_group_advances() { let mut track = track_producer("test", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); group0.finish().unwrap(); @@ -1994,8 +2006,8 @@ mod tests { let mut track = track_producer("video", hang::container::track_info()); let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_latency(Latency::max(Duration::from_millis(500))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy) + .with_test_latency(Latency::max(Duration::from_millis(500))); // Write frames using Container::Legacy encoding let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -2039,7 +2051,7 @@ mod tests { let consumer_track = track.subscribe(None); // Latency dwarfs the gap, so only duration coverage can trigger the skip. let mut consumer = - Consumer::new(consumer_track, DurationWire).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, DurationWire).with_test_latency(Latency::max(Duration::from_secs(10))); // Group 0: one frame at ts=0 lasting 33ms, never finished. let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -2080,7 +2092,7 @@ mod tests { let mut track = track_producer("test", None); let consumer_track = track.subscribe(None); let mut consumer = - Consumer::new(consumer_track, DurationWire).with_latency(Latency::max(Duration::from_secs(10))); + Consumer::new(consumer_track, DurationWire).with_test_latency(Latency::max(Duration::from_secs(10))); // Group 0: frame at ts=0 lasting only 10ms, far short of group 1 at 33ms. let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 6b9ea41d76e..47f483e0f78 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -189,8 +189,8 @@ impl Export { /// Set the latency tolerance for each per-track source. /// - /// See [`Consumer::with_latency`](crate::container::Consumer::with_latency) for the - /// per-track skip behavior. Defaults to + /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. + /// Defaults to /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (skip aggressively). pub fn with_latency(mut self, latency: crate::Latency) -> Self { self.latency = latency; diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 1086ead95ed..b3384fb1374 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -129,7 +129,7 @@ async fn import_emits_frames() { // Decode the video track back through the Legacy container. let track = consumer.track(&video_name).unwrap().subscribe(None).await.unwrap(); let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(std::time::Duration::from_secs(1))); + .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); let frame = decoder.read().await.unwrap().expect("a video frame"); assert!(frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. @@ -416,7 +416,7 @@ async fn import_enhanced_av1() { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(std::time::Duration::from_secs(1))); + .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); let frame = decoder.read().await.unwrap().expect("an AV1 frame"); assert!(frame.keyframe); assert_eq!(frame.payload.as_ref(), payload); @@ -519,7 +519,7 @@ async fn import_reports_negative_pts_and_can_resume() { let name = snap.video.renditions.keys().next().unwrap(); let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(std::time::Duration::from_secs(1))); + .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); let frame = decoder.read().await.unwrap().expect("the good frame"); assert_eq!(frame.timestamp.as_millis(), 10); } @@ -552,7 +552,7 @@ async fn import_enhanced_hvc1_applies_composition_time() { let name = snap.video.renditions.keys().next().unwrap(); let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(std::time::Duration::from_secs(1))); + .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); let frame = decoder.read().await.unwrap().expect("a video frame"); assert_eq!(frame.timestamp.as_millis(), 17); } diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index cb3d27a3bca..9a7eef94113 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -167,8 +167,8 @@ impl Export { /// Set the latency tolerance for each per-track source. /// - /// See [`Consumer::with_latency`](crate::container::Consumer::with_latency) for the - /// per-track skip behavior. Defaults to + /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. + /// Defaults to /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (skip aggressively). pub fn with_latency(mut self, latency: crate::Latency) -> Self { self.latency = latency; diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 5769d12df09..59df461e756 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -173,8 +173,8 @@ impl Export { /// Set the latency tolerance for each per-track source. /// - /// See [`Consumer::with_latency`](crate::container::Consumer::with_latency) for the - /// per-track skip behavior. Defaults to + /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. + /// Defaults to /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (skip aggressively). pub fn with_latency(mut self, latency: crate::Latency) -> Self { self.latency = latency; diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 39e24733335..6273aded154 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -249,7 +249,7 @@ impl ExportSource { .media .take() .expect("media present until the subscription resolves"); - self.state = SourceState::Active(Box::new(Consumer::new(track, media).with_latency(self.latency))); + self.state = SourceState::Active(Box::new(Consumer::new(track, media))); } loop { diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index e410afd27fe..1d8bc2a2fff 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -366,8 +366,8 @@ impl Export { /// Set the latency tolerance for each per-track source. /// - /// See [`Consumer::with_latency`](crate::container::Consumer::with_latency) for the - /// per-track skip behavior. Defaults to + /// See [`Consumer`](crate::container::Consumer) for the per-track skip behavior. + /// Defaults to /// [`Latency::REAL_TIME`](crate::Latency::REAL_TIME) (skip aggressively). pub fn with_latency(mut self, latency: crate::Latency) -> Self { self.latency = latency; diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 24c50ff066b..ec86c4a766d 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -650,7 +650,7 @@ async fn export_scte35_roundtrip() { let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let frame = scte_reader .read() .await @@ -746,7 +746,7 @@ async fn export_pes_verbatim_roundtrip() { let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let frame = reader .read() .await @@ -815,7 +815,7 @@ async fn scte35_without_video_export_is_rejected() { async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec> { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -1132,7 +1132,7 @@ fn scte_track(snap: &crate::catalog::hang::Catalog) -> Option Vec<(Vec, Timestamp)> { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let mut cues = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index a530cff802e..d5a73572a0a 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2647,7 +2647,7 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); import.finish().unwrap(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(RECORDING_LATENCY)); + let mut reader = Consumer::new(track, Container::Legacy).with_test_latency(Latency::max(RECORDING_LATENCY)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -2880,7 +2880,7 @@ mod test { .clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(RECORDING_LATENCY)); + .with_test_latency(Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { @@ -3691,7 +3691,7 @@ mod test { .await .unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(Latency::max(RECORDING_LATENCY)); + .with_test_latency(Latency::max(RECORDING_LATENCY)); let published = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await; assert!( matches!(published, Ok(Ok(Some(_)))), @@ -3946,7 +3946,7 @@ mod test { let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(RECORDING_LATENCY)); + let mut reader = Consumer::new(track, Container::Legacy).with_test_latency(Latency::max(RECORDING_LATENCY)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -4091,7 +4091,7 @@ mod test { assert_eq!(track.pid, DATA_PID, "recorded the original PID"); let track = consumer.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_latency(Latency::max(RECORDING_LATENCY)); + let mut reader = Consumer::new(track, Container::Legacy).with_test_latency(Latency::max(RECORDING_LATENCY)); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("verbatim read timed out") diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 2d86ff83401..0cff4d1ed99 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -152,7 +152,7 @@ async fn import_opus_frames() { let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -348,7 +348,7 @@ async fn survives_midstream_join() { // anchors the one and only group. let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { frames.push(frame); @@ -393,7 +393,7 @@ async fn kyrion_dirtystart_extracts_real_cues() { .expect("scte35 track"); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_latency(crate::Latency::max(RECORDING_LATENCY)); + .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); let mut cues = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { cues.push((frame.payload.to_vec(), frame.timestamp)); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 4ba34417bf5..a1ff93a6da6 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -50,7 +50,7 @@ impl Consumer { // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(&catalog.container)?; - let track = moq_mux::container::Consumer::new(track, container).with_latency(config.latency); + let track = moq_mux::container::Consumer::new(track, container); Ok(Self { decoder, diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 0717fd46bf5..9b38cf6af0a 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -45,8 +45,8 @@ pub struct Config { /// How far playback may drift from the live edge before a stalled group is /// skipped. Defaults to [`Latency::REAL_TIME`](moq_mux::Latency::REAL_TIME) /// (skip aggressively); set [`Latency::max`](moq_mux::Latency::max) to your - /// playout buffer for a softer skip. Applied to both the initial transport - /// subscription and [`moq_mux::container::Consumer::with_latency`]. + /// playout buffer for a softer skip. Applied to the initial transport + /// subscription and inherited by [`moq_mux::container::Consumer`]. pub latency: moq_mux::Latency, /// Ask the decoder to emit frames at this size (both dimensions even) instead /// of the stream's native one. Best effort: a hardware decoder with a From 91d69ebef1167833f26b0d8b027badbf63bcc278 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 02:57:32 -0700 Subject: [PATCH 22/32] fix(moq-net): wake on new presentation edges --- rs/moq-net/src/model/track.rs | 73 ++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index adebeba2b10..225c5cbd7f7 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2655,20 +2655,45 @@ impl group::Expiry for GroupExpiry { let mut expired = false; let _ = self.state.poll(waiter, |state| { let budget = clamp_latency(latency, state.latency_bound()).max; - let edge = state.live_edge(cap); - expired = state.is_stale(self.sequence, edge, budget); + loop { + let edge = state.live_edge(cap); + expired = state.is_stale(self.sequence, edge, budget); + if expired { + break; + } - if !expired { // A first timestamp can change the presentation-time verdict without - // mutating the track. Register on both the candidate and the current - // presentation edge so either first frame wakes the held reader. + // mutating the track. Register on the candidate and every unstamped + // group that could supersede the current presentation edge. If one + // raced this scan, resolve the edge again before returning Pending. + let mut timestamp_raced = false; if let Some(slot) = state.lookup.get(&self.sequence) { - let _ = slot.group.poll_timestamp(waiter); + let group = &slot.group; + if group.timestamp().is_none() + && group.poll_timestamp(waiter).is_ready() + && group.timestamp().is_some() + { + timestamp_raced = true; + } } - if let Some(edge) = edge.and_then(|edge| edge.presentation) - && let Some(slot) = state.lookup.get(&edge.sequence) - { - let _ = slot.group.poll_timestamp(waiter); + let presentation_sequence = edge + .and_then(|edge| edge.presentation) + .map_or(self.sequence, |edge| edge.sequence.max(self.sequence)); + for slot in state.lookup.values() { + let group = &slot.group; + if slot.visible + && group.sequence > presentation_sequence + && cap.is_none_or(|cap| group.sequence <= cap) + && !group.is_aborted() + && group.poll_timestamp(waiter).is_ready() + && group.timestamp().is_some() + { + timestamp_raced = true; + break; + } + } + if !timestamp_raced { + break; } } @@ -4156,6 +4181,34 @@ mod test { assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); } + #[tokio::test] + async fn a_handed_out_group_wakes_when_a_newer_group_gets_its_first_timestamp() { + let mut producer = track_producer("test", None); + let mut subscriber = + producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut old = producer.append_group().unwrap(); + old.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"old")) + .unwrap(); + + let mut held = subscriber.recv_group().await.unwrap().expect("old group"); + assert!(held.read_frame().await.unwrap().is_some()); + let mut live = producer.append_group().unwrap(); + let pending = tokio::spawn(async move { held.read_frame().await }); + tokio::task::yield_now().await; + assert!(!pending.is_finished(), "the newer group has no timestamp yet"); + + live.write_frame( + Timestamp::from_millis(1000).unwrap(), + bytes::Bytes::from_static(b"live"), + ) + .unwrap(); + tokio::task::yield_now().await; + + assert!(pending.is_finished(), "the new presentation edge wakes the held reader"); + let result = pending.await.unwrap(); + assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); + } + #[tokio::test] async fn a_drained_group_finishes_cleanly_after_the_live_edge_advances() { let mut producer = track_producer("test", None); From 0ebf511ac3dd8ba2a75bc3549a58ccc64f6d3cdb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 03:17:29 -0700 Subject: [PATCH 23/32] fix(moq-net): expire blocked final payloads --- rs/moq-net/src/coding/writer.rs | 5 +++ rs/moq-net/src/ietf/publisher.rs | 68 +++++++++++++++++++++++++++++-- rs/moq-net/src/lite/publisher.rs | 70 ++++++++++++++++++++++++++++++-- rs/moq-net/src/model/group.rs | 7 +++- 4 files changed, 143 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index 8bd4bc13244..c5c235502cc 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -49,6 +49,11 @@ impl Writer { self.buffer.extend_from_slice(bytes); } + /// Whether encoded bytes are still waiting to reach the transport. + pub fn has_pending(&self) -> bool { + !self.buffer.is_empty() + } + /// Poll until the write buffer has fully hit the stream. pub fn poll_flush(&mut self, cx: &mut Context<'_>) -> Poll> { while !self.buffer.is_empty() { diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 4b93444f487..5631581cb77 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -1556,9 +1556,10 @@ impl GroupServe { }; } GroupState::Serve { writer, frame, chunk } => { - // Keep the group guard live while the transport owns a detached - // frame chunk and may be blocked on flow control. - if self.group.poll_expired(waiter) { + // Keep the group guard live while the transport owns buffered data, + // including a final frame already removed from the group cursor. + let pending = writer.has_pending() || frame.is_some() || chunk.is_some(); + if self.group.poll_expired_while_pending(waiter, pending) { self.state = GroupState::Done; return Poll::Ready(Err(Error::Old)); } @@ -1765,6 +1766,67 @@ mod group_priority_test { assert!(matches!(serving.await, Err(Error::Old))); } + + /// The final payload remains guarded after its frame has advanced the group cursor. + #[tokio::test] + async fn blocked_final_transport_chunk_expires_with_the_group() { + tokio::time::pause(); + + let gate = kio::Producer::new(true); + let session = SinkSession::gated_uni(gate.consume()); + let mut track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "test", None); + let mut subscriber = track.subscribe(None); + let mut old = track.append_group().unwrap(); + let mut frame = old + .create_frame(frame::Info { + timestamp: crate::Timestamp::ZERO, + size: 2, + }) + .unwrap(); + frame.write(b"a".as_slice()).unwrap(); + let group = subscriber.recv_group().await.unwrap().expect("old group"); + let mut serve = GroupServe { + session, + msg: ietf::GroupHeader { + track_alias: 0, + group_id: 0, + sub_group_id: 0, + publisher_priority: 0, + flags: Default::default(), + }, + priority: 0, + group, + timescale: Timescale::default(), + version: Version::Draft19, + state: GroupState::Open, + }; + let mut serving = std::pin::pin!(kio::wait(|waiter| serve.poll_serve(waiter))); + assert!( + futures::poll!(serving.as_mut()).is_pending(), + "waiting for the final byte" + ); + + let Ok(mut open) = gate.write() else { + panic!("transport gate closed"); + }; + *open = false; + drop(open); + frame.write(b"b".as_slice()).unwrap(); + frame.finish().unwrap(); + old.finish().unwrap(); + assert!( + futures::poll!(serving.as_mut()).is_pending(), + "the final byte is transport-blocked" + ); + + tokio::time::advance(std::time::Duration::from_secs(1)).await; + let mut edge = track.append_group().unwrap(); + edge.write_frame(crate::Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) + .unwrap(); + edge.finish().unwrap(); + + assert!(matches!(serving.await, Err(Error::Old))); + } } #[cfg(test)] diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 4a36212c121..96c0194d304 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2966,9 +2966,10 @@ impl GroupServe { } let outcome = 'serve: { - // Keep the group guard live while the transport owns a detached - // frame chunk and may be blocked on flow control. - if self.group.poll_expired(waiter) { + // Keep the group guard live while the transport owns buffered data, + // including a final frame already removed from the group cursor. + let pending = writer.has_pending() || frame.is_some() || chunk.is_some(); + if self.group.poll_expired_while_pending(waiter, pending) { break 'serve Err(Error::Old); } // The peer closing first cancels the group. @@ -3265,6 +3266,69 @@ mod serve_group_test { assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); } + /// The final payload remains guarded after its frame has advanced the group cursor. + #[tokio::test] + async fn blocked_final_transport_chunk_expires_with_the_group() { + tokio::time::pause(); + + let gate = kio::Producer::new(true); + let session = SinkSession::gated_uni(gate.consume()); + let log = session.log.clone(); + let track_priority = kio::Producer::new(0u8); + let subscription = Subscription { + session, + id: 0, + track_name: "test".into(), + priority: PriorityQueue::default(), + track_priority: track_priority.consume(), + track_priority_seen: 0, + ordered: false, + version: Version::Lite06Wip, + timescale: Some(crate::Timescale::default()), + }; + + let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); + let mut subscriber = track.subscribe(None); + let mut old = track.append_group().unwrap(); + let mut frame = old + .create_frame(frame::Info { + timestamp: Timestamp::ZERO, + size: 2, + }) + .unwrap(); + frame.write(b"a".as_slice()).unwrap(); + let group = subscriber.recv_group().await.unwrap().expect("old group"); + + let handle = subscription.priority.insert(Priority::new(0, 0, 0)); + let mut serve = std::pin::pin!(subscription.serve_group(0, 0, handle, group)); + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "waiting for the final byte" + ); + + let Ok(mut open) = gate.write() else { + panic!("transport gate closed"); + }; + *open = false; + drop(open); + frame.write(b"b".as_slice()).unwrap(); + frame.finish().unwrap(); + old.finish().unwrap(); + assert!( + futures::poll!(serve.as_mut()).is_pending(), + "the final byte is transport-blocked" + ); + + tokio::time::advance(std::time::Duration::from_secs(1)).await; + let mut edge = track.append_group().unwrap(); + edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) + .unwrap(); + edge.finish().unwrap(); + + assert!(matches!(serve.await, Err(Error::Old))); + assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); + } + /// A subscription group keeps checking latency while transport stream credit is /// exhausted, so returning credit is reserved for content that is still live. #[tokio::test] diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 27e3bd55749..39f01736439 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1020,8 +1020,13 @@ impl Consumer { /// Check the parent subscription while a wire publisher drains detached payload. pub(crate) fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { + self.poll_expired_while_pending(waiter, false) + } + + /// Keep checking expiry while a wire publisher still owns buffered group data. + pub(crate) fn poll_expired_while_pending(&mut self, waiter: &kio::Waiter, pending: bool) -> bool { if !self.expired - && self.expiry_pending() + && (pending || self.expiry_pending()) && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) { self.expired = true; From 8158a551183d0661e7e8cdb4c80e7cff7d9b0e91 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 17:39:42 -0700 Subject: [PATCH 24/32] fix(moq-net): only expire a group whose read would park Post-handoff expiry judged every read, which cost two things. Semantics: under the default real-time budget, the arrival of the next group is exactly what makes the current one no longer newest, so a reader draining a buffered group lost its tail every time. That is not what the budget is for. It bounds a group that has *stalled* while the live edge moved on, so judge only a read that found nothing and is about to park. A group with frames in hand now always drains to its end. Cost: evaluating the policy walks the track's group cache under the track lock, which every subscriber of that track shares, and it also disabled the prefetch fast path in `read_frame`. Reading 20k frames with 50 groups cached went from 13ns/frame to 998ns/frame. Off the ready path it is back to 14ns. The wire publishers keep the check where the group cursor cannot see the stall: parking on transport flow control, so a blocked stream cannot pin credit for content that has gone stale. Everything else they park on (the next frame, the current frame's tail) now reports expiry itself, so the per-pass check and `Writer::has_pending` are gone. Also stop pinning track demand. `GroupExpiry` held a `kio::Consumer` of the track state, which joins the consumer count that `broadcast::Demand` reads, so a handed-out group kept an upstream subscription open past the last real subscriber. A relay serves one for the life of its stream. It holds a `ConsumerWeak` now, which needed a `poll` on that type. Co-Authored-By: Claude Opus 5 --- rs/kio/src/weak.rs | 26 +++++++ rs/moq-net/src/coding/writer.rs | 5 -- rs/moq-net/src/ietf/publisher.rs | 29 +++++--- rs/moq-net/src/lite/publisher.rs | 28 +++++--- rs/moq-net/src/model/frame.rs | 120 +++++++++++++++++-------------- rs/moq-net/src/model/group.rs | 43 ++++++++--- rs/moq-net/src/model/origin.rs | 34 +++++---- rs/moq-net/src/model/track.rs | 36 ++++++---- 8 files changed, 208 insertions(+), 113 deletions(-) diff --git a/rs/kio/src/weak.rs b/rs/kio/src/weak.rs index eb1bf8f8f56..2185871b1c2 100644 --- a/rs/kio/src/weak.rs +++ b/rs/kio/src/weak.rs @@ -275,6 +275,32 @@ impl ConsumerWeak { } } + /// Poll the shared state with a closure, exactly as [`Consumer::poll`] does. + /// + /// Available here so a watcher can register for changes without joining the + /// consumer count, which is what [`Producer::unused`](crate::Producer::unused) + /// keys off. + pub fn poll(&self, waiter: &Waiter, mut f: F) -> Poll>> + where + F: FnMut(&Ref<'_, T>) -> Poll, + { + let state = self.state.lock(); + let consumer_state = Ref { state }; + + if let Poll::Ready(res) = f(&consumer_state) { + return Poll::Ready(Ok(res)); + } + + if consumer_state.state.closed { + return Poll::Ready(Err(consumer_state)); + } + + let mut state = consumer_state.state; + waiter.register(&mut state.waiters_value); + + Poll::Pending + } + /// Returns `true` if the channel has been closed. pub fn is_closed(&self) -> bool { self.state.lock().closed diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index c5c235502cc..8bd4bc13244 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -49,11 +49,6 @@ impl Writer { self.buffer.extend_from_slice(bytes); } - /// Whether encoded bytes are still waiting to reach the transport. - pub fn has_pending(&self) -> bool { - !self.buffer.is_empty() - } - /// Poll until the write buffer has fully hit the stream. pub fn poll_flush(&mut self, cx: &mut Context<'_>) -> Poll> { while !self.buffer.is_empty() { diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 5631581cb77..47afe36ad96 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -1556,13 +1556,6 @@ impl GroupServe { }; } GroupState::Serve { writer, frame, chunk } => { - // Keep the group guard live while the transport owns buffered data, - // including a final frame already removed from the group cursor. - let pending = writer.has_pending() || frame.is_some() || chunk.is_some(); - if self.group.poll_expired_while_pending(waiter, pending) { - self.state = GroupState::Done; - return Poll::Ready(Err(Error::Old)); - } // The peer closing first cancels the group. if writer.poll_closed(&mut cx).is_ready() { self.state = GroupState::Done; @@ -1573,7 +1566,16 @@ impl GroupServe { match writer.poll_flush(&mut cx) { Poll::Ready(Ok(())) => {} Poll::Ready(Err(err)) => break 'serve Err(err), - Poll::Pending => return Poll::Pending, + // Parking on the transport is the one stall the group cursor cannot + // see, and the only place a served group applies the drift budget: + // flow control must not pin a stream that has gone stale. `true` + // because the transport still owns bytes the cursor has released. + Poll::Pending => { + if self.group.poll_expired_while_pending(waiter, true) { + break 'serve Err(Error::Old); + } + return Poll::Pending; + } } if let Some(pending) = chunk { match writer.poll_write(&mut cx, pending) { @@ -1583,7 +1585,16 @@ impl GroupServe { } } Poll::Ready(Err(err)) => break 'serve Err(err), - Poll::Pending => return Poll::Pending, + // Parking on the transport is the one stall the group cursor cannot + // see, and the only place a served group applies the drift budget: + // flow control must not pin a stream that has gone stale. `true` + // because the transport still owns bytes the cursor has released. + Poll::Pending => { + if self.group.poll_expired_while_pending(waiter, true) { + break 'serve Err(Error::Old); + } + return Poll::Pending; + } } } else if let Some(pending) = frame { match pending.poll_read_chunk(waiter) { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 96c0194d304..990f74ebaf5 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2966,12 +2966,6 @@ impl GroupServe { } let outcome = 'serve: { - // Keep the group guard live while the transport owns buffered data, - // including a final frame already removed from the group cursor. - let pending = writer.has_pending() || frame.is_some() || chunk.is_some(); - if self.group.poll_expired_while_pending(waiter, pending) { - break 'serve Err(Error::Old); - } // The peer closing first cancels the group. if writer.poll_closed(&mut cx).is_ready() { break 'serve Err(Error::Cancel); @@ -2980,7 +2974,16 @@ impl GroupServe { match writer.poll_flush(&mut cx) { Poll::Ready(Ok(())) => {} Poll::Ready(Err(err)) => break 'serve Err(err), - Poll::Pending => return Poll::Pending, + // Parking on the transport is the one stall the group cursor cannot + // see, and the only place a served group applies the drift budget: + // flow control must not pin a stream that has gone stale. `true` + // because the transport still owns bytes the cursor has released. + Poll::Pending => { + if self.group.poll_expired_while_pending(waiter, true) { + break 'serve Err(Error::Old); + } + return Poll::Pending; + } } if let Some(pending) = chunk { match writer.poll_write(&mut cx, pending) { @@ -2990,7 +2993,16 @@ impl GroupServe { } } Poll::Ready(Err(err)) => break 'serve Err(err), - Poll::Pending => return Poll::Pending, + // Parking on the transport is the one stall the group cursor cannot + // see, and the only place a served group applies the drift budget: + // flow control must not pin a stream that has gone stale. `true` + // because the transport still owns bytes the cursor has released. + Poll::Pending => { + if self.group.poll_expired_while_pending(waiter, true) { + break 'serve Err(Error::Old); + } + return Poll::Pending; + } } } else if let Some(pending) = frame { match pending.poll_read_chunk(waiter) { diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index ad4f54ac4c4..66f668e3f2a 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -509,6 +509,11 @@ impl Consumer { } } + /// Evaluate the parent subscription's drift budget. + /// + /// Only called once a read has nothing buffered to return: the budget bounds a + /// *stalled* payload, so bytes already in hand are always drained rather than + /// truncated. `self.expired` is sticky, so the answer is only ever computed once. fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool { if self.expired || self.read_idx >= self.size() { return self.expired; @@ -535,10 +540,10 @@ impl Consumer { /// /// Returns `None` once the frame is finished and all bytes have been consumed. pub fn poll_read_chunk(&mut self, waiter: &kio::Waiter) -> Poll>> { - if self.poll_expired(waiter) { + if self.expired { return Poll::Ready(Err(Error::Old)); } - match &self.source { + let buf = match &self.source { Source::Complete(bytes) => { if self.read_idx >= bytes.len() { return Poll::Ready(Ok(None)); @@ -546,38 +551,42 @@ impl Consumer { let out = bytes.slice(self.read_idx..); self.read_idx = bytes.len(); self.stats.bytes(out.len() as u64); - Poll::Ready(Ok(Some(out))) + return Poll::Ready(Ok(Some(out))); } - Source::Partial(buf) => { - let buf = buf.clone(); - let size = self.info.size as usize; - loop { - let written = buf.written(Ordering::Acquire); - if written > self.read_idx { - let out = buf.slice(self.read_idx, written); - self.read_idx = written; - self.stats.bytes(out.len() as u64); - return Poll::Ready(Ok(Some(out))); - } - if written >= size { - return Poll::Ready(Ok(None)); - } - let read_idx = self.read_idx; - // Park on the group's channel; the producer notifies it on each write and - // on abort. Re-check the atomic on wake. - ready!(poll_state(&self.state, waiter, |state| { - if let Some(err) = &state.abort { - return Poll::Ready(Err(err.clone())); - } - let w = buf.written(Ordering::Acquire); - if w > read_idx || w >= size { - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - })?); - } + Source::Partial(buf) => buf.clone(), + }; + + let size = self.info.size as usize; + loop { + let written = buf.written(Ordering::Acquire); + if written > self.read_idx { + let out = buf.slice(self.read_idx, written); + self.read_idx = written; + self.stats.bytes(out.len() as u64); + return Poll::Ready(Ok(Some(out))); + } + if written >= size { + return Poll::Ready(Ok(None)); } + // Nothing buffered and the frame isn't finished: this park is the stall the + // drift budget exists to bound, and the only place it can apply. + if self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } + let read_idx = self.read_idx; + // Park on the group's channel; the producer notifies it on each write and + // on abort. Re-check the atomic on wake. + ready!(poll_state(&self.state, waiter, |state| { + if let Some(err) = &state.abort { + return Poll::Ready(Err(err.clone())); + } + let w = buf.written(Ordering::Acquire); + if w > read_idx || w >= size { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + })?); } } @@ -588,36 +597,39 @@ impl Consumer { /// Poll for all remaining bytes, resolving once the frame is finished. pub fn poll_read_all(&mut self, waiter: &kio::Waiter) -> Poll> { - if self.poll_expired(waiter) { + if self.expired { return Poll::Ready(Err(Error::Old)); } - match &self.source { + let buf = match &self.source { Source::Complete(bytes) => { let out = bytes.slice(self.read_idx..); self.read_idx = bytes.len(); self.stats.bytes(out.len() as u64); - Poll::Ready(Ok(out)) - } - Source::Partial(buf) => { - let buf = buf.clone(); - let size = self.info.size as usize; - let read_idx = self.read_idx; - ready!(poll_state(&self.state, waiter, |state| { - if let Some(err) = &state.abort { - return Poll::Ready(Err(err.clone())); - } - if buf.written(Ordering::Acquire) >= size { - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - })?); - let out = buf.slice(read_idx, size); - self.read_idx = size; - self.stats.bytes(out.len() as u64); - Poll::Ready(Ok(out)) + return Poll::Ready(Ok(out)); } + Source::Partial(buf) => buf.clone(), + }; + + let size = self.info.size as usize; + let read_idx = self.read_idx; + // Waiting on the rest of the payload is a stall; see `poll_read_chunk`. + if buf.written(Ordering::Acquire) < size && self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); } + ready!(poll_state(&self.state, waiter, |state| { + if let Some(err) = &state.abort { + return Poll::Ready(Err(err.clone())); + } + if buf.written(Ordering::Acquire) >= size { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + })?); + let out = buf.slice(read_idx, size); + self.read_idx = size; + self.stats.bytes(out.len() as u64); + Poll::Ready(Ok(out)) } /// Return all remaining bytes, blocking until the frame is finished. diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 39f01736439..1ce86aff83c 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1023,6 +1023,24 @@ impl Consumer { self.poll_expired_while_pending(waiter, false) } + /// Apply the drift budget to a read that found nothing and is about to park. + /// + /// A group with frames in hand is always drained to its end: the budget bounds a + /// group that has *stalled* while the live edge moved on, not one whose reader is + /// merely slower than the wire. Judging every read instead would truncate the tail + /// of every group under the default real-time budget, since the arrival of the next + /// group is exactly what makes the current one no longer newest. + /// + /// Keeping it off the ready path also keeps it off the hot path: evaluating the + /// policy walks the track's group cache under its lock, which is shared by every + /// subscriber of that track. + fn poll_expired_if_blocked(&mut self, waiter: &kio::Waiter, res: Poll>) -> Poll> { + if res.is_pending() && self.poll_expired(waiter) { + return Poll::Ready(Err(Error::Old)); + } + res + } + /// Keep checking expiry while a wire publisher still owns buffered group data. pub(crate) fn poll_expired_while_pending(&mut self, waiter: &kio::Waiter, pending: bool) -> bool { if !self.expired @@ -1134,7 +1152,7 @@ impl Consumer { /// Returns None if the group is finished and the index is out of range, or the cursor /// passed the [`Self::end_at`] cap. pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { - if self.poll_expired(waiter) { + if self.expired { return Poll::Ready(Err(Error::Old)); } let stats = self.stats.clone(); @@ -1142,7 +1160,7 @@ impl Consumer { .expiry .as_ref() .map(|policy| frame::Expiry::new(policy.clone(), self.stale_stats.clone(), self.stale_counted.clone())); - match &mut self.inner { + let res = match &mut self.inner { ConsumerKind::Plain(plain) => plain.poll_next_frame(waiter, &stats, expiry), ConsumerKind::Spliced(spliced) => { // The per-route copies underneath are untagged, so meter the spliced @@ -1153,16 +1171,17 @@ impl Consumer { } Poll::Ready(Ok(res.map(|frame| frame.with_meter(stats)))) } - } + }; + self.poll_expired_if_blocked(waiter, res) } /// Read the next frame (timestamp and payload) all at once, without blocking. pub fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { - if self.poll_expired(waiter) { + if self.expired { return Poll::Ready(Err(Error::Old)); } let stats = self.stats.clone(); - match &mut self.inner { + let res = match &mut self.inner { ConsumerKind::Plain(plain) => plain.poll_read_frame(waiter, &stats), ConsumerKind::Spliced(spliced) => { let res = ready!(spliced.poll_read_frame(waiter))?; @@ -1172,12 +1191,15 @@ impl Consumer { } Poll::Ready(Ok(res)) } - } + }; + self.poll_expired_if_blocked(waiter, res) } /// Read the next frame (timestamp and payload) all at once. pub async fn read_frame(&mut self) -> Result> { - if self.expiry.is_none() + // A prefetched frame is already buffered, so the drift budget (which only judges + // a read that would park) can never apply to it. + if !self.expired && let ConsumerKind::Plain(plain) = &mut self.inner { // Serve from the prefetched batch without building a future or allocating a waker. @@ -1193,13 +1215,14 @@ impl Consumer { /// Poll for the final number of frames in the group. pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { - if self.poll_expired(waiter) { + if self.expired { return Poll::Ready(Err(Error::Old)); } - match &mut self.inner { + let res = match &mut self.inner { ConsumerKind::Plain(plain) => plain.poll(waiter, |state| state.poll_finished()), ConsumerKind::Spliced(spliced) => spliced.poll_finished(waiter), - } + }; + self.poll_expired_if_blocked(waiter, res) } /// Block until the group is finished, returning the number of frames in the group. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 14403c7f932..582ad5154bd 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3932,33 +3932,43 @@ mod tests { settle().await; let mut sub = subscribing.await.unwrap(); + // A frame whose payload is still in flight: the readers drain what has landed + // and then park on the rest, which is the stall the drift budget bounds. A + // group whose frames are all in hand is drained instead, so it would never + // reach expiry here. let mut group = producer.append_group().unwrap(); - for payload in [b"aa", b"bb", b"cc"] { - group.write_frame(Timestamp::ZERO, Bytes::from_static(payload)).unwrap(); - } - group.finish().unwrap(); + let mut writing = group + .create_frame(crate::frame::Info { + timestamp: Timestamp::ZERO, + size: 4, + }) + .unwrap(); + writing.write(Bytes::from_static(b"aa")).unwrap(); let mut reading = sub.recv_group().await.unwrap().expect("first group"); - let mut first = reading.next_frame().await.unwrap().expect("first frame"); - assert_eq!(first.read_all().await.unwrap(), Bytes::from_static(b"aa")); let mut clone = reading.clone(); + let mut first = reading.next_frame().await.unwrap().expect("first frame"); + let mut second = clone.next_frame().await.unwrap().expect("cloned frame"); + assert_eq!(first.read_chunk().await.unwrap(), Some(Bytes::from_static(b"aa"))); + assert_eq!(second.read_chunk().await.unwrap(), Some(Bytes::from_static(b"aa"))); let mut edge = producer.append_group().unwrap(); edge.write_frame(Timestamp::from_millis(1000).unwrap(), Bytes::from_static(b"edge")) .unwrap(); edge.finish().unwrap(); - assert!(matches!(reading.next_frame().await, Err(crate::Error::Old))); - assert!(matches!(clone.next_frame().await, Err(crate::Error::Old))); + assert!(matches!(first.read_chunk().await, Err(crate::Error::Old))); + assert!(matches!(second.read_chunk().await, Err(crate::Error::Old))); let report = registry.report(); let entry = report.traffic.iter().find(|e| e.path.as_str() == "demo").unwrap(); assert_eq!(entry.publisher.groups, 1, "the handed-out group was delivered"); - assert_eq!(entry.publisher.frames, 1, "the returned frame was delivered"); - assert_eq!(entry.publisher.bytes, 2, "the returned payload was delivered"); assert_eq!(entry.publisher.stale.groups, 0, "the group is not counted twice"); - assert_eq!(entry.publisher.stale.frames, 2, "only the unread frames are stale"); - assert_eq!(entry.publisher.stale.bytes, 4, "only the unread payload is stale"); + assert_eq!(entry.publisher.stale.frames, 0, "no whole frame went unread"); + assert_eq!( + entry.publisher.stale.bytes, 2, + "the undelivered half of the payload is stale, counted by one reader only" + ); } /// `Subscriber::read_frame` collapses a group to its first frame. The paths it diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 225c5cbd7f7..752a9b798d2 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2049,7 +2049,7 @@ impl Consumer { }; let sequence = group.sequence; group.with_expiry(Arc::new(GroupExpiry { - state: state.clone(), + state: state.weak(), subscription, cap, bound, @@ -2630,7 +2630,12 @@ struct Drift { /// Keeps one handed-out group tied to the subscription whose cursor selected it. struct GroupExpiry { - state: kio::Consumer, + /// Weak so a handed-out group never joins the track's consumer count, which is what + /// [`crate::broadcast::Demand`] reads to decide the track still has readers. A group + /// outlives the cursor that produced it (a relay serves one for the life of its + /// stream), and pinning demand on that would hold an upstream subscription open past + /// the last real subscriber. + state: kio::ConsumerWeak, subscription: kio::Consumer, cap: kio::Consumer>, bound: Option, @@ -2836,7 +2841,7 @@ impl PlainSubscriber { fn with_expiry(&self, group: group::Consumer) -> group::Consumer { let sequence = group.sequence; group.with_expiry(Arc::new(GroupExpiry { - state: self.state.clone(), + state: self.state.weak(), subscription: self.subscription.consume(), cap: self.drift_cap.consume(), bound: None, @@ -3529,7 +3534,7 @@ mod test { /// A bounded replay window for tests whose subject requires every buffered group. fn replay() -> Subscription { - Subscription::default().with_latency(Latency::max(Duration::from_secs(30))) + Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(30))) } /// Helper: count live cached groups in state. @@ -3976,13 +3981,13 @@ mod test { // waited for longer than the publisher keeps it. The subscriber's own preference // is stored verbatim, so what it asked for stays readable. let mut subscriber = - producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(10)))); assert_eq!(subscriber.subscription().latency.max, Duration::from_secs(10)); assert_eq!(producer.subscription().unwrap().latency.max, Duration::from_secs(2)); // A budget within the cache is left alone, and ZERO (skip immediately) stays ZERO. subscriber - .update(Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))) + .update(Subscription::default().with_latency(crate::Latency::max(Duration::from_millis(500)))) .unwrap(); assert_eq!(producer.subscription().unwrap().latency.max, Duration::from_millis(500)); @@ -4050,7 +4055,7 @@ mod test { #[test] fn latency_max_clamped_via_every_update_path() { let producer = track_producer("test", Info::default().with_latency_max(Duration::from_secs(2))); - let over = Subscription::default().with_latency(Latency::max(Duration::from_secs(10))); + let over = Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(10))); // The clamp lives in the aggregation, so it applies no matter which entry point // wrote the raw preference. Previously only `Subscriber::update` clamped. @@ -4070,8 +4075,9 @@ mod test { // The aggregate takes the max, then clamps once. Equivalent to clamping each // subscriber first, since `min` distributes over `max`. - let _a = producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); - let _b = producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let _a = + producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_millis(500)))); + let _b = producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(10)))); assert_eq!(producer.subscription().unwrap().latency.max, Duration::from_secs(2)); } @@ -4124,7 +4130,7 @@ mod test { // Two seconds of tolerance keeps the groups presenting within 2s of the live // edge (2s, 3s, 4s) and drops the two below it. let mut subscriber = - producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(2)))); + producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(2)))); assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); } @@ -4139,7 +4145,7 @@ mod test { } let mut subscriber = - producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_millis(1500)))); + producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_millis(1500)))); assert_eq!(drain(&mut subscriber), vec![2, 3]); } @@ -4185,7 +4191,7 @@ mod test { async fn a_handed_out_group_wakes_when_a_newer_group_gets_its_first_timestamp() { let mut producer = track_producer("test", None); let mut subscriber = - producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_millis(500)))); let mut old = producer.append_group().unwrap(); old.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"old")) .unwrap(); @@ -4277,7 +4283,7 @@ mod test { assert!(!pending.is_finished(), "the real-time budget skips the old frame"); control - .update(Subscription::default().with_latency(Latency::max(Duration::from_secs(2)))) + .update(Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(2)))) .unwrap(); let frame = pending @@ -4298,7 +4304,7 @@ mod test { append_at(&mut producer, 1000); let mut subscriber = - producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + producer.subscribe(Subscription::default().with_latency(crate::Latency::max(Duration::from_secs(10)))); assert_eq!(drain(&mut subscriber), vec![1]); } @@ -4318,7 +4324,7 @@ mod test { let mut patient = producer.subscribe( Subscription::default() .with_start(Position::group(0)) - .with_latency(Latency::max(Duration::from_secs(10))), + .with_latency(crate::Latency::max(Duration::from_secs(10))), ); patient.start_at(0); assert_eq!(drain(&mut patient), vec![0, 1, 2, 3]); From a901ba98eecd41eebf4dd3e4a761793c33bb5c50 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 22:33:54 -0700 Subject: [PATCH 25/32] fix(moq-net): measure drift from the reader, and end a drained group Two things a group boundary got wrong. A group was measured against the live edge from its *first* frame, so a 2s GOP read live always looked 2s behind, however current its reader was. Any budget shorter than one GOP therefore dropped the tail of every GOP, which is why the fixtures here wanted 30-second windows. A group is not late because it opened long ago; what can be late is the content its reader has yet to take. Both measures now start from the cursor: the next frame it would read (or the group's newest, once it has taken them all) for presentation time, and the group's last write for wall-clock time. A group nobody has started reading still sits at its own start, so group selection is unchanged. And expiry always reported `Error::Old`. A group-level read only parks once the cursor has taken every frame, so expiring there loses nothing: what it was waiting for was the producer's FIN, and a group abandoned at its own end is indistinguishable from one that ended there. It now ends cleanly, and `Old` is reserved for a cursor that still holds unread content (a half-read payload, a publisher with buffered frames). That matters most at a relay, which was resetting the downstream stream for a group it had delivered in full: the FIN and the next group's first frame are separate streams, so the reader is routinely parked at the group's end when the verdict is taken. Both regressions are covered, and each new test fails with only its own half reverted. Co-Authored-By: Claude Opus 5 --- rs/moq-net/src/model/frame.rs | 8 +- rs/moq-net/src/model/group.rs | 130 +++++++++++++++++++++----- rs/moq-net/src/model/resume.rs | 3 +- rs/moq-net/src/model/track.rs | 165 ++++++++++++++++++++++++++++++--- 4 files changed, 272 insertions(+), 34 deletions(-) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 66f668e3f2a..9241ec94bf2 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -521,7 +521,13 @@ impl Consumer { let Some(expiry) = &self.expiry else { return false; }; - if !expiry.policy.is_expired(waiter) { + // A half-read payload sits at its own frame's presentation time, not at the + // start of the group that carries it. + let at = group::Position { + presentation: Some(self.info.timestamp), + activity: self.state.read().activity, + }; + if !expiry.policy.is_expired(waiter, at) { return false; } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 1ce86aff83c..33ced586ced 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -132,6 +132,16 @@ pub(crate) struct GroupState { // empty group has not presented anything yet. timestamp: Option, + // The newest frame's timestamp: the group's presentation end so far. A reader + // that has taken every frame sits here, which is what a drift budget measures it + // against. Kept alongside `timestamp` for the same reasons. + latest: Option, + + // When the group last accepted a frame. The wall-clock half of the same question: + // a group is silent from here, not from whenever it was created. `None` while it + // has produced nothing, where the track's arrival time is the only thing to go on. + pub(crate) activity: Option, + // Once finalized, the total number of frames the group will ever contain. Recorded // at finish so the count outlives an abort that clears the cache. pub(crate) fin: Option, @@ -235,9 +245,24 @@ impl GroupState { } } - /// Record where the group starts in presentation time, on the first frame only. + /// Record where the group starts and currently ends in presentation time, plus + /// when it last produced. `timestamp` is the first frame only; the rest track + /// every frame, so a reader that has caught up can be measured where it sits. fn stamp(&mut self, timestamp: Timestamp) { self.timestamp.get_or_insert(timestamp); + self.latest = Some(timestamp); + self.activity = Some(web_async::time::Instant::now()); + } + + /// Where a cursor at `index` sits in presentation time: the next frame it would + /// read, or the newest frame in the group once it has taken them all. + /// + /// `None` only while the group has presented nothing at all. + fn position(&self, index: usize) -> Option { + self.frames + .get(index.saturating_sub(self.offset)) + .map(|frame| frame.timestamp) + .or(self.latest) } /// Evict completed frames from the front until within the byte budget. @@ -882,9 +907,9 @@ pub struct Consumer { /// Subscriber-specific policy for expiring a group after it was handed out. pub(crate) trait Expiry: Send + Sync { - /// Return whether the group is stale, registering `waiter` for anything that - /// could change the answer while it remains live. - fn is_expired(&self, waiter: &kio::Waiter) -> bool; + /// Return whether a reader sitting at `at` is stale, registering `waiter` for + /// anything that could change the answer while the group remains live. + fn is_expired(&self, waiter: &kio::Waiter, at: Position) -> bool; } // `Plain` is the hot path and carries an inline frame prefetch, so boxing it to even the @@ -1028,28 +1053,43 @@ impl Consumer { /// A group with frames in hand is always drained to its end: the budget bounds a /// group that has *stalled* while the live edge moved on, not one whose reader is /// merely slower than the wire. Judging every read instead would truncate the tail - /// of every group under the default real-time budget, since the arrival of the next - /// group is exactly what makes the current one no longer newest. + /// of every group, since the arrival of the next group is exactly what makes the + /// current one no longer newest. /// /// Keeping it off the ready path also keeps it off the hot path: evaluating the /// policy walks the track's group cache under its lock, which is shared by every /// subscriber of that track. - fn poll_expired_if_blocked(&mut self, waiter: &kio::Waiter, res: Poll>) -> Poll> { - if res.is_pending() && self.poll_expired(waiter) { - return Poll::Ready(Err(Error::Old)); - } - res + /// + /// `Some(false)` ends the group cleanly and `Some(true)` fails it with + /// [`Error::Old`]; see [`Self::expired_truncates`]. + fn poll_expired_if_blocked(&mut self, waiter: &kio::Waiter) -> Option { + self.poll_expired(waiter).then(|| self.expired_truncates()) + } + + /// Whether giving up on this group now loses the reader anything. + /// + /// A frame-level read only parks once the cursor has taken every frame the group + /// holds, so expiring there costs nothing: the reader got everything that exists, + /// and the group ends rather than fails. What it was still waiting for was the + /// producer's FIN, and a group abandoned at its own end is indistinguishable from + /// one that ended. A cursor that still holds unread content (a wire publisher with + /// buffered frames, a half-read payload) is genuinely truncated and reports it. + fn expired_truncates(&self) -> bool { + let unread = self.unread_content(); + unread.frames > 0 || unread.bytes > 0 } /// Keep checking expiry while a wire publisher still owns buffered group data. pub(crate) fn poll_expired_while_pending(&mut self, waiter: &kio::Waiter, pending: bool) -> bool { - if !self.expired - && (pending || self.expiry_pending()) - && self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter)) - { - self.expired = true; - if !self.stale_counted.swap(true, Ordering::Relaxed) { - self.stale_stats.stale(self.unread_content()); + if !self.expired && (pending || self.expiry_pending()) { + // Resolved here rather than up front: the sticky flag above answers most + // calls without touching the group's state at all. + let at = self.position(); + if self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter, at)) { + self.expired = true; + if !self.stale_counted.swap(true, Ordering::Relaxed) { + self.stale_stats.stale(self.unread_content()); + } } } self.expired @@ -1069,6 +1109,20 @@ impl Consumer { self.expired } + /// Where this cursor sits, for a drift budget to measure it against the live edge. + /// + /// A group is not late because it *started* long ago. What is late is the content + /// its reader has yet to take, so a reader that has drained a group sits at the + /// group's newest frame, not its first. Measuring from the first would make every + /// group one group-duration behind by construction. + pub(crate) fn position(&self) -> Position { + match &self.inner { + ConsumerKind::Plain(plain) => plain.position(), + // A spliced group's route-specific cursors carry their own policy. + ConsumerKind::Spliced(_) => Position::default(), + } + } + /// Whether the group has been aborted (including pool eviction); the abort /// dropped the cached frames, so a held consumer has nothing left to read. /// @@ -1172,7 +1226,11 @@ impl Consumer { Poll::Ready(Ok(res.map(|frame| frame.with_meter(stats)))) } }; - self.poll_expired_if_blocked(waiter, res) + match res.is_pending().then(|| self.poll_expired_if_blocked(waiter)).flatten() { + Some(true) => Poll::Ready(Err(Error::Old)), + Some(false) => Poll::Ready(Ok(None)), + None => res, + } } /// Read the next frame (timestamp and payload) all at once, without blocking. @@ -1192,7 +1250,11 @@ impl Consumer { Poll::Ready(Ok(res)) } }; - self.poll_expired_if_blocked(waiter, res) + match res.is_pending().then(|| self.poll_expired_if_blocked(waiter)).flatten() { + Some(true) => Poll::Ready(Err(Error::Old)), + Some(false) => Poll::Ready(Ok(None)), + None => res, + } } /// Read the next frame (timestamp and payload) all at once. @@ -1222,7 +1284,12 @@ impl Consumer { ConsumerKind::Plain(plain) => plain.poll(waiter, |state| state.poll_finished()), ConsumerKind::Spliced(spliced) => spliced.poll_finished(waiter), }; - self.poll_expired_if_blocked(waiter, res) + match res.is_pending().then(|| self.poll_expired_if_blocked(waiter)).flatten() { + Some(true) => Poll::Ready(Err(Error::Old)), + // The group ended where the cursor stands, so that is its frame count. + Some(false) => Poll::Ready(Ok(self.index())), + None => res, + } } /// Block until the group is finished, returning the number of frames in the group. @@ -1231,7 +1298,28 @@ impl Consumer { } } +/// Where a group cursor sits in presentation and wall-clock time. +/// +/// `None` fields mean the group has presented nothing (an empty or stalled group), +/// where the track falls back to when the group arrived. +#[derive(Clone, Copy, Default)] +pub(crate) struct Position { + pub(crate) presentation: Option, + pub(crate) activity: Option, +} + impl Plain { + fn position(&self) -> Position { + // The prefetch batch was drained under an earlier lock, so the cursor's real + // position is past it. + let index = self.index.saturating_add(self.prefetch.buffered().0 as usize); + let state = self.state.read(); + Position { + presentation: state.position(index), + activity: state.activity, + } + } + /// Whether this cursor still has unread content or may receive another frame. fn expiry_pending(&self) -> bool { if self.capped() { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index e46fea8ad53..98cf8752e23 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2618,7 +2618,8 @@ mod test { .read_frame() .now_or_never() .expect("the newer group changes the verdict"); - assert!(matches!(result, Err(Error::Old)), "the replacement expires: {result:?}"); + // Drained, so the budget ends the replacement rather than truncating it. + assert!(matches!(result, Ok(None)), "the replacement ends: {result:?}"); } /// A replacement that serves the whole group instead of the requested tail still diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 752a9b798d2..d035824184a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -394,7 +394,7 @@ impl TrackState { // The drift budget bounds this the same as the group-level reads: a frame // from a group the subscription has given up on is no fresher for being // read one frame at a time. - if self.is_stale(*sequence, drift.edge, drift.budget) { + if self.is_stale(*sequence, None, drift.edge, drift.budget) { stale.add(slot.group.content()); continue; } @@ -585,7 +585,7 @@ impl TrackState { /// The edge must sit strictly above the candidate. The live edge is never late /// against itself, and backfill or the tail of a rewound timeline can carry a high /// timestamp on a low sequence without being an edge at all. - fn is_stale(&self, sequence: u64, edge: Option, budget: Duration) -> bool { + fn is_stale(&self, sequence: u64, at: Option, edge: Option, budget: Duration) -> bool { let Some(edge) = edge else { return false; }; @@ -593,6 +593,13 @@ impl TrackState { return false; }; + // Where the candidate sits. A group nobody has started reading sits at its own + // start; one already handed out sits wherever its reader got to, so a reader + // that has kept up is not convicted by how long ago its group opened. + let at = at.unwrap_or_default(); + let presentation = at.presentation.or_else(|| slot.group.timestamp()); + let arrived = at.activity.unwrap_or(slot.arrived); + // The anchors were resolved under an earlier lock, so confirm each still names // the same servable incarnation before it convicts a candidate. Failing safe // (delivering) is right, since the next poll resolves fresh anchors. @@ -605,7 +612,7 @@ impl TrackState { || edge .wall .arrived - .checked_duration_since(slot.arrived) + .checked_duration_since(arrived) .is_some_and(|age| age > budget)); let presentation_stale = edge.presentation.is_some_and(|edge| { @@ -614,7 +621,7 @@ impl TrackState { .lookup .get(&edge.sequence) .is_some_and(|live| live.stamp == edge.stamp && !live.group.is_aborted()) - && slot.group.timestamp().is_some_and( + && presentation.is_some_and( |timestamp| matches!(edge.timestamp.checked_sub(timestamp), Ok(age) if Duration::from(age) > budget), ) }); @@ -2643,7 +2650,7 @@ struct GroupExpiry { } impl group::Expiry for GroupExpiry { - fn is_expired(&self, waiter: &kio::Waiter) -> bool { + fn is_expired(&self, waiter: &kio::Waiter, at: group::Position) -> bool { let mut latency = Latency::default(); let _ = self.subscription.poll(waiter, |subscription| { latency = subscription.latency; @@ -2662,7 +2669,7 @@ impl group::Expiry for GroupExpiry { let budget = clamp_latency(latency, state.latency_bound()).max; loop { let edge = state.live_edge(cap); - expired = state.is_stale(self.sequence, edge, budget); + expired = state.is_stale(self.sequence, Some(at), edge, budget); if expired { break; } @@ -2834,7 +2841,7 @@ impl PlainSubscriber { /// for this poll. fn poll_stale(&self, group: &group::Consumer, drift: Drift, waiter: &kio::Waiter) -> Poll> { self.poll(waiter, move |state| { - Poll::Ready(Ok(state.is_stale(group.sequence, drift.edge, drift.budget))) + Poll::Ready(Ok(state.is_stale(group.sequence, None, drift.edge, drift.budget))) }) } @@ -4183,8 +4190,12 @@ mod test { tokio::time::advance(Duration::from_secs(1)).await; append_at(&mut producer, 1000); + // It ends rather than fails: the reader took every frame the group ever had + // (none), so nothing was truncated. What it was waiting for was the producer, + // and a group abandoned where its reader stands looks exactly like one that + // ended there. let result = pending.await.unwrap(); - assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); + assert!(matches!(result, Ok(None)), "the held group ends: {result:?}"); } #[tokio::test] @@ -4211,8 +4222,137 @@ mod test { tokio::task::yield_now().await; assert!(pending.is_finished(), "the new presentation edge wakes the held reader"); + // Drained, so the budget ends the group rather than truncating it. let result = pending.await.unwrap(); - assert!(matches!(result, Err(Error::Old)), "the held group expires: {result:?}"); + assert!(matches!(result, Ok(None)), "the held group ends: {result:?}"); + } + + /// The ordinary live case, at the default real-time budget: 2s GOPs produced one at + /// a time and read as they arrive. The budget must take the live edge without + /// shortening the group the reader is already on, so every frame of every group is + /// delivered and each group *ends* rather than fails at its boundary. + /// + /// Two things would break this. Measuring drift from a group's first frame rather + /// than its reader's position convicts every group the moment its successor opens, + /// since a live reader is always a little behind the edge. And the reader is parked + /// at its group's end when that happens, because the FIN and the next group's first + /// frame are separate events and the wire does not order them. + #[tokio::test] + async fn real_time_reads_a_live_stream_without_truncating_it() { + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + + let gop = |n: u64| { + [ + Timestamp::from_millis(n * 2000).unwrap(), + Timestamp::from_millis(n * 2000 + 1900).unwrap(), + ] + }; + let write = |group: &mut group::Producer, timestamp| { + group.write_frame(timestamp, bytes::Bytes::from_static(b"x")).unwrap(); + }; + + let mut open = producer.append_group().unwrap(); + write(&mut open, gop(0)[0]); + write(&mut open, gop(0)[1]); + let mut reading = subscriber.recv_group().await.unwrap().expect("the live group"); + + let mut read = Vec::new(); + for n in 1..5u64 { + let sequence = reading.sequence; + let mut frames = 0; + while let Some(res) = reading.read_frame().now_or_never() { + match res.expect("no truncation while draining") { + Some(_) => frames += 1, + None => panic!("group {sequence} ended early"), + } + } + read.push((sequence, frames)); + + let next = { + // Parked at the end of the current group: every frame is read and no FIN + // has landed. + let mut end = std::pin::pin!(reading.read_frame()); + assert!(futures::poll!(end.as_mut()).is_pending(), "parked on the FIN"); + + // The next keyframe opens its group. The verdict is taken here, in the + // window before the previous group's FIN arrives. + let mut opened = producer.append_group().unwrap(); + write(&mut opened, gop(n)[0]); + let verdict = futures::poll!(end.as_mut()); + + open.finish().unwrap(); + let res = match verdict { + Poll::Ready(res) => res, + Poll::Pending => end.await, + }; + assert!( + matches!(res, Ok(None)), + "group {sequence} ends at the boundary rather than failing: {res:?}" + ); + opened + }; + let mut next = next; + write(&mut next, gop(n)[1]); + + reading = subscriber.recv_group().await.unwrap().expect("the next live group"); + open = next; + } + + assert_eq!(read, vec![(0, 2), (1, 2), (2, 2), (3, 2)], "every frame of every group"); + } + + /// A budget is spent from where the reader stands, not from where its group opened. + /// + /// A 2s GOP with a 1s budget: the reader has drained to 1900ms when the next group + /// opens at 2000ms, so it is 100ms behind the live edge and well inside what it + /// asked for. A straggling frame of the old group arriving after the new one opened + /// (which is ordinary, the two are separate streams) must still reach it. + /// + /// Measuring from the group's first frame instead makes the drift 2000ms, so a + /// budget shorter than one GOP would drop the tail of every GOP. + #[tokio::test] + async fn a_budget_is_measured_from_the_readers_position() { + let mut producer = track_producer("test", None); + let mut subscriber = + producer.subscribe(Subscription::default().with_latency(Latency::max(Duration::from_secs(1)))); + + let mut open = producer.append_group().unwrap(); + open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"key")) + .unwrap(); + open.write_frame( + Timestamp::from_millis(1900).unwrap(), + bytes::Bytes::from_static(b"tail"), + ) + .unwrap(); + + let mut reading = subscriber.recv_group().await.unwrap().expect("the live group"); + assert!(reading.read_frame().await.unwrap().is_some()); + assert!(reading.read_frame().await.unwrap().is_some()); + + let mut end = std::pin::pin!(reading.read_frame()); + assert!(futures::poll!(end.as_mut()).is_pending(), "parked at 1900ms"); + + // The next GOP opens. The reader is 100ms behind it, inside its 1s budget. + let mut next = producer.append_group().unwrap(); + next.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"key")) + .unwrap(); + assert!( + futures::poll!(end.as_mut()).is_pending(), + "a reader inside its budget is not expired by the next group opening" + ); + + // A straggler from the old group, still within the budget. + open.write_frame( + Timestamp::from_millis(1950).unwrap(), + bytes::Bytes::from_static(b"late"), + ) + .unwrap(); + let late = end.await.expect("the straggler is not truncated"); + assert_eq!( + late.map(|frame| frame.timestamp), + Some(Timestamp::from_millis(1950).unwrap()) + ); } #[tokio::test] @@ -4445,7 +4585,10 @@ mod test { budget: Duration::ZERO, edge: state.live_edge(None), }; - assert!(state.is_stale(0, drift.edge, drift.budget), "stale against a live edge"); + assert!( + state.is_stale(0, None, drift.edge, drift.budget), + "stale against a live edge" + ); drop(state); // The edge dies between resolving it and judging the candidate. @@ -4454,7 +4597,7 @@ mod test { let state = producer.state.read(); assert!( - !state.is_stale(0, drift.edge, drift.budget), + !state.is_stale(0, None, drift.edge, drift.budget), "a vanished edge is no reason to drop what is left" ); } From 880b4f5c787d73d1bac23f1537575a220b0d9902 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 11:20:24 -0700 Subject: [PATCH 26/32] fix(moq-net): box the largest subscribe state `TrackRun` outgrew the other variants once it started carrying the group expiry guards, and the enum is moved on every state transition. Co-Authored-By: Claude Opus 5 --- rs/moq-net/src/lite/publisher.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 990f74ebaf5..818b302cdaa 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1202,8 +1202,9 @@ enum SubscribeState { msg: lite::Subscribe<'static>, subscribing: track::Subscribing, }, - /// Streaming groups and datagrams. - Run(TrackRun), + /// Streaming groups and datagrams. Boxed: by far the largest state, and the enum + /// is moved on every transition. + Run(Box>), /// The track finished: draining the in-flight group streams before the FIN. Drain { children: kio::Tasks, @@ -1358,7 +1359,7 @@ impl SubscribeServe { }; let run = TrackRun::new(sub, track, Bounds::from(&msg), track_priority_tx); - self.state = SubscribeState::Run(run); + self.state = SubscribeState::Run(Box::new(run)); } SubscribeState::Run(run) => { let stream = self.stream.as_mut().expect("stream present"); From 7171b99ebf2a40f194045a36229ec47489ddd54a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 13:27:02 -0700 Subject: [PATCH 27/32] docs(contributing): ask PRs to flag wire behavior, not just API An API diff shows nothing when the change is which content gets delivered, or whether a stream is reset rather than FIN'd. That is still something another implementation has to match, so it belongs in the description and in a review's checklist alongside the public API list. Co-Authored-By: Claude Opus 5 --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da97feaec27..057db04296a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,7 @@ Keep the body short and structured, not narrated: - **Summary**: a few bullets on what changed and why. For a bug fix, state the root cause (see Root Cause First in [CLAUDE.md](CLAUDE.md)). - **Public API changes**: every new/renamed/removed/signature-changed `pub` item in `rs/moq-*` and `js/*`, with breaking ones called out per [Branch Targeting](#branch-targeting). Distinguish genuinely public surface from `pub(crate)`/private. +- **Wire behavior changes**: what a *peer* observes differently, even when no signature moved and the encoding is byte-identical. A change to which content gets delivered, when a stream is reset rather than FIN'd (and with which error code), what a default now implies, or when a message is sent, is something another implementation has to match, and it is invisible in an API diff. Say what the old and new behavior are and which side decides. New/changed encodings additionally need the matching `drafts/` update in the same PR, per [Cross-Package Sync](CLAUDE.md#cross-package-sync). - **Test plan**: what was run and verified. - If you skip a [Cross-Package Sync](CLAUDE.md#cross-package-sync) row, say why. @@ -57,7 +58,7 @@ CodeRabbit reviews PRs automatically, but it has an hourly quota and runs out of Reply to review comments as you address them, saying what changed or why you disagree, so the reviewer doesn't have to diff the branch to find out. Replies are GitHub prose like any other, so they carry the [AI Contributions](#ai-contributions) marker. -When reviewing a PR, always include the same public API changes list described above, and call out anything breaking per [Branch Targeting](#branch-targeting). +When reviewing a PR, always include the same public API and wire behavior lists described above, and call out anything breaking per [Branch Targeting](#branch-targeting). Wire behavior is the easier of the two to miss, since nothing in the diff's signatures flags it: read for what a peer would now see, not just for what the types say. ## Releases From 21e7b9e45b84dc133a431d0171902b80259cd363 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 14:40:27 -0700 Subject: [PATCH 28/32] test(moq-mux): set the test latency where a caller sets it `with_test_latency` was a `#[cfg(test)]` method on the public Consumer that only called `set_latency` and returned self. It existed to keep the old `with_latency` chaining style at the call sites after that method was removed, which meant this PR deleted a public builder and then quietly re-added it for its own use. Worse, it made the tests skip the API the removal was in aid of. The point of `Consumer::new` inheriting the subscription's budget is that a caller sets latency on the `Subscription` before subscribing; every test was instead subscribing with the default and patching the consumer afterwards, so nothing but `new_inherits_the_initial_subscription_latency` exercised the real path. They now all do. `container_latency_only` still gives the two layers different budgets on purpose, so it keeps the explicit `set_latency` plus a `control.update`. Co-Authored-By: Claude Opus 5 --- rs/moq-mux/src/container/consumer.rs | 226 ++++++++++---------- rs/moq-mux/src/container/flv/import_test.rs | 48 +++-- rs/moq-mux/src/container/ts/export_test.rs | 40 ++-- rs/moq-mux/src/container/ts/import.rs | 42 ++-- rs/moq-mux/src/container/ts/import_test.rs | 30 ++- 5 files changed, 225 insertions(+), 161 deletions(-) diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 5ec55b1d23e..66cb8dabdff 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -145,13 +145,6 @@ impl Consumer { } } - /// Configure both layers while constructing an in-process test fixture. - #[cfg(test)] - pub(crate) fn with_test_latency(mut self, latency: crate::Latency) -> Self { - self.set_latency(latency); - self - } - /// A counter that increments each time the consumer detects a timeline rewind and drops /// the reneged buffer. /// @@ -775,7 +768,8 @@ mod tests { /// tests are the exception, since they are about the consumer's half of it. fn container_latency_only(track: moq_net::track::Subscriber, latency: Latency) -> Consumer { let control = track.control(); - let consumer = Consumer::new(track, Container::Legacy).with_test_latency(latency); + let mut consumer = Consumer::new(track, Container::Legacy); + consumer.set_latency(latency); control .update(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(30)))) .unwrap(); @@ -799,9 +793,9 @@ mod tests { #[tokio::test] async fn read_single_group() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); @@ -818,9 +812,9 @@ mod tests { #[tokio::test] async fn read_multiple_frames_single_group() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0), ts(33_000), ts(66_000)]); track.finish().unwrap(); @@ -837,9 +831,9 @@ mod tests { #[tokio::test] async fn read_multiple_groups_within_latency() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // 5 groups, 20ms spacing. Total span = 80ms, well within 500ms latency. for i in 0..5u64 { @@ -857,9 +851,9 @@ mod tests { async fn latency_skip_delivers_recent_groups() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(100))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(100)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Group 0: 5 frames, NOT finished (blocks consumer) let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1008,9 +1002,9 @@ mod tests { async fn reset_keeps_out_of_order_new_group() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Old epoch, played forward until the live edge passes the rewind point. write_group(&mut track, 0, &[ts(0)]); @@ -1048,9 +1042,9 @@ mod tests { async fn reset_detected_behind_forward_newest_group() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Old timeline, played to a live edge of 200 ms. write_group(&mut track, 0, &[ts(0)]); @@ -1084,10 +1078,10 @@ mod tests { async fn backwards_timestamp_resets_buffer() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); // Large latency so the slow-group skip never fires; isolate the rewind path. - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Publisher runs ahead: groups 0-4 at 0, 100, 200, 300, 400 ms. for i in 0..5u64 { @@ -1111,9 +1105,9 @@ mod tests { #[tokio::test] async fn backwards_timestamp_always_resets() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 1, &[ts(500_000)]); @@ -1147,9 +1141,9 @@ mod tests { #[tokio::test] async fn empty_payload_is_skipped() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); let media = |timestamp| Frame { @@ -1177,9 +1171,9 @@ mod tests { #[tokio::test] async fn consecutive_markers_do_not_stall() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1211,9 +1205,9 @@ mod tests { async fn groups_delivered_in_sequence_order() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1248,9 +1242,9 @@ mod tests { #[tokio::test] async fn adjacent_group_flushed_immediately() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 1, &[ts(30_000)]); @@ -1267,9 +1261,9 @@ mod tests { #[tokio::test] async fn bframes_within_group() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0), ts(66_000), ts(33_000)]); track.finish().unwrap(); @@ -1287,9 +1281,9 @@ mod tests { async fn empty_track_returns_none() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); track.finish().unwrap(); @@ -1306,9 +1300,9 @@ mod tests { async fn track_closed_with_error() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0)]); track.abort(moq_net::Error::Cancel).unwrap(); @@ -1330,9 +1324,9 @@ mod tests { #[tokio::test] async fn gap_in_group_sequence_recovery() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(100))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(100)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0), ts(20_000)]); write_group(&mut track, 1, &[ts(40_000), ts(60_000)]); @@ -1349,9 +1343,9 @@ mod tests { #[tokio::test] async fn gap_at_start_of_sequence() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_millis(80))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(80)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 5, &[ts(0), ts(20_000)]); write_group(&mut track, 7, &[ts(80_000), ts(100_000)]); @@ -1373,9 +1367,9 @@ mod tests { #[tokio::test] async fn evicted_group_with_gap_skips_to_live() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(100))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(100)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Group 0: a frame the consumer reads, positioning it there. let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1417,9 +1411,9 @@ mod tests { #[tokio::test] async fn missing_sequence_skips_on_live_track() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(100))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(100)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Group 0, then group 2 -- sequence 1 is missing (evicted) and never arrives. // The track is NOT finished (live), the case that used to hang. @@ -1517,9 +1511,9 @@ mod tests { #[tokio::test] async fn frame_timestamp_and_index_decoding() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0), ts(33_333), ts(66_666)]); track.finish().unwrap(); @@ -1538,9 +1532,9 @@ mod tests { #[tokio::test] async fn frame_payload_preserved() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let payload_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05]; let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -1577,9 +1571,9 @@ mod tests { async fn no_infinite_loop_with_buffered_frames() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1624,9 +1618,9 @@ mod tests { #[tokio::test] async fn large_timestamps() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(3700))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(3700)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let one_hour = 3_600_000_000u64; write_group(&mut track, 0, &[ts(one_hour)]); @@ -1641,9 +1635,9 @@ mod tests { #[tokio::test] async fn set_latency_changes_behavior() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_secs(10))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); @@ -1660,11 +1654,11 @@ mod tests { async fn max_timestamp_tracks_through_bframes() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(110)))); // latency must exceed (group1_max - group0_min) = 100ms - 0ms = 100ms // to avoid the latency skip and test B-frame timestamp tracking. - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(110))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); for ×tamp in &[ts(0), ts(66_000), ts(33_000)] { @@ -1713,9 +1707,9 @@ mod tests { async fn startup_selects_earliest_group() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(100))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(100)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 3, &[ts(0)]); write_group(&mut track, 5, &[ts(150_000)]); @@ -1767,9 +1761,9 @@ mod tests { async fn startup_skips_groups_without_data() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let _group5 = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); write_group(&mut track, 7, &[ts(210_000)]); @@ -1791,9 +1785,9 @@ mod tests { #[tokio::test] async fn startup_single_group_mid_stream() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 100, &[ts(3_000_000)]); track.finish().unwrap(); @@ -1806,9 +1800,9 @@ mod tests { async fn multiple_sequential_latency_skips() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, Container::Legacy).with_test_latency(Latency::max(Duration::from_millis(50))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(50)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1843,9 +1837,9 @@ mod tests { async fn latency_skip_boundary_exact() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(100))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(100)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); Container::Legacy @@ -1936,9 +1930,9 @@ mod tests { #[tokio::test] async fn group_error_skips_to_next() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); group0.abort(moq_net::Error::Cancel).unwrap(); @@ -1954,9 +1948,9 @@ mod tests { async fn track_finishes_while_reading() { tokio::time::pause(); let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); write_group(&mut track, 0, &[ts(0)]); @@ -1984,9 +1978,9 @@ mod tests { #[tokio::test] async fn empty_group_advances() { let mut track = track_producer("test", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); group0.finish().unwrap(); @@ -2005,9 +1999,9 @@ mod tests { tokio::time::pause(); let mut track = track_producer("video", hang::container::track_info()); - let consumer_track = track.subscribe(None); - let mut consumer = Consumer::new(consumer_track, Container::Legacy) - .with_test_latency(Latency::max(Duration::from_millis(500))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_millis(500)))); + let mut consumer = Consumer::new(consumer_track, Container::Legacy); // Write frames using Container::Legacy encoding let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -2048,10 +2042,10 @@ mod tests { // DurationWire is a test-only container that doesn't stamp moq_net frame // timestamps; leave the track untimed so model-layer validation matches. let mut track = track_producer("test", None); - let consumer_track = track.subscribe(None); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); // Latency dwarfs the gap, so only duration coverage can trigger the skip. - let mut consumer = - Consumer::new(consumer_track, DurationWire).with_test_latency(Latency::max(Duration::from_secs(10))); + let mut consumer = Consumer::new(consumer_track, DurationWire); // Group 0: one frame at ts=0 lasting 33ms, never finished. let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); @@ -2090,9 +2084,9 @@ mod tests { tokio::time::pause(); // DurationWire is untimed at the moq_net frame layer. let mut track = track_producer("test", None); - let consumer_track = track.subscribe(None); - let mut consumer = - Consumer::new(consumer_track, DurationWire).with_test_latency(Latency::max(Duration::from_secs(10))); + let consumer_track = track + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(Duration::from_secs(10)))); + let mut consumer = Consumer::new(consumer_track, DurationWire); // Group 0: frame at ts=0 lasting only 10ms, far short of group 1 at 33ms. let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index b3384fb1374..5c9ce8bd9d9 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -127,9 +127,15 @@ async fn import_emits_frames() { let video_name = snap.video.renditions.keys().next().unwrap().clone(); // Decode the video track back through the Legacy container. - let track = consumer.track(&video_name).unwrap().subscribe(None).await.unwrap(); - let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); + let track = consumer + .track(&video_name) + .unwrap() + .subscribe( + moq_net::track::Subscription::default().with_latency(Latency::max(std::time::Duration::from_secs(1))), + ) + .await + .unwrap(); + let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let frame = decoder.read().await.unwrap().expect("a video frame"); assert!(frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. @@ -414,9 +420,15 @@ async fn import_enhanced_av1() { assert!(matches!(v.codec, VideoCodec::AV1(_))); assert_eq!(v.description.as_ref().map(|b| b.as_ref()), Some(&AV1C[..])); - let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); + let track = consumer + .track(name) + .unwrap() + .subscribe( + moq_net::track::Subscription::default().with_latency(Latency::max(std::time::Duration::from_secs(1))), + ) + .await + .unwrap(); + let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let frame = decoder.read().await.unwrap().expect("an AV1 frame"); assert!(frame.keyframe); assert_eq!(frame.payload.as_ref(), payload); @@ -517,9 +529,15 @@ async fn import_reports_negative_pts_and_can_resume() { let snap = catalog.snapshot(); let name = snap.video.renditions.keys().next().unwrap(); - let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); + let track = consumer + .track(name) + .unwrap() + .subscribe( + moq_net::track::Subscription::default().with_latency(Latency::max(std::time::Duration::from_secs(1))), + ) + .await + .unwrap(); + let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let frame = decoder.read().await.unwrap().expect("the good frame"); assert_eq!(frame.timestamp.as_millis(), 10); } @@ -550,9 +568,15 @@ async fn import_enhanced_hvc1_applies_composition_time() { let snap = catalog.snapshot(); let name = snap.video.renditions.keys().next().unwrap(); - let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(Latency::max(std::time::Duration::from_secs(1))); + let track = consumer + .track(name) + .unwrap() + .subscribe( + moq_net::track::Subscription::default().with_latency(Latency::max(std::time::Duration::from_secs(1))), + ) + .await + .unwrap(); + let mut decoder = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let frame = decoder.read().await.unwrap().expect("a video frame"); assert_eq!(frame.timestamp.as_millis(), 17); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index ec86c4a766d..c7ba6894606 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -648,9 +648,13 @@ async fn export_scte35_roundtrip() { assert_eq!(verbatim, 1, "round-trip lost the SCTE-35 track"); let name = scte_track(&snapshot).expect("a scte35 track"); - let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer2 + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy); let frame = scte_reader .read() .await @@ -744,9 +748,13 @@ async fn export_pes_verbatim_roundtrip() { assert_eq!(verbatim.stream_id, Some(STREAM_ID), "PES stream_id preserved"); let name = name.clone(); - let track = consumer2.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer2 + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy); let frame = reader .read() .await @@ -813,9 +821,13 @@ async fn scte35_without_video_export_is_rejected() { /// Subscribe to a track and read every retained frame payload it holds. async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec> { - let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -1130,9 +1142,13 @@ fn scte_track(snap: &crate::catalog::hang::Catalog) -> Option Vec<(Vec, Timestamp)> { - let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy); let mut cues = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index d5a73572a0a..d8f19e036d3 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2646,8 +2646,13 @@ mod test { // track name while it is still registered. let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); import.finish().unwrap(); - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_test_latency(Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = Consumer::new(track, Container::Legacy); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -2878,9 +2883,13 @@ mod test { .next() .expect("an audio track") .clone(); - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { @@ -3687,11 +3696,10 @@ mod test { let track = consumer .track(hang::catalog::Catalog::DEFAULT_NAME) .unwrap() - .subscribe(None) + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(RECORDING_LATENCY))) .await .unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(Latency::max(RECORDING_LATENCY)); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let published = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await; assert!( matches!(published, Ok(Ok(Some(_)))), @@ -3945,8 +3953,13 @@ mod test { import.finish().unwrap(); let name = catalog.snapshot().mpegts.tracks.keys().next().unwrap().clone(); - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_test_latency(Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = Consumer::new(track, Container::Legacy); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("cue read timed out") @@ -4090,8 +4103,13 @@ mod test { assert_eq!(verbatim.stream_id, Some(0xC0), "recorded the PES stream_id"); assert_eq!(track.pid, DATA_PID, "recorded the original PID"); - let track = consumer.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); - let mut reader = Consumer::new(track, Container::Legacy).with_test_latency(Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(name.as_str()) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = Consumer::new(track, Container::Legacy); let frame = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read()) .await .expect("verbatim read timed out") diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 0cff4d1ed99..efe35341274 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -150,9 +150,13 @@ async fn import_opus_frames() { .expect("an Opus track") .clone(); - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; @@ -346,9 +350,13 @@ async fn survives_midstream_join() { // The track resumes at the keyframe: the leading delta was dropped, the IDR // anchors the one and only group. - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { frames.push(frame); @@ -391,9 +399,13 @@ async fn kyrion_dirtystart_extracts_real_cues() { .find(|(_, t)| t.verbatim.as_ref().is_some_and(|v| v.stream_type == 0x86)) .map(|(name, _)| name.clone()) .expect("scte35 track"); - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy) - .with_test_latency(crate::Latency::max(RECORDING_LATENCY)); + let track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_latency(crate::Latency::max(RECORDING_LATENCY))) + .await + .unwrap(); + let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy); let mut cues = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { cues.push((frame.payload.to_vec(), frame.timestamp)); From 7b8c8cb4de242188b1721ff8209232ebffd1bef3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 14:56:54 -0700 Subject: [PATCH 29/32] test(moq-mux): name the replay-window helper after what it asks for `subscribe_all` said nothing about what "all" meant, read like a subscribe-to-every-track operation, and covered two different signatures in two files. It is a replay window: these tests write every group up front and only then read, so they have to ask for history rather than the live edge. Renamed to `replay()`, returning a `Subscription`, which is the shape moq-net and moq-bench already use for the same thing. Co-Authored-By: Claude Opus 5 --- rs/moq-mux/src/container/fmp4/import_test.rs | 32 ++++---------------- rs/moq-mux/src/container/producer.rs | 20 ++++++------ 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index fef38aa66ff..9d22f7eaa28 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -10,8 +10,8 @@ use mp4_atom::{Decode, Encode}; /// edge: completeness has to be asked for. const RECORDING_LATENCY: std::time::Duration = std::time::Duration::from_secs(30); -/// A subscription with that budget, for a test asserting every group is delivered. -fn subscribe_all() -> moq_net::track::Subscription { +/// A replay window with that budget, for a test asserting every group is delivered. +fn replay() -> moq_net::track::Subscription { moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(RECORDING_LATENCY)) } @@ -644,18 +644,8 @@ async fn segmented_source_groups_per_segment() { let snapshot = catalog.snapshot(); let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - let mut video_track = consumer - .track(&video_name) - .unwrap() - .subscribe(subscribe_all()) - .await - .unwrap(); - let mut audio_track = consumer - .track(&audio_name) - .unwrap() - .subscribe(subscribe_all()) - .await - .unwrap(); + let mut video_track = consumer.track(&video_name).unwrap().subscribe(replay()).await.unwrap(); + let mut audio_track = consumer.track(&audio_name).unwrap().subscribe(replay()).await.unwrap(); // Three 1s segments: 2 video fragments each (only the first an IDR) and 4 audio fragments, // so a per-fragment grouping and a per-segment one can't be confused. @@ -890,18 +880,8 @@ async fn a_single_leading_styp_still_segments_on_keyframes() { let snapshot = catalog.snapshot(); let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - let mut video_track = consumer - .track(&video_name) - .unwrap() - .subscribe(subscribe_all()) - .await - .unwrap(); - let mut audio_track = consumer - .track(&audio_name) - .unwrap() - .subscribe(subscribe_all()) - .await - .unwrap(); + let mut video_track = consumer.track(&video_name).unwrap().subscribe(replay()).await.unwrap(); + let mut audio_track = consumer.track(&audio_name).unwrap().subscribe(replay()).await.unwrap(); // One styp up front, then four segments' worth of fragments each opening on an IDR. fmp4.decode(&styp()).unwrap(); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 00e97a5d573..8e7de9a752c 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -398,13 +398,13 @@ mod tests { .unwrap() } - /// Subscribe with a budget wide enough to read a whole batch. + /// A replay window wide enough to read a whole batch back. /// /// These tests write every group up front and only then read, which the default /// [`Latency::REAL_TIME`](moq_net::Latency::REAL_TIME) budget collapses to the live - /// edge: completeness has to be asked for. - fn subscribe_all(track: &moq_net::track::Producer) -> moq_net::track::Subscriber { - track.subscribe(moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(RECORDING_LATENCY))) + /// edge: history has to be asked for. + fn replay() -> moq_net::track::Subscription { + moq_net::track::Subscription::default().with_latency(moq_net::Latency::max(RECORDING_LATENCY)) } /// The media track's full retention window, so readers started after publishing @@ -600,7 +600,7 @@ mod tests { #[tokio::test] async fn keyframe_closes_group_immediately() { let track = track_producer("test", hang::container::track_info()); - let consumer = subscribe_all(&track); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); // first frame must be a keyframe @@ -619,7 +619,7 @@ mod tests { #[tokio::test] async fn needs_keyframe_drives_audio_grouping() { let track = track_producer("test", hang::container::track_info()); - let consumer = subscribe_all(&track); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy); // Drive grouping off `needs_keyframe`, as the audio importers do: the first frame of each @@ -643,7 +643,7 @@ mod tests { #[tokio::test] async fn cut_closes_immediately() { let track = track_producer("test", hang::container::track_info()); - let consumer = subscribe_all(&track); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); @@ -661,7 +661,7 @@ mod tests { #[allow(deprecated)] async fn deprecated_finish_group_still_closes() { let track = track_producer("test", hang::container::track_info()); - let consumer = subscribe_all(&track); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); @@ -696,7 +696,7 @@ mod tests { #[tokio::test] async fn seek_uses_explicit_sequence() { let track = track_producer("test", hang::container::track_info()); - let consumer = subscribe_all(&track); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy); producer.write(frame(0, true)).unwrap(); // seq 0 @@ -711,7 +711,7 @@ mod tests { #[tokio::test] async fn seek_clears_pending_after_use() { let track = track_producer("test", hang::container::track_info()); - let consumer = subscribe_all(&track); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy); producer.seek(5).unwrap(); From a73c5d75f49785d5b9e2c4159db3521da7c60c8f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 17:36:32 -0700 Subject: [PATCH 30/32] fix(moq-mux): give the TS export's SI track the export's replay window The SI subscription added by #2909 uses the default budget, which this branch now enforces: real-time takes only the newest group, so a track carrying table snapshots at a low rate would lose the revisions between them. SI needs the same window as the media it describes, which the export already threads to every other source. The test that repoints an SI entry writes both GOPs up front and only then reads, so it asks for that window too. Co-Authored-By: Claude Opus 5 --- rs/moq-mux/src/container/ts/export.rs | 16 ++++++++++++---- rs/moq-mux/src/container/ts/export_test.rs | 5 ++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 1d8bc2a2fff..494c2a972aa 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -185,6 +185,11 @@ struct SiTrack { pending: Option<(moq_net::group::Consumer, super::si::Snapshot)>, /// Media timestamp of the last emission ([`due`]). last_emit: Option, + /// The same budget the media sources use. SI carries table snapshots at a low + /// rate, so the real-time default would take only the newest group and drop + /// the revisions between: an SI track needs the export's replay window as much + /// as the media it describes. + latency: crate::Latency, } /// The SI subscription's lifecycle, mirroring `ExportSource`'s but reading raw @@ -203,7 +208,7 @@ enum SiState { } impl SiTrack { - fn new(source: &crate::Source, track: &str, interval: Option) -> Self { + fn new(source: &crate::Source, track: &str, interval: Option, latency: crate::Latency) -> Self { Self { track: track.to_string(), interval, @@ -211,6 +216,7 @@ impl SiTrack { active: Default::default(), pending: None, last_emit: None, + latency, } } @@ -232,7 +238,9 @@ impl SiTrack { }; self.state = match resolved { Ok((broadcast, name)) => match broadcast.track(&name) { - Ok(track) => SiState::Subscribing(track.subscribe(None)), + Ok(track) => SiState::Subscribing( + track.subscribe(moq_net::track::Subscription::default().with_latency(self.latency)), + ), Err(err) => { tracing::warn!(%err, track = %name, "SI track unavailable; carrying the last snapshot"); SiState::Done @@ -575,7 +583,7 @@ impl Export { // staying attached would repeat its stale sections forever. The last // snapshot carries across so emission never goes dark mid-swap. Some(existing) if existing.track != entry.track => { - let mut replacement = SiTrack::new(&self.source, &entry.track, entry.interval); + let mut replacement = SiTrack::new(&self.source, &entry.track, entry.interval, self.latency); replacement.active = std::mem::take(&mut existing.active); replacement.last_emit = existing.last_emit; *existing = replacement; @@ -584,7 +592,7 @@ impl Export { None => { self.si.insert( (*pid, *table_id), - SiTrack::new(&self.source, &entry.track, entry.interval), + SiTrack::new(&self.source, &entry.track, entry.interval, self.latency), ); } } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index c7ba6894606..df6f273637b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2460,9 +2460,12 @@ async fn repointed_si_entry_resubscribes() { producer.cut(None).unwrap(); }; + // Writes both GOPs up front and only then reads, so it needs a replay window: + // the real-time default would take the live edge and skip the first. let mut exporter = Export::with_ts(crate::source::announced(&consumer), crate::catalog::CatalogFormat::Hang) .await - .unwrap(); + .unwrap() + .with_latency(crate::Latency::max(RECORDING_LATENCY)); let mut before = BytesMut::new(); write_key(&mut producer, 0); write_key(&mut producer, 1); From a583c8d9eb265d4d5c9662055190c33cf53bdd20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 21:30:11 -0700 Subject: [PATCH 31/32] fix(moq-net): keep a budget-ended group ended The clean-end branch reported `Ok(None)` but left only `expired` set, so the next poll took the failure guard and turned the same group into `Error::Old`. A caller is allowed to probe past the end of a group, and `moq_mux::container::Consumer` does exactly that when it follows a read with `poll_finished`. The outcome is now sticky in its own right, as it already was in the JS mirror. Found by the Codex reviewer on the PR. Co-Authored-By: Claude Opus 5 --- rs/moq-net/src/model/group.rs | 27 ++++++++++++++++++++++++++- rs/moq-net/src/model/track.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 33ced586ced..d858895f50d 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -752,6 +752,7 @@ impl Producer { stale_stats: stats::Meter::default(), expiry: None, expired: false, + ended: false, stale_counted: Arc::default(), } } @@ -900,6 +901,11 @@ pub struct Consumer { // hands it out, while its reader is waiting for the first or next frame. expiry: Option>, expired: bool, + // Sticky: the budget gave up on a cursor that had already taken every frame, so + // the group ends rather than fails. Recorded because `expired` alone would turn a + // clean end into `Error::Old` on the next poll, and a caller is allowed to probe + // again after the end. + ended: bool, // Cloned cursors are parallel views of one handed-out delivery. Whichever // observes expiry first records its unread tail; the others must not repeat it. stale_counted: Arc, @@ -965,6 +971,7 @@ impl Clone for Consumer { stale_stats: self.stale_stats.clone(), expiry: self.expiry.clone(), expired: self.expired, + ended: self.ended, stale_counted: self.stale_counted.clone(), } } @@ -1016,6 +1023,7 @@ impl Consumer { // after a takeover. expiry: None, expired: false, + ended: false, stale_counted: self.stale_counted, } } @@ -1063,7 +1071,15 @@ impl Consumer { /// `Some(false)` ends the group cleanly and `Some(true)` fails it with /// [`Error::Old`]; see [`Self::expired_truncates`]. fn poll_expired_if_blocked(&mut self, waiter: &kio::Waiter) -> Option { - self.poll_expired(waiter).then(|| self.expired_truncates()) + if self.ended { + return Some(false); + } + if !self.poll_expired(waiter) { + return None; + } + let truncates = self.expired_truncates(); + self.ended = !truncates; + Some(truncates) } /// Whether giving up on this group now loses the reader anything. @@ -1206,6 +1222,9 @@ impl Consumer { /// Returns None if the group is finished and the index is out of range, or the cursor /// passed the [`Self::end_at`] cap. pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { + if self.ended { + return Poll::Ready(Ok(None)); + } if self.expired { return Poll::Ready(Err(Error::Old)); } @@ -1235,6 +1254,9 @@ impl Consumer { /// Read the next frame (timestamp and payload) all at once, without blocking. pub fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { + if self.ended { + return Poll::Ready(Ok(None)); + } if self.expired { return Poll::Ready(Err(Error::Old)); } @@ -1277,6 +1299,9 @@ impl Consumer { /// Poll for the final number of frames in the group. pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { + if self.ended { + return Poll::Ready(Ok(self.index())); + } if self.expired { return Poll::Ready(Err(Error::Old)); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index d035824184a..84cf5fc3d10 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4355,6 +4355,35 @@ mod test { ); } + /// A group the budget ends rather than fails stays ended. `expired` alone would + /// turn the clean answer into [`Error::Old`] on the next poll, and a caller is + /// allowed to probe again past the end of a group. + #[tokio::test] + async fn an_ended_group_stays_ended_when_probed_again() { + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(None); + let mut open = producer.append_group().unwrap(); + open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")).unwrap(); + + let mut group = subscriber.recv_group().await.unwrap().expect("group"); + assert!(group.read_frame().await.unwrap().is_some()); + + let probes = tokio::spawn(async move { + let first = group.read_frame().await; + let second = group.read_frame().await; + let finished = group.finished().await; + (first, second, finished) + }); + tokio::task::yield_now().await; + + append_at(&mut producer, 1000); + + let (first, second, finished) = probes.await.unwrap(); + assert!(matches!(first, Ok(None)), "the group ends: {first:?}"); + assert!(matches!(second, Ok(None)), "and stays ended: {second:?}"); + assert!(matches!(finished, Ok(1)), "reporting what it delivered: {finished:?}"); + } + #[tokio::test] async fn a_drained_group_finishes_cleanly_after_the_live_edge_advances() { let mut producer = track_producer("test", None); @@ -6857,3 +6886,4 @@ mod test { drop(dynamic); } } + From 1d966e41996f417b54116f414b786ec824c677fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 21:36:13 -0700 Subject: [PATCH 32/32] fix(moq-net): don't serve legacy lite peers at the live edge Lite01/02 have no latency field in SUBSCRIBE, so `decode_msg` substitutes `Duration::ZERO`. That is indistinguishable from a peer asking for real time, and this branch now acts on it: the serving side discards every group but the live edge for a subscriber that never declined the backlog. Those two are `NEGOTIATED[0]` and `[1]`, so this is the ordinary wire for current moq-lite peers rather than a legacy corner. They now get a window wide enough not to drop, leaving enforcement to the receiving subscriber, which is what the IETF path already does for the same reason. `Version::carries_latency` is the one place that answers whether a decoded zero means "real time" or "not stated". Found by the Codex reviewer on the PR. Co-Authored-By: Claude Opus 5 --- rs/moq-net/src/lite/publisher.rs | 46 ++++++++++++++++++++++++++++++-- rs/moq-net/src/lite/subscribe.rs | 11 ++++++++ rs/moq-net/src/model/track.rs | 4 +-- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 818b302cdaa..5f621027eab 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -89,6 +89,23 @@ struct Shared { goaway: crate::goaway::Protocol, } +/// Largest millisecond duration every implementation can carry losslessly. +const MAX_SAFE_LATENCY_MS: u64 = (1_u64 << 53) - 1; + +/// The budget to serve a peer with, given what its wire could tell us. +/// +/// A version without the field decodes as [`Duration::ZERO`], which is +/// indistinguishable from a peer genuinely asking for the live edge. Serving that +/// as real time would discard backlog a legacy subscriber never declined, so fall +/// back to a window wide enough not to drop and leave enforcement to the receiver, +/// exactly as the IETF path does for the same reason. +fn serving_latency(version: Version, requested: Duration) -> crate::Latency { + match version.carries_latency() { + true => crate::Latency::max(requested), + false => crate::Latency::max(Duration::from_millis(MAX_SAFE_LATENCY_MS)), + } +} + impl Shared { /// The origin to resolve a peer-requested broadcast from: excludes routes /// through the peer when its SETUP declared an origin id, so a subscription @@ -1293,7 +1310,7 @@ impl SubscribeServe { let subscription = crate::track::Subscription { priority: msg.priority, ordered: msg.ordered, - latency: crate::Latency::max(msg.latency_max), + latency: serving_latency(self.shared.version, msg.latency_max), ..Bounds::from(&msg).positions() }; @@ -2770,7 +2787,7 @@ impl TrackRun { let _ = self.track.update(crate::track::Subscription { priority: upd.priority, ordered: upd.ordered, - latency: crate::Latency::max(upd.latency_max), + latency: serving_latency(self.ctx.version, upd.latency_max), ..bounds.positions() }); if let Some(start_group) = upd.start_group { @@ -3386,6 +3403,31 @@ mod serve_group_test { assert!(matches!(serve.await, Err(Error::Old))); } + /// Lite01/02 have no latency field, so a SUBSCRIBE from one decodes as + /// `Duration::ZERO`. Serving that as a real-time budget would hold every legacy + /// peer to the live edge and discard backlog it never declined, so those versions + /// get a non-dropping window and leave enforcement to the receiver. + /// + /// These are the two most-preferred negotiated versions, so this is the common + /// wire, not an edge case. + #[test] + fn a_version_without_the_field_serves_a_non_dropping_budget() { + for version in [Version::Lite01, Version::Lite02] { + let latency = serving_latency(version, Duration::ZERO); + assert!( + latency.max >= Duration::from_secs(86_400), + "{version:?} must not be served as real time: {latency:?}" + ); + } + + // A version that does carry it is taken at its word, zero included. + assert_eq!(serving_latency(Version::Lite05, Duration::ZERO).max, Duration::ZERO); + assert_eq!( + serving_latency(Version::Lite05, Duration::from_secs(3)).max, + Duration::from_secs(3) + ); + } + /// A group that completes cleanly must not reset at all. The completion path /// releases the stream via `poll_close`; leaving the writer to drop after /// `finish()` would fire the Drop fallback and tack a spurious Cancel reset diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index 1b442ec730c..7c5829b3a55 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -28,6 +28,17 @@ pub struct Subscribe<'a> { pub end_frame: Option, } +impl Version { + /// Whether this version's SUBSCRIBE carries the subscriber's latency preference. + /// + /// Lite01/02 have no field for it, so a decoded `Duration::ZERO` there means + /// "not stated", not "real time". Callers that act on the budget must tell the + /// two apart or they will hold every legacy peer to the live edge. + pub(crate) fn carries_latency(self) -> bool { + !matches!(self, Version::Lite01 | Version::Lite02) + } +} + impl Message for Subscribe<'_> { fn decode_msg(r: &mut R, version: Version) -> Result { let id = u64::decode(r, version)?; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 84cf5fc3d10..bc72c4a41b9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4363,7 +4363,8 @@ mod test { let mut producer = track_producer("test", None); let mut subscriber = producer.subscribe(None); let mut open = producer.append_group().unwrap(); - open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")).unwrap(); + open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")) + .unwrap(); let mut group = subscriber.recv_group().await.unwrap().expect("group"); assert!(group.read_frame().await.unwrap().is_some()); @@ -6886,4 +6887,3 @@ mod test { drop(dynamic); } } -