From 4b8c7a3c15873f663ded900c430e158471cf77b3 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:37:51 -0800 Subject: [PATCH 01/21] chore(quest): claim quest/m2/multicast/resend.md Quest-Claim: 97924fbdd0721e6b From 8998f887accd44e10c4c549c6492b8ea3e547e93 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:44:18 -0800 Subject: [PATCH 02/21] wip: resume a group past the frames held on a route switch --- drafts/draft-lcurley-moq-lite.md | 27 +++-- rs/moq-net/fixtures/lite.json | 10 +- rs/moq-net/src/lite/group.rs | 62 +++++----- rs/moq-net/src/lite/publisher.rs | 24 +++- rs/moq-net/src/lite/subscribe.rs | 107 +++++++++++------- rs/moq-net/src/lite/subscriber.rs | 13 ++- rs/moq-net/src/lite/version.rs | 6 - rs/moq-net/src/model/group.rs | 11 ++ rs/moq-net/src/model/resume.rs | 101 ++++++++++++++++- rs/moq-net/src/model/subscription.rs | 7 ++ rs/moq-net/src/model/track/demand.rs | 23 +++- rs/moq-net/src/test_interop.rs | 4 +- .../tests/integration/route_audit/failures.rs | 23 +--- 13 files changed, 288 insertions(+), 130 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 022454bad74..da4a4ffa7bd 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -199,8 +199,7 @@ A frame is used to represent a chunk of data with an upfront size. The contents are opaque to the moq-lite layer. Frames within a Group are numbered from 0 in the order they were produced. -This index is not transmitted per frame; it is implied by position within the Group. -A Group Stream starts at frame 0, and a Fetch Stream at its FETCH's `Frame Start`. +This index is not transmitted per frame; it is implied by position within the Group, anchored by the [GROUP](#group-message) message's `Frame Start`, or a Fetch Stream's FETCH `Frame Start`. Each frame carries a presentation timestamp expressed in the Track's `Timescale` (see [TRACK_INFO](#track-info)), used by the moq-lite layer for [expiration](#expiration) decisions. A Track with a `Timescale` of 0 is scale-less: its timestamps and Group starts are bare numbers in no known unit, and each frame's timestamp is its Group's start. @@ -213,14 +212,14 @@ A relay serves a Track's timestamps on as it received them, including ones it di A Position is a (Group Start, Frame Index) pair identifying one frame within a Track. Positions order lexicographically: by group first, then by frame within the group. -SUBSCRIBE reads whole Groups: a Range's Anchors name Groups (see [SUBSCRIBE](#subscribe)), so a subscription never starts or stops partway through one. +SUBSCRIBE reads whole Groups: a Range's Anchors name Groups (see [SUBSCRIBE](#subscribe)), except that a Range may start at a Position, so a subscription may start partway through a Group but never stops partway through one. FETCH reads one Group, or a slice of one: its `Frame Start` and `Frame End` bound the returned frames, and a frame index of 0 at the start, or an absent end frame, is the whole Group. -A subscriber that has received frames 0 through `N-1` of the Group starting at `G` and wants the rest FETCHes `G` from frame `N`, and subscribes for the Groups after it. +A subscriber that has received frames 0 through `N-1` of the Group starting at `G` and wants the rest, and the Groups after it, subscribes from the Position (`G`, `N`). This is what lets a subscriber move a Track to a different publisher partway through a Group instead of waiting for the next Group to start, which matters for Tracks whose current Group may stay open indefinitely. A Group can therefore be assembled from more than one publisher: each contributes a disjoint run of frames, and the subscriber concatenates them in index order. -A partial Group is only ever delivered to a subscriber that asked for one with FETCH. +A partial Group is only ever delivered to a subscriber that asked for one, with FETCH or a Position Start. A publisher that holds only the later frames of a Group a subscription covers names it in a [SUBSCRIBE_DROP](#subscribe-drop) rather than deliver the part it holds. One that holds its earlier frames, such as a relay whose upstream closed the Group or a Group expiring as it is sent, delivers them and closes the Group Stream without a GROUP_END. A Group is the unit of decodability: its earlier Frames may carry a keyframe or compression state the later Frames need. @@ -871,6 +870,8 @@ A Group Stream MUST start with a GROUP message and MAY be followed by any number A Group MAY contain zero FRAME messages, potentially indicating a gap in the track. A frame MAY contain an empty payload, potentially indicating a gap in the group. +The GROUP message's `Frame Start` gives the index of the first FRAME on the stream, so a stream carrying only part of a Group is self-describing. + A publisher ends a Group Stream the way its Group ended (see [Group](#group)): with a GROUP_END and a FIN for a finished Group, with a FIN alone for a closed one, and with a reset for an aborted one. A publisher MUST NOT send a GROUP_END for a Group that has not finished at this hop, and a subscriber treats bytes after a GROUP_END as a malformed Track. @@ -1475,6 +1476,7 @@ Anchor { Anchor Type (i) [Timestamp (i)] [Groups Back (i)] + [Frame (i)] } ~~~ @@ -1526,6 +1528,7 @@ Which fields follow, and what the Anchor names: - None (0x0): no fields. As a `Start`, the oldest Group (no floor); as an `End`, the live edge (unbounded). - Time (0x1): `Timestamp`, the Group containing that time: the one whose start (see [Group](#group)) is the highest at or below it. - Relative (0x2): `Groups Back`, the Group that many before the latest, 0 being the latest. +- Position (0x3): `Timestamp` then `Frame`, as Time, except that a Group starting exactly at `Timestamp` is delivered from frame `Frame` (see [Positions](#positions)). A `Start` only, and `Frame` is never 0, which Time says: anything else is a PROTOCOL_VIOLATION. Any other value is a PROTOCOL_VIOLATION. Both ends include the Group they name, so a Range from Time `a` to Time `b` delivers every Group from the one containing `a` through the one containing `b`. @@ -1543,7 +1546,10 @@ A Range joining at the live edge, with a `Start` of Relative 0 and an `End` of N The live edge is the newest frame timestamp the publisher holds; on a Track without a timescale, the Min Age is measured by when each group arrived at the publisher instead. The publisher starts from what it holds and never fetches to reach further back, so a subscriber is handed the buffer it will play, and no more, without waiting for it to fill. -A Range covers whole Groups; a subscriber reads a slice of a Group with FETCH (see [Positions](#positions)). +A Range covers whole Groups but for a Position `Start`; a subscriber reads a slice of a Group with FETCH (see [Positions](#positions)). +A Group that another Range also covers is delivered whole, since each Group is delivered once. +A subscriber resuming a Track on a new subscription, such as a relay moving to another route, starts at the Position past the frames it holds, so no frame it holds is sent again. +The Group Stream of a Group that ended at or before `Frame` carries no FRAME, only how the Group ended. ## SUBSCRIBE_UPDATE {#subscribe-update} A subscriber can modify a subscription with a SUBSCRIBE_UPDATE message. @@ -2163,6 +2169,7 @@ GROUP Message { Subscribe ID (i) Group Start (i) Previous Start (i) + Frame Start (i) } ~~~ @@ -2180,6 +2187,10 @@ No Group has a start between the two, so the subscriber accounts for the starts A publisher knows it when the starts below this Group were declared skipped down to a Group, or a Group starts just below it. A value at or above `Group Start` plus one is a malformed Track. +**Frame Start**: +The index of the first FRAME message on this stream within the group (see [Positions](#positions)). +0 means the stream carries the group from its beginning; a publisher sends a higher value only for a Range starting at that Position. + ## FRAME The FRAME message is a payload within a group. @@ -2319,7 +2330,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he ### Subscriptions {#changelog-07-subscribe} -- Replaced `Group Start`, `Group End`, `Frame Start`, and `Frame End` in SUBSCRIBE and SUBSCRIBE_UPDATE with a non-empty list of Ranges of whole Groups. Each end is an Anchor: none, a time naming the Group containing it, or a count of Groups back from the latest, and both ends include the Group they name. +- Replaced `Group Start`, `Group End`, `Frame Start`, and `Frame End` in SUBSCRIBE and SUBSCRIBE_UPDATE with a non-empty list of Ranges of whole Groups. Each end is an Anchor: none, a time naming the Group containing it, or a count of Groups back from the latest, and both ends include the Group they name. A start may also be a Position, a Group's start and a frame within it, so a subscription resumes a Group partway through. - Added `Group Order` to SUBSCRIBE and SUBSCRIBE_UPDATE: descending (0x0, the previous fixed order) or ascending (0x1). - Encoded `Subscriber Max Age` in SUBSCRIBE and SUBSCRIBE_UPDATE as milliseconds plus one, with zero meaning no limit. - Added an optional `Subscriber Min Age` to SUBSCRIBE and SUBSCRIBE_UPDATE, the subscriber's target latency, encoded as milliseconds plus one with zero meaning not stated. A Range joining at the latest group starts that far behind the live edge, from the groups the publisher holds. A relay forwards the largest stated value. @@ -2348,7 +2359,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - Replaced group datagrams with datagram Tracks, a namespace apart from Tracks of Groups. A subscriber subscribes to one on the Datagrams Stream (0x9) with DATAGRAMS and DATAGRAMS_UPDATE, which carry only a priority, and the publisher answers with DATAGRAMS_INFO. A datagram carries the Track's own Sequence and a Timestamp, with no Group, Inflated Size, or Range. Datagrams no longer count toward SUBSCRIBE_DROP, and a subscriber no longer deduplicates them against Group Streams. - Added a leading Type to datagrams, 0x0 for a Track Datagram; a receiver discards one of an unknown Type. - FRAME and datagrams always carry a timestamp, a bare number on a scale-less Track. -- Replaced GROUP's `Group Sequence` with `Group Start`, added `Previous Start`, the start of the Group before it, so a subscriber sees the gap before a Group without a SUBSCRIBE_SKIP, and removed `Frame Start`: a subscription reads whole Groups, so every Group Stream starts at frame 0. +- Replaced GROUP's `Group Sequence` with `Group Start`, added `Previous Start`, the start of the Group before it, so a subscriber sees the gap before a Group without a SUBSCRIBE_SKIP. - Added GROUP_END, a `Payload Length` of 0 followed by the Group's exclusive end: a Group or Fetch Stream ends with it when its Group finished, a FIN alone ends a closed Group whose frames may not all exist, and a reset aborts. A Group's own end bounds its reach. - Expiration also measures a group's arrival age, the time since its successor arrived at this hop. - A publisher closes the open Group Stream of a group that expires, without a GROUP_END, rather than resetting it, and a relay closes a Group it holds only the start of. diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 4de48bfb9cd..53baf85e434 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -341,21 +341,21 @@ "name": "a group sequence at 2^64-1, the largest a u64 holds", "version": "moq-lite-07-wip", "message": "group", - "hex": "0b00ffffffffffffffffff00", + "hex": "0c00ffffffffffffffffff0000", "expect": "decoded" }, { - "name": "a lite-07 group carrying a Frame Start, which only lite-06 has", + "name": "a lite-07 group starting partway through, at a Frame Start", "version": "moq-lite-07-wip", "message": "group", - "hex": "0c00ffffffffffffffffff0000", - "expect": "refused" + "hex": "0401070004", + "expect": "decoded" }, { "name": "a group naming the start just below it as the previous one, carried plus one", "version": "moq-lite-07-wip", "message": "group", - "hex": "1300ffffffffffffffffffffffffffffffffffff", + "hex": "1400ffffffffffffffffffffffffffffffffffff00", "expect": "decoded" }, { diff --git a/rs/moq-net/src/lite/group.rs b/rs/moq-net/src/lite/group.rs index fb13df29b1b..829eb68b119 100644 --- a/rs/moq-net/src/lite/group.rs +++ b/rs/moq-net/src/lite/group.rs @@ -17,15 +17,15 @@ pub struct Group { // The index of the first frame on this stream. 0 (the common case) means the stream // carries the group from its beginning; a higher value means the leading frames are - // missing, because the subscription started partway in. Lite-06 only; other versions + // missing, because the subscription started partway in. Lite06+ only; older versions // always start at 0. pub frame_start: u64, } impl Group { - /// The varints the longest layout carries: Subscribe ID, Group Start, and lite-07's - /// Previous Start or lite-06's Frame Start. - const FIELDS: usize = 3; + /// The varints the newest layout carries: Subscribe ID, Group Start, Previous Start and + /// Frame Start. + const FIELDS: usize = 4; /// The starts this header passes over as skipped; see [`passed_over`]. pub(crate) fn passed_over(&self) -> std::ops::Range { @@ -56,7 +56,7 @@ impl<'a> Message<'a> for Group { true => r.varint()?.checked_sub(1), false => None, }; - let frame_start = match version.has_group_frame_start() { + let frame_start = match version.has_frame_bounds() { true => r.varint()?, false => 0, }; @@ -80,7 +80,7 @@ impl<'a> Message<'a> for Group { w.varint(previous)?; } - if version.has_group_frame_start() { + if version.has_frame_bounds() { w.varint(self.frame_start)?; } else if self.frame_start != 0 { // The peer would number the frames from 0 and silently misalign the group. @@ -104,31 +104,29 @@ mod test { previous: None, frame_start: 4, }; - let mut buf = Vec::new(); - msg.encode_msg(&mut Encoder::new(&mut buf, Version::Lite06.into()), Version::Lite06) - .unwrap(); - let got = - crate::coding::decode_buf(&mut buf.as_slice(), Version::Lite06, |r, v| Group::decode_msg(r, v)).unwrap(); - assert_eq!((got.key, got.frame_start), (7, 4)); + for version in [Version::Lite06, Version::Lite07] { + let mut buf = Vec::new(); + msg.encode_msg(&mut Encoder::new(&mut buf, version.into()), version) + .unwrap(); + let got = crate::coding::decode_buf(&mut buf.as_slice(), version, |r, v| Group::decode_msg(r, v)).unwrap(); + assert_eq!((got.key, got.frame_start), (7, 4), "{version:?}"); + } } - /// A peer of another version would number the frames from 0 and silently misalign the group. + /// An older peer would number the frames from 0 and silently misalign the group. #[test] - fn frame_start_rejected_off_lite06() { + fn frame_start_rejected_before_lite06() { let msg = Group { subscribe: 1, key: 7, previous: None, frame_start: 4, }; - for version in [Version::Lite05, Version::Lite07] { - let mut buf = Vec::new(); - assert!( - msg.encode_msg(&mut Encoder::new(&mut buf, version.into()), version) - .is_err(), - "{version:?}" - ); - } + let mut buf = Vec::new(); + assert!( + msg.encode_msg(&mut Encoder::new(&mut buf, Version::Lite05.into()), Version::Lite05) + .is_err() + ); let whole = Group { frame_start: 0, ..msg }; let mut buf = Vec::new(); @@ -156,23 +154,23 @@ mod test { .unwrap(); buf }; - assert_eq!(encode(Some(4), Version::Lite07), [1, 7, 5]); - assert_eq!(encode(None, Version::Lite07), [1, 7, 0]); + assert_eq!(encode(Some(4), Version::Lite07), [1, 7, 5, 0]); + assert_eq!(encode(None, Version::Lite07), [1, 7, 0, 0]); assert_eq!(encode(Some(4), Version::Lite06), [1, 7, 0]); let decode = |wire: &[u8], version| { crate::coding::decode_buf(&mut &wire[..], version, |r, v| Group::decode_msg(r, v)).unwrap() }; - let got = decode(&[1, 7, 5], Version::Lite07); + let got = decode(&[1, 7, 5, 0], Version::Lite07); assert_eq!((got.previous, got.passed_over()), (Some(4), 5..7)); - let got = decode(&[1, 7, 7], Version::Lite07); + let got = decode(&[1, 7, 7, 0], Version::Lite07); assert_eq!((got.previous, got.passed_over()), (Some(6), 7..7)); - let got = decode(&[1, 7, 0], Version::Lite07); + let got = decode(&[1, 7, 0, 0], Version::Lite07); assert_eq!((got.previous, got.passed_over()), (None, 7..7)); assert_eq!(decode(&[1, 7, 0], Version::Lite06).previous, None); // One at or above the group's own start passes over nothing: the receiver refuses it. - let got = decode(&[1, 7, 8], Version::Lite07); + let got = decode(&[1, 7, 8, 0], Version::Lite07); assert_eq!((got.previous, got.passed_over()), (Some(7), 7..7)); } @@ -195,12 +193,12 @@ mod test { Form::Quic => (crate::coding::varint::MAX_QUIC, 8), Form::LeadingOnes => (u64::MAX, crate::coding::varint::MAX_LEN), }; - let fields = 2 + usize::from(version.has_group_frame_start()) + usize::from(version.has_group_end()); + let fields = 2 + usize::from(version.has_frame_bounds()) + usize::from(version.has_group_end()); let msg = Group { subscribe: max, key: max, previous: version.has_group_end().then_some(max - 1), - frame_start: if version.has_group_frame_start() { max } else { 0 }, + frame_start: if version.has_frame_bounds() { max } else { 0 }, }; let mut body = Vec::new(); @@ -221,13 +219,13 @@ mod test { ); } - assert_eq!(::MAX_SIZE, 27); + assert_eq!(::MAX_SIZE, 36); let over = ::MAX_SIZE + 1; let mut wire = vec![over as u8]; wire.resize(1 + over, 0); assert!(matches!( Group::decode_slice(&wire, Version::Lite07), - Err(DecodeError::MessageTooLarge { size: 28, max: 27 }) + Err(DecodeError::MessageTooLarge { size: 37, max: 36 }) )); } } diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 619aa413030..899c7f023aa 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -4484,8 +4484,8 @@ fn position_group(group: group::Consumer, frames: &Frames) -> Option group => false, + (_, Some(Anchor::Time(end))) if end < group => false, + _ => true, + }; + let mut covering = bounds.ranges.iter().filter(|range| covers(range)); + if covering.next().is_some() && covering.next().is_some() { + bounds.frames.first = None; + } + } bounds } @@ -6931,7 +6945,7 @@ mod serve_group_test { let (bytes, fin) = first_stream(&log); assert!(fin, "no FIN"); // Only the GROUP: the write blocked before the first frame, and none goes out after. - assert_eq!(bytes, [0, 3, 0, 0, 0]); + assert_eq!(bytes, [0, 4, 0, 0, 0, 0]); assert!(outcomes.dropped.lock().unwrap().is_empty(), "dropped"); } diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index 96781ecb8ed..78f212ddb4c 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -59,10 +59,10 @@ impl Subscribe<'_> { /// One end of a [`Range`], as the wire carries it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Anchor { - /// A position: a start delivers from it, and an end stops just before it. Only lite-06 - /// bounds a range partway through a group. Lite-07 has no such type: a group's start is its - /// time, so it goes out as a [`Anchor::Time`], an end as the time just before it, and - /// decodes as one. + /// A position: a start delivers from it, and an end stops just before it. Lite-06 bounds + /// either end partway through a group. A group's start is its time, so lite-07 sends one + /// at a group's first frame as an [`Anchor::Time`], an end as the time just before it, and + /// a start partway through a group as its Position type, which only a start may be. Absolute(crate::track::Position), /// The group this many before the latest, included as an end. Lite07+ only, except a /// start of 0 (see [`Range`]). @@ -310,8 +310,8 @@ fn decode_range_list(r: &mut Decoder<'_>) -> Result, DecodeError> { // count the bytes can't back runs out before it costs anything. let mut ranges = Vec::new(); for _ in 0..count { - let start = decode_anchor(r)?; - let end = decode_anchor(r)?; + let start = decode_anchor(r, false)?; + let end = decode_anchor(r, true)?; ranges.push(Range { start, end }); } Ok(ranges) @@ -322,10 +322,10 @@ fn encode_range_list(w: &mut Encoder<'_>, ranges: &[Range]) -> Result<(), Encode w.varint(ranges.len() as u64)?; for range in ranges { for (anchor, end) in [(range.start, false), (range.end, true)] { - let (kind, value) = wire_anchor(anchor, end)?; + let (kind, values) = wire_anchor(anchor, end)?; w.varint(kind)?; - if let Some(value) = value { - w.varint(value)?; + for value in values.iter().flatten() { + w.varint(*value)?; } } } @@ -336,39 +336,54 @@ fn encode_range_list(w: &mut Encoder<'_>, ranges: &[Range]) -> Result<(), Encode pub(super) fn range_size(range: &Range, form: Form) -> Option { let size = |value| varint::size(value, form).ok(); let anchor = |anchor, end| -> Option { - let (kind, value) = wire_anchor(anchor, end).ok()?; - Some(size(kind)? + value.map_or(Some(0), size)?) + let (kind, values) = wire_anchor(anchor, end).ok()?; + values + .iter() + .flatten() + .try_fold(size(kind)?, |total, value| Some(total + size(*value)?)) }; Some(anchor(range.start, false)? + anchor(range.end, true)?) } /// Decode one lite-07 range end: its type, then the fields that type carries. Type 0 is no -/// anchor, the oldest group as a start and the live edge as an end. -fn decode_anchor(r: &mut Decoder<'_>) -> Result, DecodeError> { +/// anchor, the oldest group as a start and the live edge as an end. Type 3, a position +/// partway through a group, is a start only, so `end` refuses it, and so is one at frame 0, +/// which type 1 says. +fn decode_anchor(r: &mut Decoder<'_>, end: bool) -> Result, DecodeError> { Ok(match r.varint()? { 0 => None, 1 => Some(Anchor::Time(r.varint()?)), 2 => Some(Anchor::Relative(r.varint()?)), + 3 if !end => { + let group = r.varint()?; + let frame = r.varint()?; + if frame == 0 { + return Err(DecodeError::InvalidValue); + } + Some(Anchor::Absolute(crate::track::Position { group, frame })) + } _ => return Err(DecodeError::InvalidValue), }) } -/// The lite-07 type and value of a range's start or `end`, the inverse of [`decode_anchor`]. +/// The lite-07 type and values of a range's start or `end`, the inverse of [`decode_anchor`]. /// -/// An absolute position goes out as its group's time: a start as is, and an exclusive end as -/// the inclusive time just before it, an end at group 0 holding nothing a time can say. A -/// range covers whole groups, so a position partway through one is refused: FETCH reads a -/// slice of a group. -fn wire_anchor(anchor: Option, end: bool) -> Result<(u64, Option), EncodeError> { +/// An absolute position at a group's first frame goes out as its group's time: a start as +/// is, and an exclusive end as the inclusive time just before it, an end at group 0 holding +/// nothing a time can say. A start partway through a group goes out as a position, its +/// group's start then the frame; an end there is refused, since FETCH reads a slice of a group. +fn wire_anchor(anchor: Option, end: bool) -> Result<(u64, [Option; 2]), EncodeError> { Ok(match anchor { - None => (0, None), - Some(Anchor::Time(time)) => (1, Some(time)), - Some(Anchor::Relative(back)) => (2, Some(back)), - Some(Anchor::Absolute(position)) if position.frame != 0 => return Err(EncodeError::Version), - Some(Anchor::Absolute(position)) if end => { - (1, Some(position.group.checked_sub(1).ok_or(EncodeError::InvalidState)?)) - } - Some(Anchor::Absolute(position)) => (1, Some(position.group)), + None => (0, [None, None]), + Some(Anchor::Time(time)) => (1, [Some(time), None]), + Some(Anchor::Relative(back)) => (2, [Some(back), None]), + Some(Anchor::Absolute(position)) if position.frame != 0 && end => return Err(EncodeError::Version), + Some(Anchor::Absolute(position)) if position.frame != 0 => (3, [Some(position.group), Some(position.frame)]), + Some(Anchor::Absolute(position)) if end => ( + 1, + [Some(position.group.checked_sub(1).ok_or(EncodeError::InvalidState)?), None], + ), + Some(Anchor::Absolute(position)) => (1, [Some(position.group), None]), }) } @@ -1799,30 +1814,36 @@ mod test { assert!(matches!(encode(&msg, Version::Lite07), Err(EncodeError::InvalidState))); } - /// A lite-07 range covers whole groups: an absolute anchor is its type and group alone, - /// and one partway through a group is refused, since FETCH reads a slice of a group. + /// A lite-07 start may fall partway through a group, as its Position type: the group's + /// start, then the frame. An end there is refused, since FETCH reads a slice of a group, + /// and so is a Position at frame 0, which a Time says. #[test] - fn lite07_ranges_cover_whole_groups() { + fn lite07_starts_partway_through_a_group() { let mut msg = framed_sample(); assert!(matches!(encode(&msg, Version::Lite07), Err(EncodeError::Version))); msg.ranges = vec![Range { start: absolute(7, 4), end: None, }]; - assert!(matches!(encode(&msg, Version::Lite07), Err(EncodeError::Version))); + assert_eq!(roundtrip(&msg, Version::Lite07).ranges, msg.ranges); + let position = encode(&msg, Version::Lite07).unwrap(); + // Type 3, start 7, frame 4, then no end. + assert!(position.ends_with(&[1, 3, 7, 4, 0]), "{position:?}"); + assert_eq!( + range_size(&msg.ranges[0], Version::Lite07.into()), + Some(4), + "the size a range takes counts both fields" + ); - msg.ranges = vec![Range { - start: absolute(7, 0), - end: absolute(9, 0), - }]; - let whole = encode(&msg, Version::Lite07).unwrap(); - msg.ranges = vec![Range { - start: Some(Anchor::Relative(7)), - end: Some(Anchor::Relative(9)), - }]; - // The same bytes but the two types: no frame follows either group. - let relative = encode(&msg, Version::Lite07).unwrap(); - assert_eq!(whole.len(), relative.len()); + let decode = |tail: &[u8]| { + let mut wire = position.clone(); + wire.truncate(position.len() - 5); + wire.extend_from_slice(tail); + crate::coding::decode_from(&mut wire.as_slice(), Version::Lite07, Subscribe::decode_msg).map(|_| ()) + }; + assert!(decode(&[1, 3, 7, 4, 0]).is_ok()); + assert!(matches!(decode(&[1, 3, 7, 0, 0]), Err(DecodeError::InvalidValue))); + assert!(matches!(decode(&[1, 1, 7, 3, 9, 4]), Err(DecodeError::InvalidValue))); } /// An update sends only the ranges that changed, and the publisher resolves the same list, diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 2077eb05beb..596e4dab6b2 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -9987,8 +9987,9 @@ mod tests { /// The ranges a SUBSCRIBE carries for `subscription`, without the empty ones. /// /// A group ID is the group's start, so a time anchor goes out as the group ID it names, -/// on every version: a time start as the start, and a time end as the exclusive group past -/// it, which lite-07 carries as the inclusive time again (see [`lite::Anchor::Absolute`]). +/// on every version: a time start as the start, from the frame it resumes at (see +/// [`Subscription::resume`]), and a time end as the exclusive group past it, which lite-07 +/// carries as the inclusive time again (see [`lite::Anchor::Absolute`]). /// `None` when nothing is left to ask for, which no wire can say. The caller narrows a /// subscription for a wire without ranges to the one span it carries first (see /// [`wire_span`]). @@ -10006,7 +10007,11 @@ fn wire_ranges(subscription: &Subscription) -> Option> { let start = match range.start { Anchor::Oldest => None, Anchor::Relative(back) => Some(lite::Anchor::Relative(back)), - Anchor::Time(time) => Some(lite::Anchor::Absolute(Position::group(time))), + // A switch resumes the group it starts at past the frames the readers hold. + Anchor::Time(time) => Some(lite::Anchor::Absolute(match subscription.resume { + Some(resume) if resume.group == time => resume, + _ => Position::group(time), + })), }; // The copy resolves a relative end itself, against the groups it receives past it, so // it asks through the live edge. SUBSCRIBE_START names a resolved start, but nothing @@ -10156,7 +10161,7 @@ impl SubStream { continue; }; match range.start { - Some(lite::Anchor::Absolute(time)) if time.frame == 0 => times.push((time.group, *start)), + Some(lite::Anchor::Absolute(time)) => times.push((time.group, *start)), Some(lite::Anchor::Relative(back)) => { if let Some(Some(latest)) = self.edges.get(&id) { relative.push((back, *start, *latest)); diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 77d0e21910b..53a842c56a0 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -166,12 +166,6 @@ impl Version { ) } - /// Whether GROUP carries `Frame Start`, the index of the stream's first frame. Lite-06 only: - /// earlier versions have no frame bounds, and a lite-07 subscription reads whole groups. - pub(crate) fn has_group_frame_start(self) -> bool { - matches!(self, Self::Lite06) - } - /// Whether SUBSCRIBE and SUBSCRIBE_UPDATE carry the subscriber's `Group Order`. /// Added in lite-07; an earlier subscription is served newest first. pub(crate) fn has_order(self) -> bool { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 585ecf92979..f15807e3744 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -4619,6 +4619,17 @@ impl Consumer { self.cursor.state.read().committed() } + /// The frames committed to the copy this reads, from the index it started at: what a route + /// switch need not ask for again. `None` once aborted, or on a sparse wire, whose object + /// IDs are not frame indices. + pub(super) fn committed_span(&self) -> Option> { + let state = self.cursor.state.read(); + if state.outcome.error().is_some() || state.sparse() { + return None; + } + Some(state.offset as u64..state.committed() as u64) + } + fn poll_read_frame_copy( &mut self, waiter: &kio::Waiter, diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index cd48532d7da..c8ca625a24b 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -179,6 +179,7 @@ impl Producer { return demand; } let mut ranges = Vec::new(); + let mut resumes = Vec::with_capacity(readers.len()); for reader in readers { let mut reader = reader.lock().unwrap_or_else(PoisonError::into_inner); let preferences = reader.subscription.read().clone(); @@ -208,8 +209,13 @@ impl Producer { _ => Vec::new(), }; ranges.extend(preferences.clamped(reader.newest, None, &resolved).ranges); + resumes.push(reader.resume_at(reader.newest)); } demand.ranges = ranges; + // As in a track's own demand, only when every reader resumes. + demand.resume = (resumes.into_iter()) + .reduce(|a, b| a.zip(b).map(|(a, b)| a.min(b))) + .flatten(); demand } @@ -600,6 +606,9 @@ struct Copy { sub: Sub, /// Where the subscription was asked to start, so demand updates never ask for less. floor: Option, + /// The frames of the group at `floor` the reader already held when this copy was asked, + /// so it asks for that group from the next frame; see [`Reader::resume_at`]. + resume: Option, /// The reader's ranges [`Self::resolved`] are for. from: Vec, /// Each of those ranges resolved, by index, so clamping never widens one: as the @@ -651,7 +660,9 @@ impl Copy { self.from = reader.ranges.clone(); } } - reader.clamped(self.floor, self.end(), &self.resolved) + let mut wire = reader.clamped(self.floor, self.end(), &self.resolved); + wire.resume = self.resume; + wire } /// The first group past where this copy was replaced, if any: one replaced before @@ -1461,6 +1472,7 @@ impl Reader { // Resume where a fresh copy would: a group it buffered while replaced, below the // newest another route handed out, is not handed out. copy.floor = copy.floor.max(self.newest); + copy.resume = self.resume_at(copy.floor); let start = copy.start(self.groups.0); if let (Sub::Ready(sub), Some(_)) = (&mut copy.sub, copy.floor) { sub.set_range(( @@ -1469,20 +1481,24 @@ impl Reader { )); } copy.resolved = resolved; - let wire = self.mirrored.clamped(copy.floor, None, ©.resolved); + let mut wire = self.mirrored.clamped(copy.floor, None, ©.resolved); + wire.resume = copy.resume; copy.send(wire); self.copies.push(copy); return; } let floor = self.newest; + let resume = self.resume_at(floor); // The wire request carries the floor from the first message: some sessions read // the demand only once. - let wire = self.mirrored.clamped(floor, None, &resolved); + let mut wire = self.mirrored.clamped(floor, None, &resolved); + wire.resume = resume; self.copies.push(Copy { generation, sub: Sub::Pending(Box::new(track.subscribe(wire))), track, floor, + resume, from: self.mirrored.ranges.clone(), resolved, replaced: false, @@ -1495,6 +1511,23 @@ impl Reader { self.learn_end(None); } + /// Where a new copy asked from group `key` resumes it: past the frames of it the copies + /// hold from its first, joined across copies, which the reader already has or reads + /// locally. A route switch behind the same next hop then sends none of them again, only + /// the rest of a group still arriving, and nothing of a finished one. `None` while no + /// copy holds its first frame. + fn resume_at(&self, key: Option) -> Option { + let key = key?; + let mut spans: Vec<_> = (self.copies.iter()) + .filter_map(|copy| copy.track.peek_group(key)?.committed_span()) + .collect(); + spans.sort_unstable_by_key(|span| span.start); + let held = spans + .into_iter() + .fold(0, |held, span| if span.start <= held { held.max(span.end) } else { held }); + (held > 0).then_some(Position::new(key, held)) + } + /// One logical track has one end, whichever route first reports it. fn learn_end(&mut self, waiter: Option<&kio::Waiter>) { for copy in &mut self.copies { @@ -2199,7 +2232,8 @@ impl Reader { copy.since = Some(copy.track.now()); copy.from = self.mirrored.ranges.clone(); copy.resolved = resolved; - let wire = self.mirrored.clamped(copy.floor, copy.end(), ©.resolved); + let mut wire = self.mirrored.clamped(copy.floor, copy.end(), ©.resolved); + wire.resume = copy.resume; copy.send(wire); return; } @@ -2209,6 +2243,8 @@ impl Reader { generation, sub: Sub::Pending(Box::new(track.subscribe(wire))), floor, + // Below every group handed out, so the reader holds none of the first. + resume: None, from: self.mirrored.ranges.clone(), resolved, // Bounded from the start: everything it may deliver is older than what the reader @@ -4002,6 +4038,63 @@ mod test { ); } + /// A switch asks the new copy for the newest group handed out from past the frames the + /// copies hold of it, never again from its first: the rest of one still arriving, and + /// nothing of a finished one. The reader reads the group on from the part the new copy + /// gets, every frame once. + #[test] + #[cfg(feature = "async")] + fn a_switch_resumes_the_newest_group_past_the_frames_held() { + let routes = Producer::new(); + let track = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&track, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + open.write_frame(ts(1), b"b".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + let asked = b.subscription().expect("subscribed"); + assert_eq!(asked.start(), Some(0)); + assert_eq!(asked.resume, Some(Position::new(0, 2)), "past the two frames held"); + + // The new copy gets only the rest, and the reader reads on from it. + let mut rest = b.create_group(group::Info { start: 0 }).unwrap(); + rest.start_at(2).unwrap(); + rest.write_frame(ts(2), b"c".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut reading), Some(b"c".to_vec())); + assert_eq!(read(&mut reading), None); + assert!(sub.recv_group().now_or_never().is_none(), "handed out twice"); + + // A finished group is resumed past its every frame, so none of it is sent again. + write(&b, 1); + assert_eq!(recv(&mut sub).start, 1); + let c = copy(); + routes.serve(c.consume()); + assert_eq!(c.subscription().expect("subscribed").resume, Some(Position::new(1, 1))); + + // A reader holding none of the group asks for it whole, and outweighs one resuming. + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut first = subscribe(&front, HOUR); + let mut second = subscribe(&front, HOUR); + write(&a, 5); + assert_eq!(recv(&mut first).start, 5); + let demand = front.subscription().unwrap(); + assert_eq!(routes.subscription(demand.clone(), 2).resume, None); + assert_eq!(recv(&mut second).start, 5); + assert_eq!(routes.subscription(demand, 2).resume, Some(Position::new(5, 1))); + } + /// Routes that close without concluding read like a conclusion: a reader stalled /// mid-group parks on the copy it is reading rather than polling the closed routes /// forever, and finishes the group from that copy. diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index 126a4178e30..2af7b12c2c3 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -48,6 +48,11 @@ pub struct Subscription { /// answer gives. A session holding no upstream subscription for its copy subscribes again /// for such a reader, whatever its ranges. Aggregated as any: one such reader is enough. pub(crate) feed: bool, + /// The frames this reader already holds of the group a range starts at, so that range asks + /// for it from the frame after them rather than whole: a route switch resuming on a new + /// copy. Only a range whose start is a [`Anchor::Time`] at exactly that group's start uses + /// it. Aggregated only when every reader resumes: the lowest position then. + pub(crate) resume: Option, } /// How far behind the live edge a subscriber plays: where it starts, and how late it may fall. @@ -372,6 +377,7 @@ impl Default for Subscription { ranges: vec![Range::oldest()], order: Order::Descending, feed: false, + resume: None, } } } @@ -570,6 +576,7 @@ impl Combined { combined.order = Order::Descending; } combined.feed |= sub.feed; + combined.resume = combined.resume.zip(sub.resume).map(|(a, b)| a.min(b)); } /// The aggregate, or `None` without a subscriber. diff --git a/rs/moq-net/src/model/track/demand.rs b/rs/moq-net/src/model/track/demand.rs index 5a365a1e429..9a59bb99259 100644 --- a/rs/moq-net/src/model/track/demand.rs +++ b/rs/moq-net/src/model/track/demand.rs @@ -9,7 +9,7 @@ use std::{ time::Duration, }; -use super::{Delay, Order, Range, Subscription, TrackState, clamp_combined}; +use super::{Delay, Order, Position, Range, Subscription, TrackState, clamp_combined}; mod ranges; use ranges::Ranges; @@ -26,6 +26,8 @@ pub(super) struct Subscriptions { descending: usize, /// Subscribers that need the live feed; see [`Subscription::feed`]. feed: usize, + /// Where resuming subscribers resume; see [`Subscription::resume`]. + resumes: BTreeMap, /// The lowest group consumers want while no refresh is pending, never lowered. wanted: u64, /// A request for a group past every key written, until a group answers it or its @@ -63,17 +65,18 @@ pub(super) enum Requested { } /// Everything the aggregate holds besides its ranges. -type Scalars = (i8, Delay, Order, bool); +type Scalars = (i8, Delay, Order, bool, Option); impl Subscriptions { pub(super) fn combined(&self) -> Option { - let (priority, delay, order, feed) = self.scalars()?; + let (priority, delay, order, feed, resume) = self.scalars()?; Some(Subscription { priority, delay, ranges: self.ranges.union(), order, feed, + resume, }) } @@ -99,6 +102,11 @@ impl Subscriptions { Order::Descending }, self.feed != 0, + // A subscriber asking for the whole group outweighs any resuming one. + match self.resumes.values().sum::() == self.count { + true => self.resumes.first_key_value().map(|(resume, _)| *resume), + false => None, + }, ) }) } @@ -267,6 +275,9 @@ impl Subscriptions { } self.descending += usize::from(new.order == Order::Descending); self.feed += usize::from(new.feed); + if let Some(resume) = new.resume { + add(&mut self.resumes, resume); + } self.count += 1; } if let Some(old) = old { @@ -280,6 +291,9 @@ impl Subscriptions { } self.descending -= usize::from(old.order == Order::Descending); self.feed -= usize::from(old.feed); + if let Some(resume) = old.resume { + remove(&mut self.resumes, resume); + } self.count -= 1; } if reshaped || self.scalars() != prev { @@ -494,6 +508,9 @@ mod tests { Order::Descending }, feed: rng.random_bool(0.25), + resume: rng + .random_bool(0.5) + .then(|| Position::new(rng.random_range(0..8), rng.random_range(1..4))), }; match (&mut slots[index], rng.random_range(0..3)) { (slot @ None, _) => *slot = Some((producer.subscribe(next.clone()), next)), diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 19f0db21774..9d904febfaf 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -274,7 +274,7 @@ fn lite_values() -> Vec { /// One GROUP header and its frames, as a publisher writes them: a zigzag timestamp delta on /// a timed track, the payload size, then the payload. The header names a previous start, -/// which only lite-07 carries, and a frame start, which only lite-06 carries. +/// which only lite-07 carries, and a frame start, which lite-06 and later carry. fn lite_group(version: lite::Version, frames: impl IntoIterator, usize)>) -> Vec { let mut buf = Vec::new(); let w = &mut Encoder::new(&mut buf, version.into()); @@ -282,7 +282,7 @@ fn lite_group(version: lite::Version, frames: impl IntoIterator Duration { None => assert_eq!(shown, None, "{case}: let go"), } } - // Pinned, not endorsed, until [the resend fix](/quest/m2/multicast/resend.md): a route - // moving behind an unchanged next hop sends that hop the newest group again, once. - let doubled = mesh.doubled(); - eprintln!("measured failures at one instant, seed {seed:#x}: groups sent twice {doubled:?}"); - let links: BTreeSet<_> = doubled.iter().map(|&(from, to, _, _)| (from, to)).collect(); - assert!( - links.len() == doubled.len() && doubled.iter().all(|doubled| doubled.3 == 2), - "seed {seed:#x}: {doubled:?}" - ); + assert_eq!(mesh.doubled(), [], "seed {seed:#x}: no group crosses a link twice"); worst } @@ -221,11 +213,10 @@ mod failures_at_one_instant { /// 1 and `Y` at 2, with `B` (`R3`) reading through `A`. The link from `X` to `A` closes: /// `A` moves onto `Y`, and `B` keeps `A` as its next hop while its route moves from `X` /// to `Y`. A lite-07 request names the relay it routes to, so `B` asks `A` anew, from the -/// newest group it handed out: a range cannot start past a group whose end it does not -/// know. That group crosses `A`-`B` twice. Pinned, not endorsed, until -/// [the resend fix](/quest/m2/multicast/resend.md). +/// position past the frames it holds of the newest group it handed out, and no group +/// crosses `A`-`B` twice. #[kio::test] -async fn a_route_moving_behind_one_next_hop_resends_a_group() { +async fn a_route_moving_behind_one_next_hop_resends_no_group() { let mut mesh = Mesh::new(4); mesh.join(0, 2, Join::peer(1)).await; mesh.join(1, 2, Join::peer(2)).await; @@ -243,11 +234,7 @@ async fn a_route_moving_behind_one_next_hop_resends_a_group() { for (i, report) in reports(&mut readers, since, "X-A cut").iter().enumerate().skip(2) { assert!(report.seamless(), "R{i}: {report:?}"); } - let doubled = mesh.doubled(); - assert!( - matches!(doubled.as_slice(), [(2, 3, _, 2)]), - "one group crosses A-B twice: {doubled:?}" - ); + assert_eq!(mesh.doubled(), [], "no group crosses A-B twice"); } /// A ring of six with a publisher on `R0` and a reader on every relay. A link from `R0` to From eeabef3a47d45d022770a5e08d98c0f33b0bfcd0 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:57:27 -0800 Subject: [PATCH 03/21] wip: a resumed copy adopts the head it was asked past --- rs/moq-net/src/model/group.rs | 16 ++----- rs/moq-net/src/model/resume.rs | 84 +++++++++++++++++++++++++++------- 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index f15807e3744..4e21b8a0734 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -401,6 +401,11 @@ impl Handoff { } impl Head { + /// How many frames it holds from the group's first, or `None` when it starts later. + pub(super) fn from_first(&self) -> Option { + (self.start == 0 && !self.entries.is_empty()).then_some(self.entries.len() as u64) + } + /// Roughly what holding it costs: payloads, fields, and their slots. fn bytes(&self) -> u64 { let payload: u64 = self.entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); @@ -4619,17 +4624,6 @@ impl Consumer { self.cursor.state.read().committed() } - /// The frames committed to the copy this reads, from the index it started at: what a route - /// switch need not ask for again. `None` once aborted, or on a sparse wire, whose object - /// IDs are not frame indices. - pub(super) fn committed_span(&self) -> Option> { - let state = self.cursor.state.read(); - if state.outcome.error().is_some() || state.sparse() { - return None; - } - Some(state.offset as u64..state.committed() as u64) - } - fn poll_read_frame_copy( &mut self, waiter: &kio::Waiter, diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c8ca625a24b..a101a4e4b99 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -209,7 +209,7 @@ impl Producer { _ => Vec::new(), }; ranges.extend(preferences.clamped(reader.newest, None, &resolved).ranges); - resumes.push(reader.resume_at(reader.newest)); + resumes.push(reader.resume_at(reader.newest).map(|(resume, _)| resume)); } demand.ranges = ranges; // As in a track's own demand, only when every reader resumes. @@ -609,6 +609,9 @@ struct Copy { /// The frames of the group at `floor` the reader already held when this copy was asked, /// so it asks for that group from the next frame; see [`Reader::resume_at`]. resume: Option, + /// Those frames, lent to this copy once the rest of their group lands here, so it holds + /// the group whole for a reader that joins later; see [`Reader::lend_heads`]. + head: Option>, /// The reader's ranges [`Self::resolved`] are for. from: Vec, /// Each of those ranges resolved, by index, so clamping never widens one: as the @@ -1472,7 +1475,7 @@ impl Reader { // Resume where a fresh copy would: a group it buffered while replaced, below the // newest another route handed out, is not handed out. copy.floor = copy.floor.max(self.newest); - copy.resume = self.resume_at(copy.floor); + (copy.resume, copy.head) = self.resume_at(copy.floor).unzip(); let start = copy.start(self.groups.0); if let (Sub::Ready(sub), Some(_)) = (&mut copy.sub, copy.floor) { sub.set_range(( @@ -1488,7 +1491,7 @@ impl Reader { return; } let floor = self.newest; - let resume = self.resume_at(floor); + let (resume, head) = self.resume_at(floor).unzip(); // The wire request carries the floor from the first message: some sessions read // the demand only once. let mut wire = self.mirrored.clamped(floor, None, &resolved); @@ -1499,6 +1502,7 @@ impl Reader { track, floor, resume, + head, from: self.mirrored.ranges.clone(), resolved, replaced: false, @@ -1511,21 +1515,36 @@ impl Reader { self.learn_end(None); } - /// Where a new copy asked from group `key` resumes it: past the frames of it the copies - /// hold from its first, joined across copies, which the reader already has or reads - /// locally. A route switch behind the same next hop then sends none of them again, only - /// the rest of a group still arriving, and nothing of a finished one. `None` while no - /// copy holds its first frame. - fn resume_at(&self, key: Option) -> Option { + /// Where a new copy asked from group `key` resumes it: past the most frames of it a copy + /// holds from its first, which the reader already has or reads locally, and those frames. + /// A route switch behind the same next hop then sends none of them again, only the rest of + /// a group still arriving, and nothing of a finished one. `None` while no copy holds its + /// first frame. + fn resume_at(&self, key: Option) -> Option<(Position, Arc)> { let key = key?; - let mut spans: Vec<_> = (self.copies.iter()) - .filter_map(|copy| copy.track.peek_group(key)?.committed_span()) - .collect(); - spans.sort_unstable_by_key(|span| span.start); - let held = spans - .into_iter() - .fold(0, |held, span| if span.start <= held { held.max(span.end) } else { held }); - (held > 0).then_some(Position::new(key, held)) + (self.copies.iter()) + .filter_map(|copy| copy.track.peek_group(key)?.handoff().head()) + .filter_map(|head| Some((head.from_first()?, head))) + .max_by_key(|(frames, _)| *frames) + .map(|(frames, head)| (Position::new(key, frames), head)) + } + + /// Lend each copy asked to resume a group the frames it was asked past, once the rest of + /// that group lands there, so it holds the group whole for a reader that joins later + /// after the copies holding them are gone. + fn lend_heads(&mut self) { + for copy in &mut self.copies { + let (Some(resume), Some(head)) = (copy.resume, ©.head) else { + continue; + }; + let Some(group) = copy.track.peek_group(resume.group) else { + continue; + }; + if group.first_frame() > 0 { + copy.track.adopt_head(resume.group, head); + } + copy.head = None; + } } /// One logical track has one end, whichever route first reports it. @@ -1740,6 +1759,7 @@ impl Reader { waiter: &kio::Waiter, ) -> Option>>> { self.sync(waiter); + self.lend_heads(); if let Some(err) = self.refusal() { return Some(Poll::Ready(Err(err))); } @@ -2245,6 +2265,7 @@ impl Reader { floor, // Below every group handed out, so the reader holds none of the first. resume: None, + head: None, from: self.mirrored.ranges.clone(), resolved, // Bounded from the start: everything it may deliver is older than what the reader @@ -4095,6 +4116,35 @@ mod test { assert_eq!(routes.subscription(demand, 2).resume, Some(Position::new(5, 1))); } + /// A copy asked past the frames a reader held of a group adopts them once the rest lands, + /// so a reader joining after the copy that held them is gone reads the group whole. + #[test] + #[cfg(feature = "async")] + fn a_resumed_copy_holds_the_group_whole_for_a_later_reader() { + let routes = Producer::new(); + let track = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut holder = subscribe(&track, HOUR); + write(&a, 0); + assert_eq!(recv(&mut holder).start, 0); + + let b = copy(); + routes.serve(b.consume()); + assert_eq!(b.subscription().unwrap().resume, Some(Position::new(0, 1))); + let mut rest = b.create_group(group::Info { start: 0 }).unwrap(); + rest.start_at(1).unwrap(); + rest.finish().unwrap(); + assert!(holder.recv_group().now_or_never().is_none(), "handed out twice"); + drop(a); + + let mut late = subscribe(&track, HOUR); + let mut group = recv(&mut late); + assert_eq!(group.start, 0); + assert_eq!(read(&mut group), Some(b"x".to_vec())); + assert_eq!(read(&mut group), None); + } + /// Routes that close without concluding read like a conclusion: a reader stalled /// mid-group parks on the copy it is reading rather than polling the closed routes /// forever, and finishes the group from that copy. From 969f365fe871c0a38678ed503974ea01892352ad Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:17:13 -0800 Subject: [PATCH 04/21] wip: js/net reads and serves a lite-07 position start --- js/net/src/lite/group.test.ts | 14 ++++--- js/net/src/lite/group.ts | 8 ++-- js/net/src/lite/subscribe.test.ts | 35 +++++++++++++++- js/net/src/lite/subscribe.ts | 67 +++++++++++++++++++++++-------- js/net/src/lite/version.ts | 14 ++----- rs/moq-net/src/model/group.rs | 2 +- rs/moq-net/src/model/resume.rs | 2 +- 7 files changed, 102 insertions(+), 40 deletions(-) diff --git a/js/net/src/lite/group.test.ts b/js/net/src/lite/group.test.ts index f1f11b6e54b..9b010d96235 100644 --- a/js/net/src/lite/group.test.ts +++ b/js/net/src/lite/group.test.ts @@ -219,23 +219,25 @@ test("a scale-less lite-07 FRAME carries its group's start", async () => { }); // GROUP carries the previous group's start plus one on lite-07, 0 when unknown, and passes over the -// starts strictly between; earlier versions carry nothing. Only lite-06 carries a Frame Start. +// starts strictly between; earlier versions carry nothing. Lite-06 on carries a Frame Start. test("a lite-07 GROUP carries its Previous Start plus one", async () => { const encode = (previous: bigint | undefined, version: Version) => Writer.collect(version, (w) => new Group({ subscribe: 1n, key: 7n, previous }).encode(w, version)); - expect(Array.from(await encode(4n, Version.DRAFT_07))).toEqual([3, 1, 7, 5]); - expect(Array.from(await encode(undefined, Version.DRAFT_07))).toEqual([3, 1, 7, 0]); + expect(Array.from(await encode(4n, Version.DRAFT_07))).toEqual([4, 1, 7, 5, 0]); + expect(Array.from(await encode(undefined, Version.DRAFT_07))).toEqual([4, 1, 7, 0, 0]); expect(Array.from(await encode(4n, Version.DRAFT_06))).toEqual([3, 1, 7, 0]); const partial = (version: Version) => Writer.collect(version, (w) => new Group({ subscribe: 1n, key: 7n, frameStart: 4 }).encode(w, version)); expect(Array.from(await partial(Version.DRAFT_06))).toEqual([3, 1, 7, 4]); - await expect(partial(Version.DRAFT_07)).rejects.toThrow(); + expect(Array.from(await partial(Version.DRAFT_07))).toEqual([4, 1, 7, 0, 4]); + await expect(partial(Version.DRAFT_05)).rejects.toThrow(); const decode = (bytes: number[], version: Version) => Group.decode(streamOf([new Uint8Array(bytes)], version), version); - const group = await decode([3, 1, 7, 5], Version.DRAFT_07); + const group = await decode([4, 1, 7, 5, 0], Version.DRAFT_07); expect([group.previous, group.passedOver(), group.frameStart]).toEqual([4n, { start: 5n, end: 7n }, 0]); - expect((await decode([3, 1, 7, 0], Version.DRAFT_07)).passedOver()).toEqual({ start: 7n, end: 7n }); + expect((await decode([4, 1, 7, 0, 2], Version.DRAFT_07)).frameStart).toBe(2); + expect((await decode([4, 1, 7, 0, 0], Version.DRAFT_07)).passedOver()).toEqual({ start: 7n, end: 7n }); expect((await decode([3, 1, 7, 0], Version.DRAFT_06)).previous).toBeUndefined(); }); diff --git a/js/net/src/lite/group.ts b/js/net/src/lite/group.ts index 08c9c8370a5..8bfbf07e85f 100644 --- a/js/net/src/lite/group.ts +++ b/js/net/src/lite/group.ts @@ -8,7 +8,7 @@ import * as Time from "../time.ts"; import { toNumber } from "../util/bigint.ts"; import { MAX_FRAME_BYTES } from "../util/limits.ts"; import * as Message from "./message.ts"; -import { hasGroupEnd, hasGroupFrameStart, type Version } from "./version.ts"; +import { hasFrameBounds, hasGroupEnd, type Version } from "./version.ts"; export class Group { subscribe: bigint; @@ -26,7 +26,7 @@ export class Group { * * 0 (the common case) means the stream carries the group from its beginning. A * higher value means the leading frames are not here, because the subscription started - * partway into the group. Lite-06 only; other versions always start at 0. + * partway into the group. Lite-06+ only; older versions always start at 0. */ frameStart: number; @@ -60,7 +60,7 @@ export class Group { await w.u62(this.key); // Plus one, so 0 says unknown. if (hasGroupEnd(version)) await w.u62(this.previous === undefined ? 0n : this.previous + 1n); - if (hasGroupFrameStart(version)) { + if (hasFrameBounds(version)) { await w.u53(this.frameStart); } else if (this.frameStart !== 0) { // The peer would number the frames from 0 and silently misalign the group. @@ -73,7 +73,7 @@ export class Group { const key = await r.u62(); const plusOne = hasGroupEnd(version) ? await r.u62() : 0n; const previous = plusOne === 0n ? undefined : plusOne - 1n; - const frameStart = hasGroupFrameStart(version) ? await r.u53() : 0; + const frameStart = hasFrameBounds(version) ? await r.u53() : 0; return new Group({ subscribe, key: key, previous, frameStart }); } diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 8af416c3dd4..3b31914e14c 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -8,6 +8,7 @@ import { encodeSubscribeResponse, inclusiveGroupEnd, LiveRanges, + rangeSize, requestedRanges, Subscribe, SubscribeDrop, @@ -239,11 +240,43 @@ test("a lite-07 list decodes its time and relative anchors", async () => { // An update may add no range, retiring some or none. expect((await update([0])).ranges).toEqual([]); - // Any other type is a violation. + // Any other type is a violation, and so is a Position at frame 0 or as an end. await expect(update([1, 3, 0, 0])).rejects.toThrow(ProtocolViolation); + await expect(update([1, 1, 0, 3, 7, 4])).rejects.toThrow(ProtocolViolation); await expect(update([1, 4, 0, 0])).rejects.toThrow(ProtocolViolation); }); +test("a lite-07 range may start partway through a group, which the span keeps", async () => { + const position = { start: { absolute: 7n, frame: 4 } }; + const update = new SubscribeUpdate({ priority: 0, ranges: [position] }); + const bytes = await encodeMessage(Version.DRAFT_07, update); + // One range: type 3, start 7, frame 4, then no end, and an absent wanted group. + expect(Array.from(bytes.slice(-6))).toEqual([1, 3, 7, 4, 0, 0]); + expect(rangeSize(position)).toBe(4); + const got = await SubscribeUpdate.decode(new Reader(undefined, bytes, Version.DRAFT_07), Version.DRAFT_07); + expect(got.ranges).toEqual([position]); + const span = spanOf(Version.DRAFT_07, got.ranges ?? []); + expect([span.startGroup, span.startFrame]).toEqual([7n, 4]); + // The model reads the group whole: the publisher cuts the frames. + expect(requestedRanges(span)).toEqual([{ start: { time: 7n } }]); + + // Another range that may cover the group asks for it whole, since a group is delivered once. + for (const other of [{ start: { time: 3n } }, { start: { relative: 0n } }, { end: { time: 9n } }]) { + expect(spanOf(Version.DRAFT_07, [position, other]).startFrame).toBe(0); + } + expect(spanOf(Version.DRAFT_07, [position, { start: { time: 9n } }]).startFrame).toBe(4); + // The span qualifies only its lowest start, so a Position above another range's goes whole. + expect(spanOf(Version.DRAFT_07, [position, { start: { time: 1n }, end: { time: 3n } }]).startFrame).toBe(0); + + // An end there can't go out. + await expect( + encodeMessage( + Version.DRAFT_07, + new SubscribeUpdate({ priority: 0, ranges: [{ end: { absolute: 9n, frame: 2 } }] }), + ), + ).rejects.toThrow("a range ends only at a group"); +}); + test("lite-07 sends an absolute anchor as a time, an end as the time before it", async () => { const update = new SubscribeUpdate({ priority: 0, diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 94e7f6972ac..04ad8800836 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -130,11 +130,12 @@ interface Span { /** * One end of a lite-07 range: a model {@link Anchor}, or a group by its start (`absolute`), as - * the Rust `lite::Anchor` holds. The wire has no absolute anchor: a group's start is its time, so - * `absolute` goes out as a time, an end (exclusive) as the time just before it, and decodes as one. + * the Rust `lite::Anchor` holds. A group's start is its time, so `absolute` goes out as a time, an + * end (exclusive) as the time just before it, and decodes as one. A start partway through the + * group, at a nonzero `frame`, goes out as a Position; an end there can't. */ export type WireAnchor = - | { absolute: bigint; relative?: never; time?: never } + | { absolute: bigint; frame?: number; relative?: never; time?: never } | { relative: bigint; absolute?: never; time?: never } | { time: bigint; absolute?: never; relative?: never }; @@ -150,12 +151,17 @@ export function toWireRanges(ranges: readonly Range[]): WireRange[] { } /** - * The lite-07 type and value of a range's start or `end`, `undefined` for none. Both time ends are + * The lite-07 type and values of a range's start or `end`, none for none. Both time ends are * inclusive, so an absolute end, exclusive, goes out as the time just before it; one at group 0 - * holds nothing a time can say. + * holds nothing a time can say. A start partway through a group is a Position: its start, then + * the frame. */ -function wireAnchor(anchor: WireAnchor | undefined, end: boolean): [number, bigint?] { +function wireAnchor(anchor: WireAnchor | undefined, end: boolean): [number, ...bigint[]] { if (anchor?.absolute !== undefined) { + if (anchor.frame) { + if (end) throw new Error("a range ends only at a group"); + return [3, anchor.absolute, BigInt(anchor.frame)]; + } if (!end) return [1, anchor.absolute]; if (anchor.absolute === 0n) throw new Error(EMPTY_RANGE); return [1, anchor.absolute - 1n]; @@ -171,9 +177,9 @@ async function encodeRange(w: Writer, range: WireRange) { [range.start, false], [range.end, true], ] as const) { - const [kind, value] = wireAnchor(anchor, end); + const [kind, ...values] = wireAnchor(anchor, end); await w.u53(kind); - if (value !== undefined) await w.u62(value); + for (const value of values) await w.u62(value); } } @@ -235,8 +241,11 @@ export function requestedRanges(msg: Span & { ranges?: WireRange[] }): Range[] { return [range]; } -/** Decode one lite-07 range end, `undefined` for none: a time, inclusive as an end, or a count back. */ -async function decodeAnchor(r: Reader): Promise { +/** + * Decode one lite-07 range end, `undefined` for none: a time, inclusive as an end, a count back, + * or a start partway through a group, never at frame 0, which a time says. + */ +async function decodeAnchor(r: Reader, end: boolean): Promise { const kind = await r.u53(); switch (kind) { case 0: @@ -245,6 +254,13 @@ async function decodeAnchor(r: Reader): Promise { return { time: await r.u62() }; case 2: return { relative: await r.u62() }; + case 3: { + if (end) throw new ProtocolViolation("a range ends partway through a group"); + const absolute = await r.u62(); + const frame = await r.u53(); + if (frame === 0) throw new ProtocolViolation("a position at frame 0"); + return { absolute, frame }; + } default: throw new ProtocolViolation(`unknown range anchor: ${kind}`); } @@ -252,8 +268,8 @@ async function decodeAnchor(r: Reader): Promise { /** Decode one lite-07 range: its start, then its end. */ async function decodeRange(r: Reader): Promise { - const start = await decodeAnchor(r); - const end = await decodeAnchor(r); + const start = await decodeAnchor(r, false); + const end = await decodeAnchor(r, true); const range: WireRange = {}; if (start) range.start = start; if (end) range.end = end; @@ -270,7 +286,9 @@ async function decodeRanges(r: Reader): Promise { /** * Lite-07 `ranges` and the one span covering every one of them, which the subscriber narrows - * again. A range covers whole groups, so the span bounds no frame. + * again. A range covers whole groups but for a Position start, whose frame the span keeps when + * the span starts there and no other range may cover that group: each group is delivered once. + * A Position above the span's start goes whole, as more than it asked. */ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { ranges: WireRange[] } { let start: bigint | undefined; @@ -293,11 +311,26 @@ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { return { ranges: [...ranges], startGroup: canonicalStartGroup(version, start, 0), - startFrame: 0, + startFrame: start === undefined ? 0 : positionFrame(ranges, start), endGroup: end === undefined ? undefined : end - 1n, }; } +/** The frame a Position start asks group `group` from, 0 unless no other range may cover it. */ +function positionFrame(ranges: readonly WireRange[], group: bigint): number { + let frame = 0; + let covering = 0; + for (const range of ranges) { + const from = range.start?.absolute ?? range.start?.time; + const to = range.end?.absolute ?? (range.end?.time === undefined ? undefined : range.end.time + 1n); + // Unknown ends, such as a count back, may cover it. + if ((from !== undefined && from > group) || (to !== undefined && to <= group)) continue; + covering++; + if (range.start?.absolute === group && range.start.frame) frame = range.start.frame; + } + return covering === 1 ? frame : 0; +} + /** * The inverse of {@link encodeSpan}. A lite-07 list decodes as its ranges and as the one span * covering every range in it, which the subscriber narrows again. @@ -340,8 +373,8 @@ export function rangeSize(range: WireRange): number { const anchor = (anchor: WireAnchor | undefined, end: boolean): number => { // An end at group 0 can't go out, which encoding refuses; size it as the smallest. if (end && anchor?.absolute === 0n) return size(1) + size(0); - const [kind, value] = wireAnchor(anchor, end); - return size(kind) + (value === undefined ? 0 : size(value)); + const [kind, ...values] = wireAnchor(anchor, end); + return values.reduce((total, value) => total + size(value), size(kind)); }; return anchor(range.start, false) + anchor(range.end, true); } @@ -349,7 +382,7 @@ export function rangeSize(range: WireRange): number { /** A range's anchors as a key, so equal ranges compare equal. */ function rangeKey(range: WireRange): string { const anchor = (anchor: WireAnchor | undefined) => { - if (anchor?.absolute !== undefined) return `a${anchor.absolute}`; + if (anchor?.absolute !== undefined) return `a${anchor.absolute}.${anchor.frame ?? 0}`; if (anchor?.relative !== undefined) return `r${anchor.relative}`; if (anchor?.time !== undefined) return `t${anchor.time}`; return "-"; diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index 47800bf27fc..201e5cf81ac 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -234,10 +234,10 @@ export function hasRouteCost(version: Version): boolean { } } -/** Whether SUBSCRIBE, SUBSCRIBE_UPDATE, and FETCH carry frame indices alongside - * their group keys, so a subscription or fetch can start and end partway through a - * group. Added in lite-06. Older versions only address whole groups, so a route change - * has to wait for the next group before it can resume. +/** Whether SUBSCRIBE, SUBSCRIBE_UPDATE, FETCH, and GROUP carry frame indices alongside + * their group keys, so a subscription or fetch can start partway through a group. Added in + * lite-06; lite-07 bounds only a range's start. Older versions only address whole groups, so a + * route change has to wait for the next group before it can resume. * * SUBSCRIBE_OK is deliberately not in that list: the resolved start frame follows from * its group plus the subscriber's own request, so it needs no frame field. */ @@ -255,12 +255,6 @@ export function hasFrameBounds(version: Version): boolean { } } -/** Whether GROUP carries `Frame Start`, the index of the stream's first frame. Lite-06 only: - * earlier versions have no frame bounds, and a lite-07 subscription reads whole groups. */ -export function hasGroupFrameStart(version: Version): boolean { - return version === Version.DRAFT_06; -} - /** Whether SUBSCRIBE and SUBSCRIBE_UPDATE carry the subscriber's `Group Order`. Added in * lite-07; an earlier subscription is served newest first. */ export function hasOrder(version: Version): boolean { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 4e21b8a0734..37306684579 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -402,7 +402,7 @@ impl Handoff { impl Head { /// How many frames it holds from the group's first, or `None` when it starts later. - pub(super) fn from_first(&self) -> Option { + pub(super) fn frames_from_first(&self) -> Option { (self.start == 0 && !self.entries.is_empty()).then_some(self.entries.len() as u64) } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index a101a4e4b99..34683570807 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1524,7 +1524,7 @@ impl Reader { let key = key?; (self.copies.iter()) .filter_map(|copy| copy.track.peek_group(key)?.handoff().head()) - .filter_map(|head| Some((head.from_first()?, head))) + .filter_map(|head| Some((head.frames_from_first()?, head))) .max_by_key(|(frames, _)| *frames) .map(|(frames, head)| (Position::new(key, frames), head)) } From b57a0c95ae64925e5d80e897174dbe0630d2763a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:41:52 -0800 Subject: [PATCH 05/21] feat(net)!: a lite-07 range may start partway through a group, so a route switch resends nothing A route switch behind one next hop asks the new copy from the position past the frames held of the newest group handed out, and that copy adopts those frames once the rest lands. GROUP carries Frame Start again on lite-07. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 2 +- js/net/src/lite/subscribe.test.ts | 9 ++- js/net/src/lite/subscribe.ts | 10 +++- quest/m2/multicast/README.md | 10 ++-- quest/m2/multicast/resend.md | 59 ------------------- rs/moq-net/src/lite/subscribe.rs | 5 +- rs/moq-net/src/lite/version.rs | 4 +- rs/moq-net/src/model/resume.rs | 3 +- .../tests/integration/receive_credit.rs | 5 +- 9 files changed, 32 insertions(+), 75 deletions(-) delete mode 100644 quest/m2/multicast/resend.md diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index da4a4ffa7bd..3d5b088423b 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -1549,7 +1549,7 @@ The publisher starts from what it holds and never fetches to reach further back, A Range covers whole Groups but for a Position `Start`; a subscriber reads a slice of a Group with FETCH (see [Positions](#positions)). A Group that another Range also covers is delivered whole, since each Group is delivered once. A subscriber resuming a Track on a new subscription, such as a relay moving to another route, starts at the Position past the frames it holds, so no frame it holds is sent again. -The Group Stream of a Group that ended at or before `Frame` carries no FRAME, only how the Group ended. +The Group Stream of a Group that ended at `Frame` carries no FRAME, only how the Group ended. ## SUBSCRIBE_UPDATE {#subscribe-update} A subscriber can modify a subscription with a SUBSCRIBE_UPDATE message. diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 3b31914e14c..10993d68561 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -189,11 +189,18 @@ test("lite-07 carries the span as its only range, exclusive at the end", async ( new Reader(undefined, await encodeMessage(Version.DRAFT_07, msg), Version.DRAFT_07), Version.DRAFT_07, ); - // A range covers whole groups: a slice of one is a FETCH. + // A range ends only at a group: a slice of one is a FETCH. await expect(encodeMessage(Version.DRAFT_07, message)).rejects.toThrow("frame bounds not supported"); await expect(encodeMessage(Version.DRAFT_07, new Subscribe({ ...message, startFrame: 0 }))).rejects.toThrow( "frame bounds not supported", ); + // It starts partway through one as a Position. + const resumed = await decode(new Subscribe({ ...message, endFrame: undefined })); + expect([resumed.startGroup, resumed.startFrame, resumed.ranges]).toEqual([ + 3n, + 2, + [{ start: { absolute: 3n, frame: 2 }, end: { time: 9n } }], + ]); const whole = await decode(new Subscribe({ ...message, startFrame: 0, endFrame: undefined })); expect([whole.startGroup, whole.startFrame, whole.endGroup, whole.endFrame]).toEqual([3n, 0, 9n, undefined]); const open = await decode( diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 04ad8800836..a97d4e94286 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -202,12 +202,16 @@ async function encodeSpan(w: Writer, version: Version, span: Span, ranges?: read return; } - // A lite-07 range covers whole groups: a slice of one is a FETCH. - if (span.startFrame !== 0 || span.endFrame !== undefined) + // A lite-07 range starts partway through a group as a Position, but ends only at one: a + // slice of a group is a FETCH. + if (span.endFrame !== undefined || (span.startFrame !== 0 && span.startGroup === undefined)) throw new Error("frame bounds not supported for this version"); await w.u53(1); const range: WireRange = {}; - if (span.startGroup !== undefined) range.start = { time: span.startGroup }; + if (span.startGroup !== undefined) + range.start = span.startFrame + ? { absolute: span.startGroup, frame: span.startFrame } + : { time: span.startGroup }; if (span.endGroup !== undefined) range.end = { time: span.endGroup }; await encodeRange(w, range); } diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index c3a17568a5d..3b5f105444e 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -33,11 +33,11 @@ Its proof is `rs/moq-net/verify/route-algebra/`, and the quests written against the old wire were retargeted or deleted. [moq-dev/moq#9111](https://github.com/moq-dev/moq/pull/9111) ran the cases no -test covered on real sessions. The lite-07 cut still waits on this epic's -children, the two flaws it found, -[a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md) -and [a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md), -whose fix changes the wire, +test covered on real sessions; of the two flaws it found, a route moving +behind one next hop no longer resends a group, since a lite-07 range may +start partway through one. The lite-07 cut still waits on this epic's +children, the other flaw, +[a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md), [the mesh replay](/quest/m2/multicast/mesh-replay.md), and [draining dialed links](/quest/m2/multicast/drain-dialed-links.md), and on [route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring diff --git a/quest/m2/multicast/resend.md b/quest/m2/multicast/resend.md deleted file mode 100644 index 9002824c84a..00000000000 --- a/quest/m2/multicast/resend.md +++ /dev/null @@ -1,59 +0,0 @@ -# [M] A route moving behind one next hop resends no group - -## Goal - -When a relay's route moves to another announcing relay behind the same next -hop, the subscription it opens on that hop starts after the newest group its -reader handed out, so no group crosses the link twice. -`route_audit::failures::a_route_moving_behind_one_next_hop_resends_a_group` -and `failures_at_one_instant` in `rs/moq-net/tests/integration/route_audit/` -assert that no link carries a group twice. - -## Plan - -Found by [the uncovered cases](https://github.com/moq-dev/moq/pull/9111): in 2 -of 8 seeded 12-relay meshes with simultaneous failures, one group (a whole -GOP for video) crossed a link twice. - -Root cause: routes are per announcing relay, and a lite-07 request names the -routing node its first hop chose. So when `A`'s route moves from `X` to `Y`, -`B`'s front changes copies although its next hop, `A`, is the same session, -and opens a second SUBSCRIBE on it. The new copy starts at the newest group -the reader handed out, inclusive: `Reader::sync` makes `self.newest` the -copy's floor (`rs/moq-net/src/model/resume.rs`), and `clamped` turns it into -`Anchor::Time(floor)` (`rs/moq-net/src/model/subscription.rs`). `B`'s reader -skips the repeat, but its bytes crossed `A`-`B` twice. It is inclusive because -a lite-07 range cannot say "after group K": a time anchor names the group -containing that time, the next group's start is unknown on a timed track, -and a start partway through a group is refused on lite-07 -(`wire_anchor` in `rs/moq-net/src/lite/subscribe.rs`). Lite-06 could bound -frames this way. A switch to another neighbour sends the group over another -link, so no single link carries it twice, though the bytes are still -redundant. - -Decided 2026-11-19 (maintainer, in the coordinating session, on -[moq-dev/moq#9111](https://github.com/moq-dev/moq/pull/9111)): ✅ option 1, -a SUBSCRIBE range may start at an absolute group and frame position on -lite-07-wip (rejected: 2, the partial fix with no wire change; 4, accepting -the duplicate; 3 stays rejected). It lands before the cut. Warm routing -moves pulls for price, so these switches become common once it ships. - -Options, as they were put: - -1. Wire, on lite-07-wip: a SUBSCRIBE range start may be an absolute - (group, frame) position again, as on lite-06. The GROUP header already - carries `frame_start`. A switch then asks for group K from frame N, N the - frames already handed out: nothing for a finished group, only the unread - tail of one in flight. That also stops a recovered partial group from - crossing whole. Recommended, and decided before the cut, since the wire - freezes then. Updates `js/net` and the lite draft in the same PR. -2. No wire change, partial: when the newest handed-out group finished in the - replaced copy and its end is known (untimed tracks with consecutive keys, - or groups finished with `finish_at`), floor at the next key or the end. - Local to `Reader::sync`, but timed tracks closed with plain `finish()` - keep the duplicate. -3. Keep the existing subscription when only the announcing relay behind the - same next-hop session changes. Rejected: it conflicts with a request - naming its first hop's routing node, which keeps requests loop-free. -4. Accept one group per same-next-hop switch and document the cost. - diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index 78f212ddb4c..decfb09e695 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -381,7 +381,10 @@ fn wire_anchor(anchor: Option, end: bool) -> Result<(u64, [Option; Some(Anchor::Absolute(position)) if position.frame != 0 => (3, [Some(position.group), Some(position.frame)]), Some(Anchor::Absolute(position)) if end => ( 1, - [Some(position.group.checked_sub(1).ok_or(EncodeError::InvalidState)?), None], + [ + Some(position.group.checked_sub(1).ok_or(EncodeError::InvalidState)?), + None, + ], ), Some(Anchor::Absolute(position)) => (1, [Some(position.group), None]), }) diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 53a842c56a0..97d0c3f6baf 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -154,8 +154,8 @@ impl Version { } /// Whether SUBSCRIBE, SUBSCRIBE_UPDATE, FETCH, and GROUP carry frame indices - /// alongside their group keys, so a subscription or fetch can start and end - /// partway through a group. Added in lite-06. + /// alongside their group keys, so a subscription or fetch can start partway through a + /// group. Added in lite-06; lite-07 bounds only a range's start. /// /// Older versions only address whole groups, so a route change has to wait for the /// next group before it can resume. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 34683570807..94d9a1428e7 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -7,7 +7,8 @@ //! [`Subscriber`]: nothing is copied, and with one route it is a passthrough. //! //! When the front switches, each reader subscribes to the new copy from the newest group -//! it handed out. A replaced copy keeps delivering until the new copy delivers its first +//! it handed out, past the frames of it a copy holds, which that copy adopts once the rest +//! lands so a later reader finds the group whole. A replaced copy keeps delivering until the new copy delivers its first //! group, then is bounded just below it (or at the newest group handed out, if later), so a //! group only the old route got after the switch is not lost. It is kept while it may still //! deliver a group under its bound the reader lacks (unread, or still landing within the diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index ee3aa5324a8..0d0782872fb 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -626,8 +626,9 @@ fn group_stream(subscribe: u64, sequence: u64, size: usize) -> Vec { let mut header = Vec::new(); varint(&mut header, subscribe); varint(&mut header, sequence); - // No previous start: 0 says unknown, since the field carries it plus one. A lite-07 - // GROUP has no Frame Start: every Group Stream starts at frame 0. + // No previous start: 0 says unknown, since the field carries it plus one. + varint(&mut header, 0); + // Frame Start: the stream carries the group from its first frame. varint(&mut header, 0); // GROUP, then its size-prefixed header. From 3a51533288cb48fb9629bb92a93181f66bc5c891 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:26:02 -0800 Subject: [PATCH 06/21] fix(net): lend the resumed head on arrival, ask old wires whole, keep every frame start - A copy asked to resume a group takes the frames the reader held when the group arrives, so the original reader no longer has to keep polling. - A wire without frame bounds asks the resumed group whole instead of failing to encode. - The JS span keeps group 0 named when a Position qualifies it. - The publisher keeps a frame start for each group only one range covers. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscribe.test.ts | 3 ++ js/net/src/lite/subscribe.ts | 5 ++- rs/moq-net/src/lite/publisher.rs | 71 +++++++++++++++++++++++-------- rs/moq-net/src/lite/subscriber.rs | 4 ++ rs/moq-net/src/model/resume.rs | 49 ++++++++------------- rs/moq-net/src/model/track.rs | 43 +++++++++++++++++++ 6 files changed, 124 insertions(+), 51 deletions(-) diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 10993d68561..20724828900 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -264,6 +264,9 @@ test("a lite-07 range may start partway through a group, which the span keeps", expect(got.ranges).toEqual([position]); const span = spanOf(Version.DRAFT_07, got.ranges ?? []); expect([span.startGroup, span.startFrame]).toEqual([7n, 4]); + // Group 0 stays named when a frame qualifies it. + const zero = spanOf(Version.DRAFT_07, [{ start: { absolute: 0n, frame: 2 } }]); + expect([zero.startGroup, zero.startFrame]).toEqual([0n, 2]); // The model reads the group whole: the publisher cuts the frames. expect(requestedRanges(span)).toEqual([{ start: { time: 7n } }]); diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index a97d4e94286..37a8292e62f 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -312,10 +312,11 @@ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { if (live) end = undefined; // An exclusive end at the very first group names nothing. if (end === 0n) throw new ProtocolViolation(EMPTY_RANGE); + const startFrame = start === undefined ? 0 : positionFrame(ranges, start); return { ranges: [...ranges], - startGroup: canonicalStartGroup(version, start, 0), - startFrame: start === undefined ? 0 : positionFrame(ranges, start), + startGroup: canonicalStartGroup(version, start, startFrame), + startFrame, endGroup: end === undefined ? undefined : end - 1n, }; } diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 899c7f023aa..9448dc6635b 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -4489,10 +4489,10 @@ fn position_group(group: group::Consumer, frames: &Frames) -> Option, + /// Each group a range starts partway through, and the first frame it serves of it. + first: Vec<(u64, u64)>, /// The group the range ends partway through, and the last frame it serves of it, inclusive. last: Option<(u64, u64)>, /// The lowest group a legacy wire's start serves: its start names the first group at or @@ -4503,10 +4503,9 @@ struct Frames { impl Frames { /// The first frame to serve of `group`: 0 unless the range starts partway through it. fn start(&self, group: u64) -> u64 { - match self.first { - Some((at, first)) if at == group => first, - _ => 0, - } + (self.first.iter()) + .find(|(at, _)| *at == group) + .map_or(0, |(_, first)| *first) } /// The last frame to serve of `group`, inclusive, or `None` for all of it. @@ -4564,7 +4563,7 @@ impl Bounds { if let Some(lite::Anchor::Absolute(start)) = range.start && start.frame != 0 { - bounds.frames.first = Some((start.group, start.frame)); + bounds.frames.first.push((start.group, start.frame)); } let end = match range.end { None => None, @@ -4589,17 +4588,15 @@ impl Bounds { } // A group delivers once across the ranges, so a frame start applies only to a group no // other range may cover: another range's would ask for it from frame 0. - if let Some((group, _)) = bounds.frames.first { - let covers = |range: &Range| match (range.start, range.end) { - (Anchor::Time(start), _) if start > group => false, - (_, Some(Anchor::Time(end))) if end < group => false, + let ranges = &bounds.ranges; + bounds.frames.first.retain(|(group, _)| { + let covers = |range: &&Range| match (range.start, range.end) { + (Anchor::Time(start), _) if start > *group => false, + (_, Some(Anchor::Time(end))) if end < *group => false, _ => true, }; - let mut covering = bounds.ranges.iter().filter(|range| covers(range)); - if covering.next().is_some() && covering.next().is_some() { - bounds.frames.first = None; - } - } + ranges.iter().filter(covers).nth(1).is_none() + }); bounds } @@ -6640,6 +6637,44 @@ mod serve_group_test { assert_eq!((inside.frames.start(4), inside.frames.end(6)), (0, None)); } + /// Each lite-07 range starting partway through a group serves that group from its frame, + /// unless another range may cover it too, which asks for it whole. + #[test] + fn frame_starts_apply_per_group() { + let at = |group, frame| Some(lite::Anchor::Absolute(track::Position { group, frame })); + let time = |time| Some(lite::Anchor::Time(time)); + let disjoint = Bounds::new( + &[ + lite::Range { + start: at(7, 4), + end: time(8), + }, + lite::Range { + start: at(9, 2), + end: None, + }, + ], + false, + ); + assert_eq!((disjoint.frames.start(7), disjoint.frames.start(9)), (4, 2)); + assert_eq!(disjoint.frames.start(8), 0); + + let overlapping = Bounds::new( + &[ + lite::Range { + start: at(7, 4), + end: None, + }, + lite::Range { + start: at(9, 2), + end: None, + }, + ], + false, + ); + assert_eq!((overlapping.frames.start(7), overlapping.frames.start(9)), (4, 0)); + } + /// A group whose head the publisher no longer holds is skipped, not served short: /// only a subscriber that asked for a partial group may receive one. #[test] @@ -6653,7 +6688,7 @@ mod serve_group_test { group.start_at(5).unwrap(); group.write_frame(group.key_time(), b"tail".to_vec()).unwrap(); let starting = |frame| Frames { - first: Some((3, frame)), + first: vec![(3, frame)], ..Frames::default() }; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 596e4dab6b2..0041cbe11a3 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10905,6 +10905,10 @@ impl TrackServe { if opening && idle { subscription = subscription.with_start(newest); } + // A wire without frame bounds asks the group whole. + if !self.subscriber.version.has_frame_bounds() { + subscription.resume = None; + } wire_ranges(&subscription).map(|ranges| (subscription, ranges)) } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 94d9a1428e7..a499ad0699b 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -210,7 +210,7 @@ impl Producer { _ => Vec::new(), }; ranges.extend(preferences.clamped(reader.newest, None, &resolved).ranges); - resumes.push(reader.resume_at(reader.newest).map(|(resume, _)| resume)); + resumes.push(reader.held_at(reader.newest).map(|(resume, _)| resume)); } demand.ranges = ranges; // As in a track's own demand, only when every reader resumes. @@ -610,9 +610,6 @@ struct Copy { /// The frames of the group at `floor` the reader already held when this copy was asked, /// so it asks for that group from the next frame; see [`Reader::resume_at`]. resume: Option, - /// Those frames, lent to this copy once the rest of their group lands here, so it holds - /// the group whole for a reader that joins later; see [`Reader::lend_heads`]. - head: Option>, /// The reader's ranges [`Self::resolved`] are for. from: Vec, /// Each of those ranges resolved, by index, so clamping never widens one: as the @@ -1476,7 +1473,7 @@ impl Reader { // Resume where a fresh copy would: a group it buffered while replaced, below the // newest another route handed out, is not handed out. copy.floor = copy.floor.max(self.newest); - (copy.resume, copy.head) = self.resume_at(copy.floor).unzip(); + copy.resume = self.resume_at(copy.floor, ©.track); let start = copy.start(self.groups.0); if let (Sub::Ready(sub), Some(_)) = (&mut copy.sub, copy.floor) { sub.set_range(( @@ -1492,7 +1489,7 @@ impl Reader { return; } let floor = self.newest; - let (resume, head) = self.resume_at(floor).unzip(); + let resume = self.resume_at(floor, &track); // The wire request carries the floor from the first message: some sessions read // the demand only once. let mut wire = self.mirrored.clamped(floor, None, &resolved); @@ -1503,7 +1500,6 @@ impl Reader { track, floor, resume, - head, from: self.mirrored.ranges.clone(), resolved, replaced: false, @@ -1516,12 +1512,21 @@ impl Reader { self.learn_end(None); } + /// Ask `track` to resume group `key` past the frames held of it (see [`Self::held_at`]), + /// lending it those frames, so once the rest arrives it holds the group whole for a + /// reader that joins later, after the copies holding them are gone. + fn resume_at(&self, key: Option, track: &track::Consumer) -> Option { + let (resume, head) = self.held_at(key)?; + track.lend_head(resume.group, head); + Some(resume) + } + /// Where a new copy asked from group `key` resumes it: past the most frames of it a copy /// holds from its first, which the reader already has or reads locally, and those frames. /// A route switch behind the same next hop then sends none of them again, only the rest of /// a group still arriving, and nothing of a finished one. `None` while no copy holds its /// first frame. - fn resume_at(&self, key: Option) -> Option<(Position, Arc)> { + fn held_at(&self, key: Option) -> Option<(Position, Arc)> { let key = key?; (self.copies.iter()) .filter_map(|copy| copy.track.peek_group(key)?.handoff().head()) @@ -1530,24 +1535,6 @@ impl Reader { .map(|(frames, head)| (Position::new(key, frames), head)) } - /// Lend each copy asked to resume a group the frames it was asked past, once the rest of - /// that group lands there, so it holds the group whole for a reader that joins later - /// after the copies holding them are gone. - fn lend_heads(&mut self) { - for copy in &mut self.copies { - let (Some(resume), Some(head)) = (copy.resume, ©.head) else { - continue; - }; - let Some(group) = copy.track.peek_group(resume.group) else { - continue; - }; - if group.first_frame() > 0 { - copy.track.adopt_head(resume.group, head); - } - copy.head = None; - } - } - /// One logical track has one end, whichever route first reports it. fn learn_end(&mut self, waiter: Option<&kio::Waiter>) { for copy in &mut self.copies { @@ -1760,7 +1747,6 @@ impl Reader { waiter: &kio::Waiter, ) -> Option>>> { self.sync(waiter); - self.lend_heads(); if let Some(err) = self.refusal() { return Some(Poll::Ready(Err(err))); } @@ -2266,7 +2252,6 @@ impl Reader { floor, // Below every group handed out, so the reader holds none of the first. resume: None, - head: None, from: self.mirrored.ranges.clone(), resolved, // Bounded from the start: everything it may deliver is older than what the reader @@ -4133,10 +4118,12 @@ mod test { let b = copy(); routes.serve(b.consume()); assert_eq!(b.subscription().unwrap().resume, Some(Position::new(0, 1))); - let mut rest = b.create_group(group::Info { start: 0 }).unwrap(); - rest.start_at(1).unwrap(); + // As a session receives it, partway through. The holder never polls again, so the + // copy takes the head on arrival, not from the holder's reads. + let track::Received::Group(rest) = b.create_group_at(group::Info { start: 0 }, 1).unwrap() else { + panic!("nothing else holds the group"); + }; rest.finish().unwrap(); - assert!(holder.recv_group().now_or_never().is_none(), "handed out twice"); drop(a); let mut late = subscribe(&track, HOUR); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 550c1b2c8de..31b7ae1a3e8 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -702,6 +702,12 @@ pub(crate) struct TrackState { // the read guard a first frame holds, so it needs its own lock, never contended. bounds: Mutex>, + // The frames of a group a reader already held when it asked this copy to resume that + // group past them, by key, which the group adopts when it arrives (see + // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One + // entry: a later loan replaces it, so nothing grows with switches. + lent: Mutex)>>, + // The key and start of the highest-key group that left the cache holding a // checked start (see [`group::Origin::Bound`]). Eviction is not a key prefix, so a // group above it is checked against it as well as its cached neighbors; one at or below @@ -1206,6 +1212,12 @@ impl Holdings for TrackState { } impl TrackState { + /// The head lent for group `key`, taken; see [`Consumer::lend_head`]. + fn take_lent(&self, key: u64) -> Option> { + let mut lent = self.lent.lock().unwrap_or_else(PoisonError::into_inner); + lent.take_if(|(lent, _)| *lent == key).map(|(_, head)| head) + } + /// How many keys just below `key` the publisher declared it never produced: the skip /// run that ends at `key`, as a moq-transport Prior Group ID Gap names it. fn skipped_below(&self, key: u64) -> u64 { @@ -4516,9 +4528,17 @@ impl Producer { } state.cap(&group.handle())?; state.commit_group(&group.handle()); + let lent = match start { + 0 => None, + _ => state.take_lent(info.start), + }; let deferred = state.deferred(); drop(state); deferred.wake(); + // With the track lock released (lock order is track then group). + if let Some(head) = lent { + group.handle().adopt_head(&head); + } Ok(Received::Group(group)) } @@ -6592,6 +6612,29 @@ impl Consumer { }) } + /// Lend `head` to group `key` of this copy: to the copies it caches now, or else to the + /// first that arrives partway through the group, which then holds it whole. A later loan + /// replaces one still waiting; see [`group::Handle::adopt_head`]. + pub(super) fn lend_head(&self, key: u64, head: Arc) { + let copies: Vec = self.with_state(|state| { + let Some(state) = state else { + return Vec::new(); + }; + let state = state.read(); + match state.lookup.get(&key) { + Some(slot) => slot.copies().cloned().collect(), + // Under the read guard, so the group cannot arrive before the loan is held. + None => { + *state.lent.lock().unwrap_or_else(PoisonError::into_inner) = Some((key, head.clone())); + Vec::new() + } + } + }); + for group in copies { + group.adopt_head(&head); + } + } + /// Lend `head` to the cached copies of group `key`; see /// [`group::Handle::adopt_head`]. pub(super) fn adopt_head(&self, key: u64, head: &group::Head) { From 8287195e7644ce481c88251e2830c79737a256b5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 04:18:45 -0800 Subject: [PATCH 07/21] fix(net): a lent head per group, adopted before the group is visible; JS serves every Position - A copy keeps a loan for each group its readers resumed it at, the longer one standing per group, so readers resuming at different groups all keep their heads. - The arriving group takes the head before it is committed, so no reader or publisher sees it without its head. Group::take_head and settle_head split adoption so the track-facing half runs after the track lock is released. - The JS publisher reads a frame start for every group a single range starts partway through, as Rust does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/publisher.ts | 8 +++- js/net/src/lite/subscribe.test.ts | 10 ++++- js/net/src/lite/subscribe.ts | 39 ++++++++++------- rs/moq-net/src/model/group.rs | 40 +++++++++++++---- rs/moq-net/src/model/track.rs | 73 +++++++++++++++++++++++-------- 5 files changed, 127 insertions(+), 43 deletions(-) diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 28c4e2414a9..76d3575fde1 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -62,6 +62,7 @@ import { Probe } from "./probe.ts"; import { encodeSubscribeResponse, LiveRanges, + positionFrames, rangesOf, requestedRanges, type Subscribe, @@ -273,6 +274,8 @@ type FrameBounds = { endGroup?: bigint; /** Last frame (inclusive) to send within {@link endGroup}; every other group runs to its end. */ endFrame?: number; + /** Lite-07: the first frame of each group a range starts partway through (see {@link positionFrames}). */ + starts: ReadonlyMap; }; /** @@ -288,7 +291,7 @@ type FrameBounds = { */ function frameRange(bounds: FrameBounds, key: bigint): { start: number; end?: number } { return { - start: bounds.startGroup === key ? bounds.startFrame : 0, + start: bounds.starts.get(key) ?? (bounds.startGroup === key ? bounds.startFrame : 0), end: bounds.endGroup === key ? bounds.endFrame : undefined, }; } @@ -994,6 +997,7 @@ export class Publisher { startFrame: msg.startFrame, endGroup: msg.endGroup, endFrame: msg.endFrame, + starts: hasRanges(this.version) ? positionFrames(msg.ranges ?? []) : new Map(), }; let legacy: datagram.Consumer | undefined; const legacyPriority = new Signal(msg.priority); @@ -1647,6 +1651,8 @@ export class Publisher { bounds.startFrame = update.startFrame; bounds.endGroup = update.endGroup; bounds.endFrame = update.endFrame; + // The update carries every live range by now (see `apply`). + bounds.starts = hasRanges(this.version) ? positionFrames(update.ranges ?? []) : new Map(); moveCursor(); // A run elided for a group the ranges no longer name goes out after all. if (!(await sendElided())) return false; diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 20724828900..23abc93cb4d 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -8,6 +8,7 @@ import { encodeSubscribeResponse, inclusiveGroupEnd, LiveRanges, + positionFrames, rangeSize, requestedRanges, Subscribe, @@ -275,8 +276,15 @@ test("a lite-07 range may start partway through a group, which the span keeps", expect(spanOf(Version.DRAFT_07, [position, other]).startFrame).toBe(0); } expect(spanOf(Version.DRAFT_07, [position, { start: { time: 9n } }]).startFrame).toBe(4); - // The span qualifies only its lowest start, so a Position above another range's goes whole. + // The span qualifies only its lowest start; the publisher reads every Position from the ranges. expect(spanOf(Version.DRAFT_07, [position, { start: { time: 1n }, end: { time: 3n } }]).startFrame).toBe(0); + const disjoint = [{ start: { absolute: 7n, frame: 4 }, end: { time: 8n } }, { start: { absolute: 9n, frame: 2 } }]; + expect([...positionFrames(disjoint)]).toEqual([ + [7n, 4], + [9n, 2], + ]); + const overlapping = [{ start: { absolute: 7n, frame: 4 } }, { start: { absolute: 9n, frame: 2 } }]; + expect([...positionFrames(overlapping)]).toEqual([[7n, 4]]); // An end there can't go out. await expect( diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 37a8292e62f..e793cc5556d 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -290,9 +290,9 @@ async function decodeRanges(r: Reader): Promise { /** * Lite-07 `ranges` and the one span covering every one of them, which the subscriber narrows - * again. A range covers whole groups but for a Position start, whose frame the span keeps when - * the span starts there and no other range may cover that group: each group is delivered once. - * A Position above the span's start goes whole, as more than it asked. + * again. A range covers whole groups but for a Position start (see {@link positionFrames}), + * whose frame the span keeps when the span starts there. The publisher reads every Position + * from the ranges themselves. */ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { ranges: WireRange[] } { let start: bigint | undefined; @@ -312,7 +312,7 @@ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { if (live) end = undefined; // An exclusive end at the very first group names nothing. if (end === 0n) throw new ProtocolViolation(EMPTY_RANGE); - const startFrame = start === undefined ? 0 : positionFrame(ranges, start); + const startFrame = start === undefined ? 0 : (positionFrames(ranges).get(start) ?? 0); return { ranges: [...ranges], startGroup: canonicalStartGroup(version, start, startFrame), @@ -321,19 +321,28 @@ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { }; } -/** The frame a Position start asks group `group` from, 0 unless no other range may cover it. */ -function positionFrame(ranges: readonly WireRange[], group: bigint): number { - let frame = 0; - let covering = 0; +/** + * The first frame each lite-07 Position start asks its group from, by group, for each group no + * other range may cover: a group is delivered once, so one another range covers goes whole. + */ +export function positionFrames(ranges: readonly WireRange[]): Map { + const frames = new Map(); for (const range of ranges) { - const from = range.start?.absolute ?? range.start?.time; - const to = range.end?.absolute ?? (range.end?.time === undefined ? undefined : range.end.time + 1n); - // Unknown ends, such as a count back, may cover it. - if ((from !== undefined && from > group) || (to !== undefined && to <= group)) continue; - covering++; - if (range.start?.absolute === group && range.start.frame) frame = range.start.frame; + const start = range.start; + if (start?.absolute === undefined || !start.frame) continue; + const group = start.absolute; + const frame = start.frame; + let covering = 0; + for (const other of ranges) { + const from = other.start?.absolute ?? other.start?.time; + const to = other.end?.absolute ?? (other.end?.time === undefined ? undefined : other.end.time + 1n); + // Unknown ends, such as a count back, may cover it. + if ((from !== undefined && from > group) || (to !== undefined && to <= group)) continue; + covering++; + } + if (covering === 1) frames.set(group, frame); } - return covering === 1 ? frame : 0; + return frames; } /** diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 37306684579..f9369e41a05 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -346,6 +346,14 @@ impl Fall { } } +/// A head [`Handle::take_head`] adopted, still to settle with the track and the cache; see +/// [`Handle::settle_head`]. +#[must_use] +pub(super) struct Adopted { + now: Option, + latest: Option, +} + /// The frames a copy of a group holds, from where it starts, which a successor copy of /// the same group on another route adopts below where it starts. pub(super) struct Head { @@ -2720,8 +2728,17 @@ impl Handle { /// compressed group adopts only a compressed head, taking its inflated sizes beside the /// slices, so its plain readers inflate from the adopted frames. pub(super) fn adopt_head(&self, head: &Head) { + if let Some(adopted) = self.take_head(head) { + self.settle_head(adopted); + } + } + + /// The part of [`Self::adopt_head`] that touches only this group, so a track can run it + /// under its own lock before any reader sees the group: then [`Self::settle_head`], with + /// that lock released. + pub(super) fn take_head(&self, head: &Head) -> Option { let Ok(mut state) = self.state.write() else { - return; + return None; }; let offset = state.offset; let reaches = head.start.saturating_add(head.entries.len()) >= offset; @@ -2729,7 +2746,7 @@ impl Handle { // A copy that shed its front holds no run from `offset` for the head to precede. let shed = state.held() != offset; if state.outcome.error().is_some() || state.sparse() || !same_form || shed || head.start >= offset || !reaches { - return; + return None; } let count = offset - head.start; let entries = &head.entries[..count]; @@ -2743,13 +2760,13 @@ impl Handle { // shed to make room for. let slots = state.slots() + (count * size_of::()) as u64; if state.tail().saturating_add(payload + metadata + slots) > MAX_CACHE_BYTES { - return; + return None; } let mut adopted = Vec::with_capacity(count); for (index, entry) in (head.start..).zip(entries) { let mut entry = entry.clone(); if state.canonicalize(index, &mut entry).is_err() { - return; + return None; } // Below the log, so off this copy's pace scale: taking it reports nothing. entry.end = 0; @@ -2792,11 +2809,18 @@ impl Handle { wire.last = wire.last.or(Some(offset as u64 - 1)); } let extra = metadata + growth + if state.content.is_none() { payload } else { 0 }; - let now = state.charge.add(extra); - drop(state); + Some(Adopted { + now: state.charge.add(extra), + latest: head_latest, + }) + } + + /// Settle a head [`Self::take_head`] adopted with the track and the cache: the group + /// counts as content now, as a first frame would make it. + pub(super) fn settle_head(&self, adopted: Adopted) { self.start(None); - self.cache.settle(now); - if let Some(latest) = head_latest { + self.cache.settle(adopted.now); + if let Some(latest) = adopted.latest { self.cache.wakes().presented(latest); } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 31b7ae1a3e8..9e94e341c37 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -702,11 +702,12 @@ pub(crate) struct TrackState { // the read guard a first frame holds, so it needs its own lock, never contended. bounds: Mutex>, - // The frames of a group a reader already held when it asked this copy to resume that - // group past them, by key, which the group adopts when it arrives (see - // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One - // entry: a later loan replaces it, so nothing grows with switches. - lent: Mutex)>>, + // The frames of each group a reader already held when it asked this copy to resume that + // group past them, by key, which the group adopts as it arrives (see + // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One per + // group, taken as that group arrives, so it holds at most the groups readers resumed + // this copy at. + lent: Mutex>>, // The key and start of the highest-key group that left the cache holding a // checked start (see [`group::Origin::Bound`]). Eviction is not a key prefix, so a @@ -1214,8 +1215,7 @@ impl Holdings for TrackState { impl TrackState { /// The head lent for group `key`, taken; see [`Consumer::lend_head`]. fn take_lent(&self, key: u64) -> Option> { - let mut lent = self.lent.lock().unwrap_or_else(PoisonError::into_inner); - lent.take_if(|(lent, _)| *lent == key).map(|(_, head)| head) + self.lent.lock().unwrap_or_else(PoisonError::into_inner).remove(&key) } /// How many keys just below `key` the publisher declared it never produced: the skip @@ -4526,19 +4526,21 @@ impl Producer { if let Some(content) = content { group.handle().share_content(content); } + // Before any reader can see it, so none takes it without the head (lock order is track + // then group), settled once the track lock is released. A whole copy needs no loan, + // and drops it. + let adopted = match state.take_lent(info.start) { + Some(head) if start != 0 => group.handle().take_head(&head), + _ => None, + }; state.cap(&group.handle())?; state.commit_group(&group.handle()); - let lent = match start { - 0 => None, - _ => state.take_lent(info.start), - }; let deferred = state.deferred(); drop(state); - deferred.wake(); - // With the track lock released (lock order is track then group). - if let Some(head) = lent { - group.handle().adopt_head(&head); + if let Some(adopted) = adopted { + group.handle().settle_head(adopted); } + deferred.wake(); Ok(Received::Group(group)) } @@ -6613,8 +6615,8 @@ impl Consumer { } /// Lend `head` to group `key` of this copy: to the copies it caches now, or else to the - /// first that arrives partway through the group, which then holds it whole. A later loan - /// replaces one still waiting; see [`group::Handle::adopt_head`]. + /// first that arrives, which then holds it whole if it arrives partway through. Of two + /// loans for one group the longer stands; see [`group::Handle::adopt_head`]. pub(super) fn lend_head(&self, key: u64, head: Arc) { let copies: Vec = self.with_state(|state| { let Some(state) = state else { @@ -6625,7 +6627,12 @@ impl Consumer { Some(slot) => slot.copies().cloned().collect(), // Under the read guard, so the group cannot arrive before the loan is held. None => { - *state.lent.lock().unwrap_or_else(PoisonError::into_inner) = Some((key, head.clone())); + let mut lent = state.lent.lock().unwrap_or_else(PoisonError::into_inner); + let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); + let slot = lent.entry(key).or_insert_with(|| head.clone()); + if held(&head) > held(slot) { + *slot = head.clone(); + } Vec::new() } } @@ -15811,6 +15818,36 @@ mod test { edge.finish().unwrap(); } + /// A copy lent the heads of two groups its readers resumed it at holds each whole as it + /// arrives partway through, from the moment a reader can see it. + #[test] + #[cfg(feature = "async")] + fn lent_heads_complete_each_group_as_it_arrives() { + let producer = track_producer("test", Info::timed()); + let consumer = producer.consume(); + let head = |start: u64, frames: u64| { + let mut prefix = group::Info { start }.produce(); + for _ in 0..frames { + prefix + .write_frame(Timestamp::from_millis(start).unwrap(), bytes::Bytes::new()) + .unwrap(); + } + prefix.consume().handoff().head().unwrap() + }; + consumer.lend_head(7, head(7, 2)); + consumer.lend_head(9, head(9, 1)); + // A shorter loan for the same group leaves the longer one standing. + consumer.lend_head(7, head(7, 1)); + for (key, at) in [(7, 2), (9, 1)] { + let Received::Group(group) = producer.create_group_at(group::Info { start: key }, at).unwrap() else { + panic!("nothing else holds group {key}"); + }; + let seen = consumer.peek_group(key).expect("cached"); + assert_eq!(seen.first_frame(), 0, "group {key} is whole once visible"); + drop(group); + } + } + #[test] #[cfg(feature = "async")] fn an_adopted_high_pts_wakes_the_group_drift_deadline() { From 1a858ac49c322a97c5a4b05e6d8d319fc7c957f4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 04:45:28 -0800 Subject: [PATCH 08/21] fix(net): drop a head loan once its group is given up A loan lives until its group arrives or the upstream drops or skips it, so change_drops clears the loans for any run it records. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 9e94e341c37..34ffb7846d1 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -705,8 +705,8 @@ pub(crate) struct TrackState { // The frames of each group a reader already held when it asked this copy to resume that // group past them, by key, which the group adopts as it arrives (see // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One per - // group, taken as that group arrives, so it holds at most the groups readers resumed - // this copy at. + // group, taken as that group arrives or dropped as it is given up, so it holds only the + // groups readers resumed this copy at that are still to come. lent: Mutex>>, // The key and start of the highest-key group that left the cache holding a @@ -1581,6 +1581,11 @@ impl TrackState { /// Keep the change index aligned with the range map, including the neighbours an /// insertion coalesces and the two pieces a removal can leave. fn change_drops(&mut self, groups: ops::Range, hole: Option) { + // A group given up never arrives to take the head lent for it. + if hole.is_some() { + let lent = self.lent.get_mut().unwrap_or_else(PoisonError::into_inner); + lent.retain(|key, _| !groups.contains(key)); + } if hole.as_ref().is_none_or(|hole| hole.stale_budget().is_none()) && self .drops @@ -15838,6 +15843,10 @@ mod test { consumer.lend_head(9, head(9, 1)); // A shorter loan for the same group leaves the longer one standing. consumer.lend_head(7, head(7, 1)); + // A group given up drops its loan. + consumer.lend_head(8, head(8, 1)); + producer.skip_groups(8..9); + assert!(!producer.modify().unwrap().lent.get_mut().unwrap().contains_key(&8)); for (key, at) in [(7, 2), (9, 1)] { let Received::Group(group) = producer.create_group_at(group::Info { start: key }, at).unwrap() else { panic!("nothing else holds group {key}"); From 5e89cec41d7c593ea7f98399c2928408675865d6 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:13:42 -0800 Subject: [PATCH 09/21] fix(net): a copy that tears down drops every head lent to it Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 34ffb7846d1..64522ef7334 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -705,8 +705,8 @@ pub(crate) struct TrackState { // The frames of each group a reader already held when it asked this copy to resume that // group past them, by key, which the group adopts as it arrives (see // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One per - // group, taken as that group arrives or dropped as it is given up, so it holds only the - // groups readers resumed this copy at that are still to come. + // group, taken as that group arrives or dropped as it is given up or the track tears down, + // so it holds only the groups readers resumed this copy at that are still to come. lent: Mutex>>, // The key and start of the highest-key group that left the cache holding a @@ -2711,6 +2711,8 @@ impl TrackState { /// reader that already pulled one ends with `err` too and its writer fails. fn tear_down(&mut self, err: Error) -> (Option, OpenGroups) { self.release_ingress(); + // Nothing arrives after this to take a lent head. + self.lent.get_mut().unwrap_or_else(PoisonError::into_inner).clear(); let resume = self.resume_position(); let groups = self .lookup @@ -15847,6 +15849,12 @@ mod test { consumer.lend_head(8, head(8, 1)); producer.skip_groups(8..9); assert!(!producer.modify().unwrap().lent.get_mut().unwrap().contains_key(&8)); + // So does every one still waiting once the copy tears down. + let aborted = track_producer("aborted", Info::timed()); + let reader = aborted.consume(); + reader.lend_head(3, head(3, 1)); + aborted.abort(Error::Cancel).unwrap(); + assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().is_empty())); for (key, at) in [(7, 2), (9, 1)] { let Received::Group(group) = producer.create_group_at(group::Info { start: key }, at).unwrap() else { panic!("nothing else holds group {key}"); From 0aaab3e77f6b479bda32c7abbf532975f4667ab4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:45:46 -0800 Subject: [PATCH 10/21] quest: the Rust audio subscriber closes within its drain Co-Authored-By: Claude Opus 5.5 --- .../test-flakes-3/rust-audio-close-drain.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 quest/m0/test-flakes-3/rust-audio-close-drain.md diff --git a/quest/m0/test-flakes-3/rust-audio-close-drain.md b/quest/m0/test-flakes-3/rust-audio-close-drain.md new file mode 100644 index 00000000000..0d5a3434dbe --- /dev/null +++ b/quest/m0/test-flakes-3/rust-audio-close-drain.md @@ -0,0 +1,22 @@ +# [S] The Rust audio subscriber closes within its drain + +## Goal + +`just test interop --all` passes its `* -> rust` cells under load: the +`interop-audio` client (`test/interop/clients/rust/audio.rs`) closes its +session without `Session::close` returning `Error::Timeout`. + +## Plan + +Seen once on 2026-11-19 while validating +[moq-dev/moq#9155](https://github.com/moq-dev/moq/pull/9155): the +`python -> rust` cell decoded the whole tone (`decoded 28848 samples of tone +(opus) with libopus`), then failed with `Error: timeout` after 7 s of its 30 s +budget. The client had connected over the WebSocket fallback, a sign the host +was loaded. Three reruns of the cell alone passed, and main's scheduled +Interop runs show no such cell. So the subscriber's close drain (one second, +`rs/moq-net/src/session.rs` `poll_drain`) found something still owed to the +relay under load. Find what a subscribing session still waits on at close, +and fix that cause rather than the deadline or the client's `?`. + +Public API: none. Wire: none. From 84b5a69526ede1aba8825ee1297365d5f36f80fc Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:27:07 -0800 Subject: [PATCH 11/21] fix(net): a query resuming a new copy lends it the head, and the draft says which ranges cover a group Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 3 +- rs/moq-net/src/model/origin.rs | 6 +- rs/moq-net/src/model/resume.rs | 107 ++++++++++++++++++++++++++----- rs/moq-net/src/model/track.rs | 44 +++++++------ 4 files changed, 121 insertions(+), 39 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index b8c3f5a718c..d8304bf36f8 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -1548,7 +1548,8 @@ The live edge is the newest frame timestamp the publisher holds; on a Track with The publisher starts from what it holds and never fetches to reach further back, so a subscriber is handed the buffer it will play, and no more, without waiting for it to fill. A Range covers whole Groups but for a Position `Start`; a subscriber reads a slice of a Group with FETCH (see [Positions](#positions)). -A Group that another Range also covers is delivered whole, since each Group is delivered once. +A Group that another Range may also cover is delivered whole, since each Group is delivered once: one whose `Start` is at or below the Group's start, or is None or Relative, and whose `End` is not a Time below the Group's start. +A Time `Start` inside the Group, after its start, does not count: whether it falls before the next Group's start is unknown until that Group arrives. A subscriber resuming a Track on a new subscription, such as a relay moving to another route, starts at the Position past the frames it holds, so no frame it holds is sent again. The Group Stream of a Group that ended at `Frame` carries no FRAME, only how the Group ended. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index d092c4133cf..7d47ef4f4bc 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4467,10 +4467,10 @@ fn follow_demand( } match (weak.subscription(), held.as_mut()) { (Some((subscription, subscribers)), Some(hold)) => { - let _ = hold.update(routes.subscription(subscription, subscribers)); + let _ = routes.subscription(subscription, subscribers).update(hold); } (Some((subscription, subscribers)), None) => { - *held = copy.map(|copy| copy.subscribe(routes.subscription(subscription, subscribers))); + *held = copy.map(|copy| routes.subscription(subscription, subscribers).subscribe(copy)); } (None, _) => *held = None, } @@ -5171,7 +5171,7 @@ impl kio::Task for FrontServe { // Subscribe before the query exposes upstream demand, and query // that same instance even if the broadcast changes meanwhile. let sub = io.weak.subscription().map(|(demand, subscribers)| { - lookup.subscribe(io.routes.subscription(demand, subscribers)) + io.routes.subscription(demand, subscribers).subscribe(&lookup) }); let query = match &sub { Some(sub) => sub.info(), diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 84e996dff83..28bda7093ff 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -146,6 +146,35 @@ impl Phase { /// one that is not reading right now still gets what that copy delivers. type Readers = Arc>>>>; +/// The readers' demand on a new copy ([`Producer::subscription`]) and the frames they hold +/// of the group it resumes past, which that copy is lent so it caches the group whole. +#[must_use] +pub(super) struct Resumed { + demand: Subscription, + head: Option>, +} + +impl Resumed { + /// Subscribe `copy` to the demand, lending it the head the demand resumes past. + pub(super) fn subscribe(self, copy: &track::Consumer) -> kio::Pending { + let resume = self.demand.resume; + let sub = copy.subscribe(self.demand); + // Once subscribed, so the copy's state exists to hold the loan. + if let (Some(head), Some(resume)) = (self.head, resume) { + sub.lend_head(resume.group, head); + } + sub + } + + /// Move `sub` to the demand, lending its copy the head the demand resumes past. + pub(super) fn update(self, sub: &mut track::Subscribing) -> Result<()> { + if let (Some(head), Some(resume)) = (self.head, self.demand.resume) { + sub.lend_head(resume.group, head); + } + sub.update(self.demand) + } +} + /// The front's side of a logical track: which copy serves it, and how it ends. /// /// Dropping it concludes the track: readers follow the last copy to its end, since no @@ -166,7 +195,7 @@ impl Producer { } /// The readers' demand on a new copy, with each reader resuming where it got to. - pub(super) fn subscription(&self, mut demand: Subscription, subscribers: usize) -> Subscription { + pub(super) fn subscription(&self, mut demand: Subscription, subscribers: usize) -> Resumed { let readers: Vec<_> = self .readers .lock() @@ -177,7 +206,7 @@ impl Producer { // A subscription not yet polled has demand but no reader. It still needs the // original ranges, so never narrow the aggregate until every contribution is here. if readers.len() != subscribers || readers.is_empty() { - return demand; + return Resumed { demand, head: None }; } let mut ranges = Vec::new(); let mut resumes = Vec::with_capacity(readers.len()); @@ -210,14 +239,18 @@ impl Producer { _ => Vec::new(), }; ranges.extend(preferences.clamped(reader.newest, None, &resolved).ranges); - resumes.push(reader.held_at(reader.newest).map(|(resume, _)| resume)); + resumes.push(reader.held_at(reader.newest)); } demand.ranges = ranges; // As in a track's own demand, only when every reader resumes. - demand.resume = (resumes.into_iter()) - .reduce(|a, b| a.zip(b).map(|(a, b)| a.min(b))) + let resume = (resumes.into_iter()) + .reduce(|a, b| a.zip(b).map(|(a, b)| if b.0 < a.0 { b } else { a })) .flatten(); - demand + demand.resume = resume.as_ref().map(|(resume, _)| *resume); + Resumed { + demand, + head: resume.map(|(_, head)| head), + } } /// Stage `copy`, whose info is still in flight, for fetches until the returned @@ -4117,9 +4150,9 @@ mod test { write(&a, 5); assert_eq!(recv(&mut first).start, 5); let demand = front.subscription().unwrap(); - assert_eq!(routes.subscription(demand.clone(), 2).resume, None); + assert_eq!(routes.subscription(demand.clone(), 2).demand.resume, None); assert_eq!(recv(&mut second).start, 5); - assert_eq!(routes.subscription(demand, 2).resume, Some(Position::new(5, 1))); + assert_eq!(routes.subscription(demand, 2).demand.resume, Some(Position::new(5, 1))); } /// A copy asked past the frames a reader held of a group adopts them once the rest lands, @@ -4153,6 +4186,45 @@ mod test { assert_eq!(read(&mut group), None); } + /// The subscription an origin opens on a new copy before serving it (its query, then the + /// one it holds for the readers) lends the head it resumes past, as a reader's own does. + /// Otherwise a reader moving on before the switch lends the next group's head instead, + /// and the copy caches the group the query resumed without its head. + #[test] + #[cfg(feature = "async")] + fn a_query_resuming_a_new_copy_lends_it_the_head() { + let routes = Producer::new(); + let track = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut holder = subscribe(&track, HOUR); + write(&a, 0); + assert_eq!(recv(&mut holder).start, 0); + + let b = copy(); + let mut held = routes + .subscription(track.subscription().unwrap(), 1) + .subscribe(&b.consume()); + assert_eq!(b.subscription().unwrap().resume, Some(Position::new(0, 1))); + write(&a, 1); + assert_eq!(recv(&mut holder).start, 1); + // As the held subscription follows the readers, past the next group too. + routes + .subscription(track.subscription().unwrap(), 1) + .update(&mut held) + .unwrap(); + routes.serve(b.consume()); + + for key in [0, 1] { + let track::Received::Group(rest) = b.create_group_at(group::Info { start: key }, 1).unwrap() else { + panic!("nothing else holds group {key}"); + }; + rest.finish().unwrap(); + let cached = b.consume().peek_group(key).expect("cached"); + assert_eq!(cached.first_frame(), 0, "group {key} is whole once visible"); + } + } + /// Routes that close without concluding read like a conclusion: a reader stalled /// mid-group parks on the copy it is reading rather than polling the closed routes /// forever, and finishes the group from that copy. @@ -8220,7 +8292,7 @@ mod test { for sequence in 11..=20 { write(&a, sequence); } - let start = || routes.subscription(Subscription::default(), 1).ranges[0].start; + let start = || routes.subscription(Subscription::default(), 1).demand.ranges[0].start; assert_eq!(start(), track::Anchor::Time(5)); // Changed, but not yet mirrored into the copies. @@ -8250,11 +8322,11 @@ mod test { assert_eq!(recv(sub).start, 5); } let demand = front.subscription().unwrap(); - assert_eq!(routes.subscription(demand.clone(), 5).start(), Some(5)); + assert_eq!(routes.subscription(demand.clone(), 5).demand.start(), Some(5)); for _ in 0..64 { drop(subscribe(&front, HOUR)); } - assert_eq!(routes.subscription(demand, 5).start(), Some(5)); + assert_eq!(routes.subscription(demand, 5).demand.start(), Some(5)); // Swept each time it fills, so it grows by doubling over the most readers alive at // once: here 6. let len = routes.readers.lock().unwrap().len(); @@ -8273,19 +8345,22 @@ mod test { assert_eq!(recv(&mut first).start, 5); let demand = front.subscription().unwrap(); // One reader has not read anything yet, so it still needs the oldest range. - assert_eq!(routes.subscription(demand.clone(), 2).start(), None); + assert_eq!(routes.subscription(demand.clone(), 2).demand.start(), None); assert_eq!(recv(&mut second).start, 5); - assert_eq!(routes.subscription(demand.clone(), 2).start(), Some(5)); + assert_eq!(routes.subscription(demand.clone(), 2).demand.start(), Some(5)); write(&a, 10); assert_eq!(recv(&mut first).start, 10); - assert_eq!(routes.subscription(demand.clone(), 2).start(), Some(5)); + assert_eq!(routes.subscription(demand.clone(), 2).demand.start(), Some(5)); let pending = front .consume() .subscribe(Subscription::default().with_delay(Delay::default().with_max(HOUR))); - assert_eq!(routes.subscription(front.subscription().unwrap(), 3).start(), None); + assert_eq!( + routes.subscription(front.subscription().unwrap(), 3).demand.start(), + None + ); drop(pending); drop(second); - assert_eq!(routes.subscription(demand, 1).start(), Some(10)); + assert_eq!(routes.subscription(demand, 1).demand.start(), Some(10)); } #[derive(Default)] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ed512be5e22..c2e28e70316 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1219,6 +1219,22 @@ impl TrackState { self.lent.lock().unwrap_or_else(PoisonError::into_inner).remove(&key) } + /// Hold `head` for group `key` until it arrives, or return the copies of it already + /// cached, to adopt it with this state's lock released; see [`Consumer::lend_head`]. + fn lend(&self, key: u64, head: &Arc) -> Vec { + if let Some(slot) = self.lookup.get(&key) { + return slot.copies().cloned().collect(); + } + // Under the caller's read guard, so the group cannot arrive before the loan is held. + let mut lent = self.lent.lock().unwrap_or_else(PoisonError::into_inner); + let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); + let slot = lent.entry(key).or_insert_with(|| head.clone()); + if held(head) > held(slot) { + *slot = head.clone(); + } + Vec::new() + } + /// How many keys just below `key` the publisher declared it never produced: the skip /// run that ends at `key`, as a moq-transport Prior Group ID Gap names it. fn skipped_below(&self, key: u64) -> u64 { @@ -6580,25 +6596,7 @@ impl Consumer { /// first that arrives, which then holds it whole if it arrives partway through. Of two /// loans for one group the longer stands; see [`group::Handle::adopt_head`]. pub(super) fn lend_head(&self, key: u64, head: Arc) { - let copies: Vec = self.with_state(|state| { - let Some(state) = state else { - return Vec::new(); - }; - let state = state.read(); - match state.lookup.get(&key) { - Some(slot) => slot.copies().cloned().collect(), - // Under the read guard, so the group cannot arrive before the loan is held. - None => { - let mut lent = state.lent.lock().unwrap_or_else(PoisonError::into_inner); - let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); - let slot = lent.entry(key).or_insert_with(|| head.clone()); - if held(&head) > held(slot) { - *slot = head.clone(); - } - Vec::new() - } - } - }); + let copies = self.with_state(|state| state.map_or_else(Vec::new, |state| state.read().lend(key, &head))); for group in copies { group.adopt_head(&head); } @@ -7127,6 +7125,14 @@ pub struct Subscribing { } impl Subscribing { + /// Lend `head` to group `key` of the subscribed copy; see [`Consumer::lend_head`]. + pub(super) fn lend_head(&self, key: u64, head: Arc) { + let copies = self.state.read().lend(key, &head); + for group in copies { + group.adopt_head(&head); + } + } + /// Resolve the same track's info with this subscription already registered. pub(super) fn info(&self) -> Resolving { Resolving { From dd342920656d7781c839019157d2adf21e8b8e08 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:45:54 -0800 Subject: [PATCH 12/21] fix(net): bound lent heads, sweep Position coverage, and model the resume loan Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-failover/SKILL.md | 9 +- js/net/src/lite/subscribe.test.ts | 34 +++++++ js/net/src/lite/subscribe.ts | 53 ++++++++--- quest/m4/js-route-resume.md | 14 +++ rs/moq-net/src/lite/publisher.rs | 84 ++++++++++++++--- rs/moq-net/src/model/group.rs | 2 +- rs/moq-net/src/model/track.rs | 94 +++++++++++++++++--- rs/moq-net/verify/failover/FailoverChain.tla | 41 +++++++-- 8 files changed, 281 insertions(+), 50 deletions(-) create mode 100644 quest/m4/js-route-resume.md diff --git a/.claude/skills/verify-failover/SKILL.md b/.claude/skills/verify-failover/SKILL.md index a4c5baf4dea..670b6928ac1 100644 --- a/.claude/skills/verify-failover/SKILL.md +++ b/.claude/skills/verify-failover/SKILL.md @@ -5,7 +5,7 @@ description: Model-check failover along a chain of relays (`rs/moq-net/verify/fa # Verify failover along a relay chain -`rs/moq-net/verify/failover/FailoverChain.tla` models one track served to a client `C` through relay `E`, which starts on relay `A` and may fail over to relay `B`; both pull from the origin `O`, and with `Mesh` `B` may also route through `A`. Each layer is abstracted to the steps its own model checked, so the state stays small: a route change is one `Switch`, a switch asks the new copy from where the readers got to and each reader hands a group out once and ends once nothing it replaced may still deliver (`verify-resume`), and a group handed out crosses a hop unless its session dies (`verify-subscribe`, with an hour's budget). What the chain adds is frames: a group whose copy fails or stalls mid-read continues on another copy at the same frame (`Recover`), from a copy its relay holds, from the serving copy, or by fetching the rest from the serving route, which asks its own upstream in turn. A relay whose track ended keeps moving the reads that still hold it (`Switch` after `FrontFinish`) and forgets the track only once nothing reads it (`Forget`, the linger). +`rs/moq-net/verify/failover/FailoverChain.tla` models one track served to a client `C` through relay `E`, which starts on relay `A` and may fail over to relay `B`; both pull from the origin `O`, and with `Mesh` `B` may also route through `A`. Each layer is abstracted to the steps its own model checked, so the state stays small: a route change is one `Switch`, a switch asks the new copy from where the readers got to, past the frames of the newest group its relay holds, which it lends the new copy, and each reader hands a group out once and ends once nothing it replaced may still deliver (`verify-resume`), and a group handed out crosses a hop unless its session dies (`verify-subscribe`, with an hour's budget). What the chain adds is frames: a group whose copy fails or stalls mid-read continues on another copy at the same frame (`Recover`), from a copy its relay holds, from the serving copy, or by fetching the rest from the serving route, which asks its own upstream in turn. A relay whose track ended keeps moving the reads that still hold it (`Switch` after `FrontFinish`) and forgets the track only once nothing reads it (`Forget`, the linger). ## Rules @@ -27,7 +27,7 @@ quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq- quest gate just rs tla moq-net/verify/failover/FailoverChain.tla -config rs/moq-net/verify/failover/Mesh.cfg -simulate num=200000 -depth 100 ``` -The first checks every rule with `E` failing over between `A` and `B`, one death, and one switch per relay, with a client that stays, so a deadlock is a playback hang: about 2.6M states. The second lets the client leave at any point, so a deadlock is a relay still holding a copy after it left; leaving is always possible there, which is why it cannot stand in for the first: about 2.8M states. Each takes about a minute and passes at the sync commit. The third adds `B`'s route through `A`, so the route also moves at the middle hop, after `B`'s track ended too (`Action::Reroute`). It is too large to exhaust (past 139M distinct states at depth 38 after an hour, its queue still growing), so its `Simulate:` line samples 800k random behaviors in about 7 minutes, and Nightly samples new ones each run. An exhaustive run passed every state through depth 37. Before the reroute, a sample that size missed the depth-31 hang that BFS found after about 48M states, so a passing sample is weak evidence: check a change at the middle hop by running it without the `-simulate` arguments for as long as it takes. +The first checks every rule with `E` failing over between `A` and `B`, one death, and one switch per relay, with a client that stays, so a deadlock is a playback hang: about 3.8M states. The second lets the client leave at any point, so a deadlock is a relay still holding a copy after it left; leaving is always possible there, which is why it cannot stand in for the first: about 4.1M states. Each takes about three minutes and passes at the sync commit. The third adds `B`'s route through `A`, so the route also moves at the middle hop, after `B`'s track ended too (`Action::Reroute`). It is too large to exhaust (past 139M distinct states at depth 38 after an hour, its queue still growing), so its `Simulate:` line samples 800k random behaviors in about 7 minutes, and Nightly samples new ones each run. An exhaustive run passed every state through depth 37. Before the reroute, a sample that size missed the depth-31 hang that BFS found after about 48M states, so a passing sample is weak evidence: check a change at the middle hop by running it without the `-simulate` arguments for as long as it takes. ## Reading a failure @@ -35,11 +35,12 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048](https://github.com/moq-dev/moq/pull/9048), which mirrored the code since 88572c583e and kept a finished track a read holds past its linger. Run `git log 61f8956839.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048](https://github.com/moq-dev/moq/pull/9048), which mirrored the code since 88572c583e and kept a finished track a read holds past its linger, and [moq-dev/moq#9155](https://github.com/moq-dev/moq/pull/9155), which resumes the newest group past the frames held and lends them to the new copy (`Resume`, `Lend`). Run `git log 61f8956839.. -- rs/moq-net/src/model/resume.rs rs/moq-net/src/model/front.rs rs/moq-net/src/model/origin.rs rs/moq-net/src/model/track.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | | `rs/moq-net/src/model/resume.rs` `Reader::sync`, `backfill_gaps`, `Producer::subscription` | `ResumeAsk`, `Demand`, `Switch`, `Subscribe` | +| `rs/moq-net/src/model/resume.rs` `Reader::resume_at`, `held_at`, `Resumed`; `rs/moq-net/src/model/track.rs` `Consumer::lend_head`, the loan taken in `create_group_at` | `Resume`, `Lend`, `HandOut` | | `rs/moq-net/src/model/resume.rs` `Reader::take`, `poll_end` | `HandOut`, `Drained`, `EndOk` | | `rs/moq-net/src/model/resume.rs` `Reader::let_go`, `Copy::retains`, `Producer::park` | `LetGo`, `Park` | | `rs/moq-net/src/model/resume.rs` `Recover::wants`, `poll`, `held`, `poll_serving` | `Recover`, `Holders`, `Exhausted`, `Abort` | @@ -51,4 +52,4 @@ Sync commit: `61f89568394638553a519d7d8cfc0019385dd3a0`, plus [moq-dev/moq#9048] ## Not covered -Routing is abstracted to any live upstream winning, so which route wins and announce order are the router's own tests' (`rs/moq-net/src/route/tests`); how a reader picks among its copies, bounds replaced copies, and orders groups is `verify-resume`'s; drops, skips, and re-offers on a hop are `verify-subscribe`'s. The client's track fails only through `FrontFail`, so whether one dead relay can end playback is that step's guard, not a rule. Every hop has an hour's budget, so nothing goes stale or expires. No source refuses the track, so a finished track's reroute never concludes (`Action::Conclude`). Time is abstracted, so the linger may run out at any point once nothing reads the track. The origin writes each group whole and never fails, `E` never dies, a dead relay never returns, and a relay's cache is dropped when it parks. A fetched group holds its head from frame 0 (`lend_head`), and compressed groups are left out. A pass covers two groups of two frames, one death, one switch per relay, and one client. +Routing is abstracted to any live upstream winning, so which route wins and announce order are the router's own tests' (`rs/moq-net/src/route/tests`); how a reader picks among its copies, bounds replaced copies, and orders groups is `verify-resume`'s; drops, skips, and re-offers on a hop are `verify-subscribe`'s. The client's track fails only through `FrontFail`, so whether one dead relay can end playback is that step's guard, not a rule. Every hop has an hour's budget, so nothing goes stale or expires. No source refuses the track, so a finished track's reroute never concludes (`Action::Conclude`). Time is abstracted, so the linger may run out at any point once nothing reads the track. The origin writes each group whole and never fails, `E` never dies, a dead relay never returns, and a relay's cache is dropped when it parks. A fetched group holds its head from frame 0 (`lend_head`), and compressed groups are left out. The model lends only at a switch, so the query a front opens on a route before switching to it (`Action::Query`, `follow_demand`) is the unit test `a_query_resuming_a_new_copy_lends_it_the_head`, and a loan never runs out of cache budget. A pass covers two groups of two frames, one death, one switch per relay, and one client. diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 23abc93cb4d..486f46a0f1b 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -286,6 +286,40 @@ test("a lite-07 range may start partway through a group, which the span keeps", const overlapping = [{ start: { absolute: 7n, frame: 4 } }, { start: { absolute: 9n, frame: 2 } }]; expect([...positionFrames(overlapping)]).toEqual([[7n, 4]]); + // The sorted sweep keeps exactly what a scan of every range per Position keeps. + let seed = 0x6672616d; + const random = (n: number) => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + return seed % n; + }; + for (let i = 0; i < 2000; i++) { + const anchor = () => (random(3) === 0 ? { relative: BigInt(random(3)) } : { time: BigInt(random(8)) }); + const ranges: WireRange[] = []; + for (let n = random(6); n > 0; n--) { + const range: WireRange = + random(3) === 0 + ? { start: { absolute: BigInt(random(8)), frame: 1 + random(3) } } + : { start: anchor() }; + if (random(5) < 3) range.end = random(4) === 0 ? { relative: 0n } : { time: BigInt(random(8)) }; + ranges.push(range); + } + const scanned = new Map(); + for (const range of ranges) { + const start = range.start; + if (start?.absolute === undefined || !start.frame) continue; + let covering = 0; + for (const other of ranges) { + const from = other.start?.absolute ?? other.start?.time; + const to = other.end?.absolute ?? (other.end?.time === undefined ? undefined : other.end.time + 1n); + if ((from !== undefined && from > start.absolute) || (to !== undefined && to <= start.absolute)) + continue; + covering++; + } + if (covering === 1) scanned.set(start.absolute, start.frame); + } + expect(positionFrames(ranges)).toEqual(scanned); + } + // An end there can't go out. await expect( encodeMessage( diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 06a5f3c7e25..f1c282fceaa 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -324,27 +324,54 @@ export function spanOf(version: Version, ranges: readonly WireRange[]): Span & { /** * The first frame each lite-07 Position start asks its group from, by group, for each group no * other range may cover: a group is delivered once, so one another range covers goes whole. + * A peer may send thousands of ranges, so this sorts their bounds once rather than scanning + * them per Position. */ export function positionFrames(ranges: readonly WireRange[]): Map { const frames = new Map(); + const positions: [bigint, number][] = []; + for (const { start } of ranges) { + if (start?.absolute !== undefined && start.frame) positions.push([start.absolute, start.frame]); + } + if (positions.length === 0) return frames; + // A range may cover a group unless it starts past it or ends at or before it. One that + // starts at or past its end covers none, so with it left out, no range does both and the + // ranges covering a group are the rest less those two counts. + const starts: bigint[] = []; + const ends: bigint[] = []; + let count = 0; for (const range of ranges) { - const start = range.start; - if (start?.absolute === undefined || !start.frame) continue; - const group = start.absolute; - const frame = start.frame; - let covering = 0; - for (const other of ranges) { - const from = other.start?.absolute ?? other.start?.time; - const to = other.end?.absolute ?? (other.end?.time === undefined ? undefined : other.end.time + 1n); - // Unknown ends, such as a count back, may cover it. - if ((from !== undefined && from > group) || (to !== undefined && to <= group)) continue; - covering++; - } - if (covering === 1) frames.set(group, frame); + const from = range.start?.absolute ?? range.start?.time; + // Unknown ends, such as a count back, may cover it. + const to = range.end?.absolute ?? (range.end?.time === undefined ? undefined : range.end.time + 1n); + if (from !== undefined && to !== undefined && from >= to) continue; + count++; + if (from !== undefined) starts.push(from); + if (to !== undefined) ends.push(to); + } + const order = (a: bigint, b: bigint) => (a < b ? -1 : a > b ? 1 : 0); + starts.sort(order); + ends.sort(order); + for (const [group, frame] of positions) { + const past = starts.length - countAtOrBelow(starts, group); + const before = countAtOrBelow(ends, group); + if (count - past - before === 1) frames.set(group, frame); } return frames; } +/** How many of the `sorted` values are at or below `value`. */ +function countAtOrBelow(sorted: readonly bigint[], value: bigint): number { + let low = 0; + let high = sorted.length; + while (low < high) { + const mid = (low + high) >> 1; + if (sorted[mid] <= value) low = mid + 1; + else high = mid; + } + return low; +} + /** * The inverse of {@link encodeSpan}. A lite-07 list decodes as its ranges and as the one span * covering every range in it, which the subscriber narrows again. diff --git a/quest/m4/js-route-resume.md b/quest/m4/js-route-resume.md new file mode 100644 index 00000000000..2e4085207ea --- /dev/null +++ b/quest/m4/js-route-resume.md @@ -0,0 +1,14 @@ +# [L] A JS route switch resumes the newest group past the frames held + +## Goal + +A JS reader (`js/net/src/resume.ts`) whose route moves to a new copy asks that copy for the newest group it handed out from the frame after those a copy holds, as Rust does since [moq-dev/moq#9155](https://github.com/moq-dev/moq/pull/9155), so a route moving behind one next hop sends none of that group again over a lite-07 link. +The new copy's cache still holds the group whole for a reader that joins after the old copy is gone. + +## Plan + +- Rust's shape to mirror: `Reader::held_at` and `resume_at` pick the position, `Producer::subscription` returns a `Resumed` that lends the head with the demand, `Subscription::resume` aggregates only when every reader resumes, and the copy's track keeps one loan per group (`TrackState::lent`), bounded by `group::MAX_CACHE_BYTES`, which the group takes as it arrives partway through. +- JS has no cross-copy head adoption in its cache today, so the loan needs that first: a group arriving partway through takes the frames lent for it before any reader sees it. +- The wire already works in JS: the publisher serves a Position start (`positionFrames`), and `encodeSpan` writes one when a span's `startFrame` is set; only the reader side never asks for one. +- Port Rust's tests: `a_switch_resumes_the_newest_group_past_the_frames_held`, `a_resumed_copy_holds_the_group_whole_for_a_later_reader`, `a_query_resuming_a_new_copy_lends_it_the_head`, and the loan tests in `model/track.rs`. +- Today a JS relay resends that one group, which costs bandwidth but loses nothing, so this waits behind the Rust relay work. diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index c87732440bb..2e5d5cfccad 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -4704,23 +4704,46 @@ impl Bounds { /// The frame starts of `positions` that apply among `ranges`: a group delivers once across /// the ranges, so a frame start applies only to a group no other range may cover, since -/// another range's would ask for it from frame 0. Costs the ranges once per position, and -/// nothing while no range starts partway through a group. +/// another range's would ask for it from frame 0. A peer may send thousands of ranges, so this +/// sorts the ranges' times once rather than scanning them per position, and costs nothing +/// while no range starts partway through a group. fn first_frames<'a>( positions: impl IntoIterator, - ranges: impl IntoIterator + Clone, + ranges: impl IntoIterator, ) -> Vec<(u64, u64)> { use crate::track::Anchor; - (positions.into_iter().copied()) - .filter(|(group, _)| { - let covers = |range: &&crate::track::Range| match (range.start, range.end) { - (Anchor::Time(start), _) if start > *group => false, - (_, Some(Anchor::Time(end))) if end < *group => false, - _ => true, - }; - ranges.clone().into_iter().filter(covers).nth(1).is_none() - }) - .collect() + let mut positions: Vec<_> = positions.into_iter().copied().collect(); + if positions.is_empty() { + return positions; + } + // A range may cover a group unless it starts past it or ends before it, by time. One that + // starts past where it ends covers none, so with it left out, no range does both and the + // ranges covering a group are the rest less those two counts. + let (mut count, mut starts, mut ends) = (0, Vec::new(), Vec::new()); + for range in ranges { + let start = match range.start { + Anchor::Time(start) => Some(start), + _ => None, + }; + let end = match range.end { + Some(Anchor::Time(end)) => Some(end), + _ => None, + }; + if start.zip(end).is_some_and(|(start, end)| start > end) { + continue; + } + count += 1; + starts.extend(start); + ends.extend(end); + } + starts.sort_unstable(); + ends.sort_unstable(); + positions.retain(|(group, _)| { + let past = starts.len() - starts.partition_point(|start| start <= group); + let before = ends.partition_point(|end| end < group); + count - past - before <= 1 + }); + positions } /// A subscription's requested ranges in the model's whole groups, by Range ID, with the @@ -7078,6 +7101,41 @@ mod serve_group_test { assert_eq!((inside.frames.start(4), inside.frames.end(6)), (0, None)); } + /// The sweep in `first_frames` keeps exactly the positions a scan of every range per + /// position keeps: those no more than one range may cover. + #[test] + fn first_frames_matches_a_scan_of_every_range() { + use rand::{RngExt, SeedableRng}; + use track::Anchor; + let mut rng = rand::rngs::StdRng::seed_from_u64(0x6672616d6573); + for _ in 0..2000 { + let anchor = |rng: &mut rand::rngs::StdRng| match rng.random_range(0..3) { + 0 => Anchor::Relative(rng.random_range(0..3)), + _ => Anchor::Time(rng.random_range(0..8)), + }; + let ranges: Vec = (0..rng.random_range(0..6)) + .map(|_| track::Range { + start: anchor(&mut rng), + end: rng.random_bool(0.6).then(|| anchor(&mut rng)), + }) + .collect(); + let positions: Vec<(u64, u64)> = (0..rng.random_range(0..4)) + .map(|_| (rng.random_range(0..8), rng.random_range(1..4))) + .collect(); + let scanned: Vec<(u64, u64)> = (positions.iter().copied()) + .filter(|(group, _)| { + let covers = |range: &&track::Range| match (range.start, range.end) { + (Anchor::Time(start), _) if start > *group => false, + (_, Some(Anchor::Time(end))) if end < *group => false, + _ => true, + }; + ranges.iter().filter(covers).nth(1).is_none() + }) + .collect(); + assert_eq!(first_frames(&positions, &ranges), scanned, "{ranges:?} {positions:?}"); + } + } + /// Each lite-07 range starting partway through a group serves that group from its frame, /// unless another range may cover it too, which asks for it whole. #[test] diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 34e2ea3dcc3..111d0b25a4d 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -415,7 +415,7 @@ impl Head { } /// Roughly what holding it costs: payloads, fields, and their slots. - fn bytes(&self) -> u64 { + pub(super) fn bytes(&self) -> u64 { let payload: u64 = self.entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); let fields: u64 = self .entries diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 29cbadc1a0f..9dc4c036e8e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -707,8 +707,9 @@ pub(crate) struct TrackState { // group past them, by key, which the group adopts as it arrives (see // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One per // group, taken as that group arrives or dropped as it is given up or the track tears down, - // so it holds only the groups readers resumed this copy at that are still to come. - lent: Mutex>>, + // and together never past what one group may cache, so it holds only the newest groups + // readers resumed this copy at that are still to come. + lent: Mutex, // The key and start of the highest-key group that left the cache holding a // checked start (see [`group::Origin::Bound`]). Eviction is not a key prefix, so a @@ -1258,10 +1259,52 @@ impl Holdings for TrackState { } } +/// The heads lent to a copy for the groups its readers resumed it at, by key; see +/// [`Consumer::lend_head`]. +#[derive(Default)] +struct Lent { + heads: BTreeMap>, + /// What they hold together, never past [`group::MAX_CACHE_BYTES`]: no group adopts a + /// head past what it may cache, and readers that keep moving on while the copy delivers + /// nothing keep lending it newer groups, so the oldest go first. + bytes: u64, +} + +impl Lent { + /// Hold `head` for group `key`. Of two loans for one group the longer stands. + fn lend(&mut self, key: u64, head: &Arc) { + let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); + if self.heads.get(&key).is_some_and(|slot| held(slot) >= held(head)) { + return; + } + self.take(key); + self.bytes += head.bytes(); + self.heads.insert(key, head.clone()); + while self.bytes > group::MAX_CACHE_BYTES + && let Some((_, head)) = self.heads.pop_first() + { + self.bytes -= head.bytes(); + } + } + + fn take(&mut self, key: u64) -> Option> { + let head = self.heads.remove(&key)?; + self.bytes -= head.bytes(); + Some(head) + } + + fn retain(&mut self, keep: impl Fn(u64) -> bool) { + let dropped: Vec = self.heads.keys().copied().filter(|key| !keep(*key)).collect(); + for key in dropped { + self.take(key); + } + } +} + impl TrackState { /// The head lent for group `key`, taken; see [`Consumer::lend_head`]. fn take_lent(&self, key: u64) -> Option> { - self.lent.lock().unwrap_or_else(PoisonError::into_inner).remove(&key) + self.lent.lock().unwrap_or_else(PoisonError::into_inner).take(key) } /// Hold `head` for group `key` until it arrives, or return the copies of it already @@ -1271,12 +1314,7 @@ impl TrackState { return slot.copies().cloned().collect(); } // Under the caller's read guard, so the group cannot arrive before the loan is held. - let mut lent = self.lent.lock().unwrap_or_else(PoisonError::into_inner); - let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); - let slot = lent.entry(key).or_insert_with(|| head.clone()); - if held(head) > held(slot) { - *slot = head.clone(); - } + self.lent.lock().unwrap_or_else(PoisonError::into_inner).lend(key, head); Vec::new() } @@ -1646,7 +1684,7 @@ impl TrackState { // A group given up never arrives to take the head lent for it. if hole.is_some() { let lent = self.lent.get_mut().unwrap_or_else(PoisonError::into_inner); - lent.retain(|key, _| !groups.contains(key)); + lent.retain(|key| !groups.contains(&key)); } if hole.as_ref().is_none_or(|hole| hole.stale_budget().is_none()) && self @@ -2728,7 +2766,7 @@ impl TrackState { fn tear_down(&mut self, err: Error) -> (Option, OpenGroups) { self.release_ingress(); // Nothing arrives after this to take a lent head. - self.lent.get_mut().unwrap_or_else(PoisonError::into_inner).clear(); + *self.lent.get_mut().unwrap_or_else(PoisonError::into_inner) = Lent::default(); let resume = self.resume_position(); let groups = self .lookup @@ -16062,6 +16100,27 @@ mod test { edge.finish().unwrap(); } + /// The heads lent to a copy never hold more together than one group may cache: readers + /// moving on while the copy delivers nothing keep lending newer groups, and the oldest go. + #[test] + #[cfg(feature = "async")] + fn lent_heads_keep_within_what_a_group_caches() { + let producer = track_producer("test", Info::timed()); + let consumer = producer.consume(); + let half = (group::MAX_CACHE_BYTES / 2) as usize; + for key in [1, 2, 3] { + let mut prefix = group::Info { start: key }.produce(); + prefix + .write_frame(Timestamp::from_millis(key).unwrap(), bytes::Bytes::from(vec![0; half])) + .unwrap(); + consumer.lend_head(key, prefix.consume().handoff().head().unwrap()); + } + let lent = producer.modify().unwrap(); + let lent = lent.lent.lock().unwrap(); + assert_eq!(lent.heads.keys().copied().collect::>(), [3]); + assert!(lent.bytes <= group::MAX_CACHE_BYTES); + } + /// A copy lent the heads of two groups its readers resumed it at holds each whole as it /// arrives partway through, from the moment a reader can see it. #[test] @@ -16085,13 +16144,22 @@ mod test { // A group given up drops its loan. consumer.lend_head(8, head(8, 1)); producer.skip_groups(8..9); - assert!(!producer.modify().unwrap().lent.get_mut().unwrap().contains_key(&8)); + assert!( + !producer + .modify() + .unwrap() + .lent + .get_mut() + .unwrap() + .heads + .contains_key(&8) + ); // So does every one still waiting once the copy tears down. let aborted = track_producer("aborted", Info::timed()); let reader = aborted.consume(); reader.lend_head(3, head(3, 1)); aborted.abort(Error::Cancel).unwrap(); - assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().is_empty())); + assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().heads.is_empty())); for (key, at) in [(7, 2), (9, 1)] { let Received::Group(group) = producer.create_group_at(group::Info { start: key }, at).unwrap() else { panic!("nothing else holds group {key}"); diff --git a/rs/moq-net/verify/failover/FailoverChain.tla b/rs/moq-net/verify/failover/FailoverChain.tla index f36f4beb3cd..dd562ca87d6 100644 --- a/rs/moq-net/verify/failover/FailoverChain.tla +++ b/rs/moq-net/verify/failover/FailoverChain.tla @@ -89,15 +89,19 @@ vars == <> (* bad groups the upstream aborted mid-read. *) (* fin the upstream's reader ended the subscription cleanly. *) (* src src[g]: the upstream's copy its reader reads g from (`Recover`).*) +(* lent lent[g]: frames of g a switch lent it, which it holds from the *) +(* moment g is handed out, asking the upstream only for the rest. *) (***************************************************************************) Fresh(ask) == [st |-> "live", ask |-> ask, fetch |-> {}, fr |-> [g \in Groups |-> -1], - bad |-> {}, fin |-> FALSE, src |-> [g \in Groups |-> None]] + bad |-> {}, fin |-> FALSE, src |-> [g \in Groups |-> None], + lent |-> [g \in Groups |-> 0]] Gone == [st |-> "none", ask |-> {}, fetch |-> {}, fr |-> [g \in Groups |-> -1], - bad |-> {}, fin |-> FALSE, src |-> [g \in Groups |-> None]] + bad |-> {}, fin |-> FALSE, src |-> [g \in Groups |-> None], + lent |-> [g \in Groups |-> 0]] Live(e) == cp[e].st = "live" Held(e) == cp[e].st # "none" @@ -127,6 +131,30 @@ ResumeAsk(e) == from == IF got = {} THEN 0 ELSE IF gaps # {} THEN Min(gaps) ELSE newest IN {g \in cp[e].ask : g >= from} +\* Where a switch at u resumes its readers (`Producer::subscription`, `Reader::resume_at`): +\* each open reader's newest group, past the most frames of it a copy at u holds, when it +\* lacks no group below it; the lowest of those, as a set of one, or none when any reader +\* resumes nothing. +Resume(u) == + LET readers == {e \in Downs(u) : Live(e) /\ ~cp[e].fin} + At(e) == + LET got == {g \in Groups : cp[e].fr[g] >= 0} + newest == Max(got) + gaps == {g \in cp[e].ask : g < newest /\ cp[e].fr[g] = -1} + held == Max({SrcFr(u, s, newest) : s \in Srcs(u)} \cup {0}) + IN IF got = {} \/ gaps # {} \/ held = 0 THEN {} ELSE {<>} + at == UNION {At(e) : e \in readers} + IN IF readers = {} \/ \E e \in readers : At(e) = {} THEN {} + ELSE {CHOOSE p \in at : \A q \in at : p[1] < q[1] \/ (p[1] = q[1] /\ p[2] <= q[2])} + +\* A copy asked to resume a group past the frames held is lent them, unless it already +\* holds that group (`lend_head`); of two loans the longer stands. +Lend(u, c) == + IF Resume(u) = {} THEN c + ELSE LET p == CHOOSE p \in Resume(u) : TRUE IN + IF c.fr[p[1]] >= 0 THEN c + ELSE [c EXCEPT !.lent[p[1]] = IF @ > p[2] THEN @ ELSE p[2]] + \* A downstream copy still reads the track at its upstream: its subscription is open, or \* a group it was handed or fetched is not over (`TrackWeak::is_held`). Reads(e) == @@ -219,7 +247,7 @@ HandOut(e, g, s) == /\ ~cp[e].fin \/ g \in cp[e].fetch /\ s \in Srcs(e[2]) /\ SrcFr(e[2], s, g) >= 0 - /\ cp' = [cp EXCEPT ![e].fr[g] = 0, ![e].src[g] = s] + /\ cp' = [cp EXCEPT ![e].fr[g] = cp[e].lent[g], ![e].src[g] = s] /\ UNCHANGED <> \* The next frame of a group crosses the hop. @@ -320,9 +348,10 @@ Switch(u, w) == /\ usable => front[u].sw < MaxSwitches /\ front' = [front EXCEPT ![u].cur = w, ![u].sw = @ + (IF usable THEN 1 ELSE 0)] /\ cp' = [cp EXCEPT ![e1] = - IF Live(e1) /\ ~cp[e1].fin /\ open THEN [@ EXCEPT !.ask = Demand(u)] + IF Live(e1) /\ ~cp[e1].fin /\ open THEN Lend(u, [@ EXCEPT !.ask = Demand(u)]) ELSE IF Live(e1) THEN @ - ELSE Fresh(IF open THEN Demand(u) ELSE {})] + ELSE IF open THEN Lend(u, Fresh(Demand(u))) + ELSE Fresh({})] /\ UNCHANGED <> \* The serving copy completed: the front ends the track (`Action::Finish`). @@ -468,7 +497,7 @@ TypeOK == /\ \A e \in Edges : /\ cp[e].st \in {"none", "live", "failed"} /\ cp[e].bad \subseteq Groups - /\ \A g \in Groups : cp[e].fr[g] \in -1 .. F + /\ \A g \in Groups : cp[e].fr[g] \in -1 .. F /\ cp[e].lent[g] \in 0 .. F /\ \A r \in Relays : front[r].end \in {"open", "ok", "err"} ============================================================================= From 51e86d0911167d6eadd9740ed293c57d36c6b8d4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:10:22 -0800 Subject: [PATCH 13/21] fix(net): a lent head is held and reclaimed like a failed copy's head Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 52 +++++++++++++++++++--- rs/moq-net/src/model/track.rs | 82 +++++++++-------------------------- 2 files changed, 66 insertions(+), 68 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 111d0b25a4d..40c06bf6cc8 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -370,9 +370,10 @@ pub(super) struct Head { /// /// Read from the copy while it holds its frames. Once it fails, its release moves them /// here instead, so a plain consumer of the failed copy still reads the abort and pins -/// nothing, and the frames go with the last reader holding this. +/// nothing, and the frames go with the last reader holding this. A loan (see +/// [`Self::lent`]) holds a head with no copy behind it. pub(super) struct Handoff { - state: kio::ConsumerWeak, + state: Option>, salvaged: Mutex>, } @@ -387,11 +388,24 @@ impl Handoff { pub(super) fn head(&self) -> Option> { // The group's lock first, in `release`'s order: under it the frames are either // still in the copy or already moved here, never in neither. - let state = self.state.read(); + let state = self.state.as_ref().map(|state| state.read()); let salvaged = (self.salvaged.lock().unwrap_or_else(PoisonError::into_inner)) .as_ref() .map(|salvage| salvage.head.clone()); - salvaged.or_else(|| state.head().map(Arc::new)) + salvaged.or_else(|| state?.head().map(Arc::new)) + } + + /// Hold `head` for a copy of its group still to arrive, which adopts it then, billed to + /// that copy's pool and reclaimed like a failed copy's head: under capacity pressure, or + /// once idle. `None` when the pool has no room for it. + pub(super) fn lent(head: Arc, cache: &Arc) -> Option> { + let bill = cache.hold(head.bytes())?; + let handoff = Arc::new(Self { + state: None, + salvaged: Mutex::new(Some(Salvage { head, bill })), + }); + cache.pool().hold_head(&handoff); + Some(handoff) } /// Drop the held head when `reclaim`, or once the pool's idle window passed, returning @@ -415,7 +429,7 @@ impl Head { } /// Roughly what holding it costs: payloads, fields, and their slots. - pub(super) fn bytes(&self) -> u64 { + fn bytes(&self) -> u64 { let payload: u64 = self.entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); let fields: u64 = self .entries @@ -3999,7 +4013,7 @@ impl Consumer { return handoff; } let shared = Arc::new(Handoff { - state: self.cursor.state.weak(), + state: Some(self.cursor.state.weak()), salvaged: Mutex::default(), }); *handoff = Arc::downgrade(&shared); @@ -5746,6 +5760,32 @@ mod test { assert_eq!(pool.used(), 0); } + /// A head lent to a copy for a group still to arrive is billed to that copy's pool like a + /// failed copy's held head, and reclaimed the same way: refused when the pool is full, and + /// dropped once idle past the window. + #[test] + fn a_lent_head_is_billed_and_expires_with_the_pool_window() { + let source = cache::Track::new(cache::Pool::unbounded(), Default::default(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), source); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"a")) + .unwrap(); + let head = producer.consume().handoff().head().unwrap(); + + let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); + let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); + let loan = Handoff::lent(head.clone(), &cache).expect("room for it"); + assert!(pool.used() > 0, "billed while lent"); + pool.step(std::time::Duration::from_secs(2)); + pool.sweep(); + assert!(loan.head().is_none(), "the sweep drops an idle loan"); + assert_eq!(pool.used(), 0); + + let full = cache::Pool::new(cache::Config::default().with_capacity(1)); + let cache = cache::Track::new(full, Default::default(), kio::Weak::new()); + assert!(Handoff::lent(head, &cache).is_none(), "a full pool lends nothing"); + } + /// The registry the sweep expires held heads from grows with the heads held, not with /// every group that failed, and gives back what a burst grew it to once they expire. #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 9dc4c036e8e..5bae73a7e6e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -707,8 +707,8 @@ pub(crate) struct TrackState { // group past them, by key, which the group adopts as it arrives (see // [`Consumer::lend_head`]). Filled through a read guard, so it needs its own lock. One per // group, taken as that group arrives or dropped as it is given up or the track tears down, - // and together never past what one group may cache, so it holds only the newest groups - // readers resumed this copy at that are still to come. + // so it holds only the groups readers resumed this copy at that are still to come, each + // billed to the pool, which reclaims it as it would a failed copy's head. lent: Mutex, // The key and start of the highest-key group that left the cache holding a @@ -1259,45 +1259,33 @@ impl Holdings for TrackState { } } -/// The heads lent to a copy for the groups its readers resumed it at, by key; see -/// [`Consumer::lend_head`]. +/// The heads lent to a copy for the groups its readers resumed it at, by key, each held as a +/// failed copy's head is (see [`group::Handoff::lent`]); see [`Consumer::lend_head`]. #[derive(Default)] -struct Lent { - heads: BTreeMap>, - /// What they hold together, never past [`group::MAX_CACHE_BYTES`]: no group adopts a - /// head past what it may cache, and readers that keep moving on while the copy delivers - /// nothing keep lending it newer groups, so the oldest go first. - bytes: u64, -} +struct Lent(BTreeMap>); impl Lent { - /// Hold `head` for group `key`. Of two loans for one group the longer stands. - fn lend(&mut self, key: u64, head: &Arc) { + /// Hold `head` for group `key`, billed to `cache`. Of two loans for one group the longer + /// stands. + fn lend(&mut self, key: u64, head: &Arc, cache: &Arc) { let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); - if self.heads.get(&key).is_some_and(|slot| held(slot) >= held(head)) { + let standing = self.0.get(&key).and_then(|loan| loan.head()); + if standing.is_some_and(|slot| held(&slot) >= held(head)) { return; } - self.take(key); - self.bytes += head.bytes(); - self.heads.insert(key, head.clone()); - while self.bytes > group::MAX_CACHE_BYTES - && let Some((_, head)) = self.heads.pop_first() - { - self.bytes -= head.bytes(); - } + match group::Handoff::lent(head.clone(), cache) { + Some(loan) => self.0.insert(key, loan), + None => self.0.remove(&key), + }; } + /// The head lent for group `key`, unless its pool reclaimed it. fn take(&mut self, key: u64) -> Option> { - let head = self.heads.remove(&key)?; - self.bytes -= head.bytes(); - Some(head) + self.0.remove(&key)?.head() } fn retain(&mut self, keep: impl Fn(u64) -> bool) { - let dropped: Vec = self.heads.keys().copied().filter(|key| !keep(*key)).collect(); - for key in dropped { - self.take(key); - } + self.0.retain(|key, _| keep(*key)); } } @@ -1314,7 +1302,7 @@ impl TrackState { return slot.copies().cloned().collect(); } // Under the caller's read guard, so the group cannot arrive before the loan is held. - self.lent.lock().unwrap_or_else(PoisonError::into_inner).lend(key, head); + (self.lent.lock().unwrap_or_else(PoisonError::into_inner)).lend(key, head, &self.cache); Vec::new() } @@ -16100,27 +16088,6 @@ mod test { edge.finish().unwrap(); } - /// The heads lent to a copy never hold more together than one group may cache: readers - /// moving on while the copy delivers nothing keep lending newer groups, and the oldest go. - #[test] - #[cfg(feature = "async")] - fn lent_heads_keep_within_what_a_group_caches() { - let producer = track_producer("test", Info::timed()); - let consumer = producer.consume(); - let half = (group::MAX_CACHE_BYTES / 2) as usize; - for key in [1, 2, 3] { - let mut prefix = group::Info { start: key }.produce(); - prefix - .write_frame(Timestamp::from_millis(key).unwrap(), bytes::Bytes::from(vec![0; half])) - .unwrap(); - consumer.lend_head(key, prefix.consume().handoff().head().unwrap()); - } - let lent = producer.modify().unwrap(); - let lent = lent.lent.lock().unwrap(); - assert_eq!(lent.heads.keys().copied().collect::>(), [3]); - assert!(lent.bytes <= group::MAX_CACHE_BYTES); - } - /// A copy lent the heads of two groups its readers resumed it at holds each whole as it /// arrives partway through, from the moment a reader can see it. #[test] @@ -16144,22 +16111,13 @@ mod test { // A group given up drops its loan. consumer.lend_head(8, head(8, 1)); producer.skip_groups(8..9); - assert!( - !producer - .modify() - .unwrap() - .lent - .get_mut() - .unwrap() - .heads - .contains_key(&8) - ); + assert!(!producer.modify().unwrap().lent.get_mut().unwrap().0.contains_key(&8)); // So does every one still waiting once the copy tears down. let aborted = track_producer("aborted", Info::timed()); let reader = aborted.consume(); reader.lend_head(3, head(3, 1)); aborted.abort(Error::Cancel).unwrap(); - assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().heads.is_empty())); + assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().0.is_empty())); for (key, at) in [(7, 2), (9, 1)] { let Received::Group(group) = producer.create_group_at(group::Info { start: key }, at).unwrap() else { panic!("nothing else holds group {key}"); From 1275b09868d8a88b15c632b152657a6ce683987d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:33:13 -0800 Subject: [PATCH 14/21] fix(net): a copy forgets the loans its pool reclaimed Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 47 ++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 5bae73a7e6e..9c4cb2c73a1 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1273,10 +1273,12 @@ impl Lent { if standing.is_some_and(|slot| held(&slot) >= held(head)) { return; } - match group::Handoff::lent(head.clone(), cache) { - Some(loan) => self.0.insert(key, loan), - None => self.0.remove(&key), - }; + // Forget the loans the pool reclaimed, so the map grows no further than the loans it + // bills, however many groups readers resume a copy at that never arrive. + self.0.retain(|_, loan| loan.head().is_some()); + if let Some(loan) = group::Handoff::lent(head.clone(), cache) { + self.0.insert(key, loan); + } } /// The head lent for group `key`, unless its pool reclaimed it. @@ -16088,6 +16090,43 @@ mod test { edge.finish().unwrap(); } + /// A copy whose readers keep resuming it at groups that never arrive keeps no more loans + /// than its pool bills: one the pool reclaimed is forgotten at the next loan. + #[test] + #[cfg(feature = "async")] + fn reclaimed_loans_leave_the_copy() { + let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); + let producer = track_producer_pooled("test", pool.clone()); + let consumer = producer.consume(); + let head = |start: u64| { + let mut prefix = group::Info { start }.produce(); + prefix + .write_frame(Timestamp::from_millis(start).unwrap(), bytes::Bytes::new()) + .unwrap(); + prefix.consume().handoff().head().unwrap() + }; + let lent = |producer: &Producer| { + producer + .state + .read() + .lent + .lock() + .unwrap() + .0 + .keys() + .copied() + .collect::>() + }; + consumer.lend_head(1, head(1)); + consumer.lend_head(2, head(2)); + assert_eq!(lent(&producer), [1, 2]); + pool.step(Duration::from_secs(2)); + pool.sweep(); + assert_eq!(pool.used(), 0, "the sweep reclaimed both"); + consumer.lend_head(3, head(3)); + assert_eq!(lent(&producer), [3]); + } + /// A copy lent the heads of two groups its readers resumed it at holds each whole as it /// arrives partway through, from the moment a reader can see it. #[test] From 9e5ca308d35be831fd83dd94e906cb3d2b2c98a7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:50:56 -0800 Subject: [PATCH 15/21] perf(net): a copy forgets reclaimed loans once they double, not on every loan Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 47 ++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 9c4cb2c73a1..6601b350b52 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1262,32 +1262,40 @@ impl Holdings for TrackState { /// The heads lent to a copy for the groups its readers resumed it at, by key, each held as a /// failed copy's head is (see [`group::Handoff::lent`]); see [`Consumer::lend_head`]. #[derive(Default)] -struct Lent(BTreeMap>); +struct Lent { + loans: BTreeMap>, + /// How many loans it holds before it next forgets those the pool reclaimed: twice what + /// the last pass kept, so a pass costs no more than the loans added since. + prune_at: usize, +} impl Lent { /// Hold `head` for group `key`, billed to `cache`. Of two loans for one group the longer /// stands. fn lend(&mut self, key: u64, head: &Arc, cache: &Arc) { let held = |head: &group::Head| head.frames_from_first().unwrap_or_default(); - let standing = self.0.get(&key).and_then(|loan| loan.head()); + let standing = self.loans.get(&key).and_then(|loan| loan.head()); if standing.is_some_and(|slot| held(&slot) >= held(head)) { return; } - // Forget the loans the pool reclaimed, so the map grows no further than the loans it - // bills, however many groups readers resume a copy at that never arrive. - self.0.retain(|_, loan| loan.head().is_some()); + // Forget the loans the pool reclaimed, so the map grows no further than twice the loans + // it bills, however many groups readers resume a copy at that never arrive. + if self.loans.len() >= self.prune_at { + self.loans.retain(|_, loan| loan.head().is_some()); + self.prune_at = 2 * self.loans.len() + 1; + } if let Some(loan) = group::Handoff::lent(head.clone(), cache) { - self.0.insert(key, loan); + self.loans.insert(key, loan); } } /// The head lent for group `key`, unless its pool reclaimed it. fn take(&mut self, key: u64) -> Option> { - self.0.remove(&key)?.head() + self.loans.remove(&key)?.head() } fn retain(&mut self, keep: impl Fn(u64) -> bool) { - self.0.retain(|key, _| keep(*key)); + self.loans.retain(|key, _| keep(*key)); } } @@ -16090,8 +16098,8 @@ mod test { edge.finish().unwrap(); } - /// A copy whose readers keep resuming it at groups that never arrive keeps no more loans - /// than its pool bills: one the pool reclaimed is forgotten at the next loan. + /// A copy whose readers keep resuming it at groups that never arrive keeps no more than + /// twice the loans its pool bills: those the pool reclaimed are forgotten as loans grow. #[test] #[cfg(feature = "async")] fn reclaimed_loans_leave_the_copy() { @@ -16112,7 +16120,7 @@ mod test { .lent .lock() .unwrap() - .0 + .loans .keys() .copied() .collect::>() @@ -16123,8 +16131,10 @@ mod test { pool.step(Duration::from_secs(2)); pool.sweep(); assert_eq!(pool.used(), 0, "the sweep reclaimed both"); + // Forgotten once the loans double since the last pass, so a pass costs what was added. consumer.lend_head(3, head(3)); - assert_eq!(lent(&producer), [3]); + consumer.lend_head(4, head(4)); + assert_eq!(lent(&producer), [3, 4]); } /// A copy lent the heads of two groups its readers resumed it at holds each whole as it @@ -16150,13 +16160,22 @@ mod test { // A group given up drops its loan. consumer.lend_head(8, head(8, 1)); producer.skip_groups(8..9); - assert!(!producer.modify().unwrap().lent.get_mut().unwrap().0.contains_key(&8)); + assert!( + !producer + .modify() + .unwrap() + .lent + .get_mut() + .unwrap() + .loans + .contains_key(&8) + ); // So does every one still waiting once the copy tears down. let aborted = track_producer("aborted", Info::timed()); let reader = aborted.consume(); reader.lend_head(3, head(3, 1)); aborted.abort(Error::Cancel).unwrap(); - assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().0.is_empty())); + assert!(reader.with_state(|state| state.unwrap().read().lent.lock().unwrap().loans.is_empty())); for (key, at) in [(7, 2), (9, 1)] { let Received::Group(group) = producer.create_group_at(group::Info { start: key }, at).unwrap() else { panic!("nothing else holds group {key}"); From f5805dce41dd26548e86535ce94eef0a4e58c842 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:56:41 -0800 Subject: [PATCH 16/21] fix(net): a loan for a cached group stands for a copy that displaces it Co-Authored-By: Claude Opus 5.5 --- .../test-flakes-3/rust-audio-close-drain.md | 11 +++--- rs/moq-net/src/model/track.rs | 36 ++++++++++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/quest/m0/test-flakes-3/rust-audio-close-drain.md b/quest/m0/test-flakes-3/rust-audio-close-drain.md index 0d5a3434dbe..41a1a5b8309 100644 --- a/quest/m0/test-flakes-3/rust-audio-close-drain.md +++ b/quest/m0/test-flakes-3/rust-audio-close-drain.md @@ -14,9 +14,12 @@ Seen once on 2026-11-19 while validating (opus) with libopus`), then failed with `Error: timeout` after 7 s of its 30 s budget. The client had connected over the WebSocket fallback, a sign the host was loaded. Three reruns of the cell alone passed, and main's scheduled -Interop runs show no such cell. So the subscriber's close drain (one second, -`rs/moq-net/src/session.rs` `poll_drain`) found something still owed to the -relay under load. Find what a subscribing session still waits on at close, -and fix that cause rather than the deadline or the client's `?`. +Interop runs show no such cell. The output does not say which deadline +returned the timeout. The likeliest is the subscriber's close drain (one second, +`rs/moq-net/src/session.rs` `poll_drain`), but `Error::Timeout` also comes +from the route stream (`rs/moq-net/src/lite/subscriber/route_stream.rs`) and +the publisher and server deadlines. First find which one fired, by logging the +error's path in the client after it decodes, then what it still waited on, and +fix that cause rather than the deadline or the client's `?`. Public API: none. Wire: none. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6601b350b52..36357fd0869 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1308,12 +1308,11 @@ impl TrackState { /// Hold `head` for group `key` until it arrives, or return the copies of it already /// cached, to adopt it with this state's lock released; see [`Consumer::lend_head`]. fn lend(&self, key: u64, head: &Arc) -> Vec { - if let Some(slot) = self.lookup.get(&key) { - return slot.copies().cloned().collect(); - } // Under the caller's read guard, so the group cannot arrive before the loan is held. + // Held even when copies are cached: one arriving to displace them once the guard is + // released, before they adopt the head, takes the loan as it lands. (self.lent.lock().unwrap_or_else(PoisonError::into_inner)).lend(key, head, &self.cache); - Vec::new() + (self.lookup.get(&key)).map_or_else(Vec::new, |slot| slot.copies().cloned().collect()) } /// How many keys just below `key` the publisher declared it never produced: the skip @@ -16137,6 +16136,35 @@ mod test { assert_eq!(lent(&producer), [3, 4]); } + /// A loan for a group the copy already caches still stands for a copy of it that lands + /// later and displaces the cached one, which adopts it as it lands. + #[test] + #[cfg(feature = "async")] + fn a_displacing_arrival_takes_the_loan() { + let producer = track_producer("test", Info::timed()); + let consumer = producer.consume(); + let Received::Group(late) = producer.create_group_at(group::Info { start: 5 }, 2).unwrap() else { + panic!("nothing else holds group 5"); + }; + let mut prefix = group::Info { start: 5 }.produce(); + prefix + .write_frame(Timestamp::from_millis(5).unwrap(), bytes::Bytes::new()) + .unwrap(); + // One frame does not reach the cached copy's first, so that copy cannot adopt it. + consumer.lend_head(5, prefix.consume().handoff().head().unwrap()); + assert_eq!(consumer.peek_group(5).expect("cached").first_frame(), 2); + + let Received::Group(wider) = producer.create_group_at(group::Info { start: 5 }, 1).unwrap() else { + panic!("a copy from frame 1 displaces one from frame 2"); + }; + assert_eq!( + consumer.peek_group(5).expect("cached").first_frame(), + 0, + "whole once visible" + ); + drop((late, wider)); + } + /// A copy lent the heads of two groups its readers resumed it at holds each whole as it /// arrives partway through, from the moment a reader can see it. #[test] From a287918bea384cab8870617cd73cf9aab4f452af Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:44:41 -0800 Subject: [PATCH 17/21] fix(net): an empty Position range starts no group partway, and resumes are counted as they change Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscribe.test.ts | 4 ++++ js/net/src/lite/subscribe.ts | 8 +++++-- rs/moq-net/src/lite/publisher.rs | 31 ++++++++++++++++++++++------ rs/moq-net/src/model/track/demand.rs | 6 +++++- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 402a85af281..72b95844fa7 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -285,6 +285,8 @@ test("a lite-07 range may start partway through a group, which the span keeps", ]); const overlapping = [{ start: { absolute: 7n, frame: 4 } }, { start: { absolute: 9n, frame: 2 } }]; expect([...positionFrames(overlapping)]).toEqual([[7n, 4]]); + // A Position range ending before its group covers nothing, so the group goes whole. + expect([...positionFrames([{ start: { absolute: 7n, frame: 4 }, end: { time: 6n } }, {}])]).toEqual([]); // The sorted sweep keeps exactly what a scan of every range per Position keeps. let seed = 0x6672616d; @@ -307,6 +309,8 @@ test("a lite-07 range may start partway through a group, which the span keeps", for (const range of ranges) { const start = range.start; if (start?.absolute === undefined || !start.frame) continue; + const own = range.end?.absolute ?? (range.end?.time === undefined ? undefined : range.end.time + 1n); + if (own !== undefined && own <= start.absolute) continue; let covering = 0; for (const other of ranges) { const from = other.start?.absolute ?? other.start?.time; diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 5ee9eccb4e7..e7f08dc0349 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -351,8 +351,12 @@ export function spanBetween( export function positionFrames(ranges: readonly WireRange[]): Map { const frames = new Map(); const positions: [bigint, number][] = []; - for (const { start } of ranges) { - if (start?.absolute !== undefined && start.frame) positions.push([start.absolute, start.frame]); + for (const range of ranges) { + const start = range.start; + // One ending at or before the group it starts partway through covers nothing. + const [, to] = rangeBounds(range); + if (start?.absolute !== undefined && start.frame && (to === undefined || to > start.absolute)) + positions.push([start.absolute, start.frame]); } if (positions.length === 0) return frames; // A range may cover a group unless it starts past it or ends at or before it. One that diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 9fbe93f22f8..f93def98457 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -4659,12 +4659,10 @@ impl Bounds { if legacy && let Some(lite::Anchor::Absolute(start)) = range.start { bounds.frames.floor = bounds.frames.floor.max(start.group); } - if let Some(lite::Anchor::Absolute(start)) = range.start - && start.frame != 0 - { - let index = bounds.ranges.len() as u64; - bounds.positions.insert(index, (start.group, start.frame)); - } + let position = match range.start { + Some(lite::Anchor::Absolute(start)) if start.frame != 0 => Some((start.group, start.frame)), + _ => None, + }; let end = match range.end { None => None, // A mid-group end includes its group, up to the frame before it. @@ -4684,6 +4682,13 @@ impl Bounds { }, Some(end) => Some(anchor(end)), }; + // One ending before the group it starts partway through covers nothing, so it starts + // no group partway. + if let Some((group, frame)) = position + && !matches!(end, Some(Anchor::Time(end)) if end < group) + { + bounds.positions.insert(bounds.ranges.len() as u64, (group, frame)); + } bounds.ranges.push(Range { start, end }); } bounds.frames.first = first_frames(bounds.positions.values(), &bounds.ranges); @@ -7184,6 +7189,20 @@ mod serve_group_test { assert_eq!((disjoint.frames.start(7), disjoint.frames.start(9)), (4, 2)); assert_eq!(disjoint.frames.start(8), 0); + // A range ending before the group it starts partway through covers nothing, so another + // range covering that group asks for it whole. + let empty = Bounds::new( + &[ + lite::Range { + start: at(7, 4), + end: time(6), + }, + lite::Range { start: None, end: None }, + ], + false, + ); + assert_eq!(empty.frames.start(7), 0); + let overlapping = Bounds::new( &[ lite::Range { diff --git a/rs/moq-net/src/model/track/demand.rs b/rs/moq-net/src/model/track/demand.rs index 35c4adeb8b7..4bc36908188 100644 --- a/rs/moq-net/src/model/track/demand.rs +++ b/rs/moq-net/src/model/track/demand.rs @@ -25,6 +25,8 @@ pub(super) struct Subscriptions { feed: usize, /// Where resuming subscribers resume; see [`Subscription::resume`]. resumes: BTreeMap, + /// How many subscribers resume, so the aggregate asks without summing [`Self::resumes`]. + resuming: usize, /// The lowest group consumers want while no refresh is pending, never lowered. wanted: u64, /// A request for a group past every key written, until a group answers it or its @@ -113,7 +115,7 @@ impl Subscriptions { }, self.feed != 0, // A subscriber asking for the whole group outweighs any resuming one. - match self.resumes.values().sum::() == self.count { + match self.resuming == self.count { true => self.resumes.first_key_value().map(|(resume, _)| *resume), false => None, }, @@ -287,6 +289,7 @@ impl Subscriptions { self.feed += usize::from(new.feed); if let Some(resume) = new.resume { add(&mut self.resumes, resume); + self.resuming += 1; } self.count += 1; } @@ -303,6 +306,7 @@ impl Subscriptions { self.feed -= usize::from(old.feed); if let Some(resume) = old.resume { remove(&mut self.resumes, resume); + self.resuming -= 1; } self.count -= 1; } From 6044a0bb15876e05ec3df28edc6c327e7c8ec36a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:56:54 -0800 Subject: [PATCH 18/21] doc(lite): a subscription reads whole Groups but for a Position start Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index d8304bf36f8..69d2ec63a89 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -157,7 +157,7 @@ The duration before an incomplete group is dropped is determined by the applicat Every subscription is scoped to a single Track. A subscription starts at a configurable Group (defaulting to the latest) and continues until a configurable end Group or until either the publisher or subscriber cancels the subscription. -A subscription reads whole Groups; a FETCH reads one Group, or a slice of one (see [Positions](#positions)). +A subscription reads whole Groups, but may start partway through one at a Position; a FETCH reads one Group, or a slice of one (see [Positions](#positions)). The subscriber and publisher both indicate their delivery preference: - `Priority` indicates if Track A should be transmitted instead of Track B. @@ -2337,7 +2337,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - Added `Group Order` to SUBSCRIBE and SUBSCRIBE_UPDATE: descending (0x0, the previous fixed order) or ascending (0x1). - Encoded `Subscriber Max Age` in SUBSCRIBE and SUBSCRIBE_UPDATE as milliseconds plus one, with zero meaning no limit. - Added an optional `Subscriber Min Age` to SUBSCRIBE and SUBSCRIBE_UPDATE, the subscriber's target latency, encoded as milliseconds plus one with zero meaning not stated. A Range joining at the latest group starts that far behind the live edge, from the groups the publisher holds. A relay forwards the largest stated value. -- FETCH reads one Group or a slice of one, and SUBSCRIBE reads whole Groups: a read starting partway through a Group FETCHes it. A publisher holding only part of a Group a subscription covers drops it. +- FETCH reads one Group or a slice of one, and SUBSCRIBE reads whole Groups but for a Position `Start`: any other read of part of a Group FETCHes it. A publisher holding only part of a Group a subscription covers drops it. - A SUBSCRIBE_UPDATE that lowers a `Start` covers the groups below the old start that the publisher still holds, not only those that arrive later. A group already sent is not sent again. - Replaced SUBSCRIBE_OK `Group` with a `Range Start` per Range: the Group where that Range's delivery starts, including at a Relative or Time Anchor, from which its Groups are owed. - Each Range takes an implicit Range ID, and SUBSCRIBE_UPDATE edits the live Ranges instead of restating them: `Remove Count` Range IDs to retire, then the Ranges to add. The live Ranges must stay non-empty and fit in one SUBSCRIBE. A Range resolves its Anchors once, however many updates leave it live. From abff800b0ee87b4d8f1e7b514f4d46f5806910ae Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:45:23 -0800 Subject: [PATCH 19/21] fix(net): a drop keeps the head lent for its group; only a skip releases it A stale drop can be followed by the group arriving partway through, which still takes the loan. A loan never taken is reclaimed by the pool. Co-Authored-By: Claude Opus 5.5 --- quest/m4/js-route-resume.md | 2 +- rs/moq-net/src/model/track.rs | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/quest/m4/js-route-resume.md b/quest/m4/js-route-resume.md index 2e4085207ea..33d0b1ae5bf 100644 --- a/quest/m4/js-route-resume.md +++ b/quest/m4/js-route-resume.md @@ -7,7 +7,7 @@ The new copy's cache still holds the group whole for a reader that joins after t ## Plan -- Rust's shape to mirror: `Reader::held_at` and `resume_at` pick the position, `Producer::subscription` returns a `Resumed` that lends the head with the demand, `Subscription::resume` aggregates only when every reader resumes, and the copy's track keeps one loan per group (`TrackState::lent`), bounded by `group::MAX_CACHE_BYTES`, which the group takes as it arrives partway through. +- Rust's shape to mirror: `Reader::held_at` and `resume_at` pick the position, `Producer::subscribe` lends the head with the demand, `Subscription::resume` aggregates only when every reader resumes, and the copy's track keeps one loan per group (`TrackState::lent`), billed to the cache pool and reclaimed like a salvaged head, which the group takes as it arrives partway through and only a skip releases early. - JS has no cross-copy head adoption in its cache today, so the loan needs that first: a group arriving partway through takes the frames lent for it before any reader sees it. - The wire already works in JS: the publisher serves a Position start (`positionFrames`), and `encodeSpan` writes one when a span's `startFrame` is set; only the reader side never asks for one. - Port Rust's tests: `a_switch_resumes_the_newest_group_past_the_frames_held`, `a_resumed_copy_holds_the_group_whole_for_a_later_reader`, `a_query_resuming_a_new_copy_lends_it_the_head`, and the loan tests in `model/track.rs`. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index bf681add94c..751805f37bc 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1713,8 +1713,9 @@ impl TrackState { /// Keep the change index aligned with the range map, including the neighbours an /// insertion coalesces and the two pieces a removal can leave. fn change_drops(&mut self, groups: ops::Range, hole: Option) { - // A group given up never arrives to take the head lent for it. - if hole.is_some() { + // A group skipped never arrives to take the head lent for it. A drop, stale or not, can + // still be followed by the group, so its loan stands until the pool reclaims it. + if hole == Some(Hole::Skip) { let lent = self.lent.get_mut().unwrap_or_else(PoisonError::into_inner); lent.retain(|key| !groups.contains(&key)); } @@ -16507,6 +16508,31 @@ mod test { drop((late, wider)); } + /// A drop is not final: a group arriving after one still takes the head lent for it. + #[test] + #[cfg(feature = "async")] + fn a_stale_drop_keeps_the_loan() { + let producer = track_producer("test", Info::timed()); + let consumer = producer.consume(); + let mut prefix = group::Info { start: 7 }.produce(); + for _ in 0..2 { + prefix + .write_frame(Timestamp::from_millis(7).unwrap(), bytes::Bytes::new()) + .unwrap(); + } + consumer.lend_head(7, prefix.consume().handoff().head().unwrap()); + producer.give_up(7..8, Hole::stale(Duration::from_millis(1))); + let Received::Group(late) = producer.create_group_at(group::Info { start: 7 }, 2).unwrap() else { + panic!("a stale drop does not refuse the group"); + }; + assert_eq!( + consumer.peek_group(7).expect("cached").first_frame(), + 0, + "whole once visible" + ); + drop(late); + } + /// A copy lent the heads of two groups its readers resumed it at holds each whole as it /// arrives partway through, from the moment a reader can see it. #[test] From 5c81f26fabe314f172c4188b6a60e67ed4baf615 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:05:06 -0800 Subject: [PATCH 20/21] fix(net): only an adopting arrival takes the loan, and a reader's ranges and resume share one lock An arrival the lent head does not reach leaves the loan for a wider one, so the group's frame 0 is not lost. Reading each reader's ranges and resume under one lock keeps a reader moving on between them from asking a group whole that it resumes past. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 37 ++++++++++++++++------ rs/moq-net/src/model/track.rs | 56 +++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 5c2f7423329..cfb4c06ada5 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -207,13 +207,17 @@ impl Producer { return (demand, None); }; let mut ranges = Vec::new(); + let mut heads = Vec::with_capacity(readers.len()); for reader in &readers { + // Ranges and resume under one lock, so a reader moving on between them cannot ask + // a range from one group and resume past another. let mut reader = reader.lock().unwrap_or_else(PoisonError::into_inner); let preferences: Vec<_> = reader.subscription.read().ranges().collect(); ranges.extend(reader.resumed(&preferences).into_iter().map(|(_, range)| range)); + heads.push(reader.held_at(reader.newest)); } demand.ranges = ranges; - let resume = resume_of(&readers); + let resume = resume_of(heads); demand.resume = resume.as_ref().map(|(resume, _)| *resume); (demand, resume.map(|(_, head)| head)) } @@ -241,12 +245,20 @@ impl Producer { ..Default::default() }; let mut ranges = Vec::new(); + let mut heads = Vec::with_capacity(readers.len()); for reader in &readers { - ranges.extend(fold.fold(reader, None).into_iter().map(|(_, range)| range)); + // Ranges and resume under one lock, as `asked` takes them. + let mut locked = reader.lock().unwrap_or_else(PoisonError::into_inner); + ranges.extend( + fold.fold_locked(reader, &mut locked, None) + .into_iter() + .map(|(_, range)| range), + ); + heads.push(locked.held_at(locked.newest)); } // Asked whole, it resumes where the readers got to, as a new copy is asked. An edit // keeps that resume, as a reader's own copy does. - let resume = resume_of(&readers); + let resume = resume_of(heads); if let Some((resume, head)) = &resume { held.sub.lend_head(resume.group, head.clone()); } @@ -1437,12 +1449,8 @@ impl Candidate { /// Where every reader resumes the newest group it handed out, past the frames a copy holds /// of it, the lowest of those and the frames; `None` unless every reader resumes, as in a /// track's own demand. -fn resume_of(readers: &[Arc>]) -> Option<(Position, Arc)> { - (readers.iter()) - .map(|reader| { - let reader = reader.lock().unwrap_or_else(PoisonError::into_inner); - reader.held_at(reader.newest) - }) +fn resume_of(held: Vec)>>) -> Option<(Position, Arc)> { + (held.into_iter()) .reduce(|a, b| a.zip(b).map(|(a, b)| if b.0 < a.0 { b } else { a })) .flatten() } @@ -1492,6 +1500,16 @@ impl Fold { /// each under the held key it now has. fn fold(&mut self, reader: &Arc>, keys: Option>) -> Vec<(u64, track::Range)> { let mut locked = reader.lock().unwrap_or_else(PoisonError::into_inner); + self.fold_locked(reader, &mut locked, keys) + } + + /// [`Self::fold`] with `reader` already locked as `locked`. + fn fold_locked( + &mut self, + reader: &Arc>, + locked: &mut Reader, + keys: Option>, + ) -> Vec<(u64, track::Range)> { let (preferences, revision) = { let current = locked.subscription.read(); let preferences: Vec<_> = match &keys { @@ -1504,7 +1522,6 @@ impl Fold { (preferences, current.revision) }; let resumed = locked.resumed(&preferences); - drop(locked); let folded = self .readers .entry(Arc::as_ptr(reader) as usize) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 751805f37bc..ead2ef5aac9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1308,6 +1308,11 @@ impl Lent { self.loans.remove(&key)?.head() } + /// The head lent for group `key`, left in place, unless its pool reclaimed it. + fn peek(&self, key: u64) -> Option> { + self.loans.get(&key)?.head() + } + fn retain(&mut self, keep: impl Fn(u64) -> bool) { self.loans.retain(|key, _| keep(*key)); } @@ -1319,6 +1324,11 @@ impl TrackState { self.lent.lock().unwrap_or_else(PoisonError::into_inner).take(key) } + /// The head lent for group `key`, left for an arrival that can adopt it. + fn peek_lent(&self, key: u64) -> Option> { + self.lent.lock().unwrap_or_else(PoisonError::into_inner).peek(key) + } + /// Hold `head` for group `key` until it arrives, or return the copies of it already /// cached, to adopt it with this state's lock released; see [`Consumer::lend_head`]. fn lend(&self, key: u64, head: &Arc) -> Vec { @@ -4622,10 +4632,18 @@ impl Producer { } // Before any reader can see it, so none takes it without the head (lock order is track // then group), settled once the track lock is released. A whole copy needs no loan, - // and drops it. - let adopted = match state.take_lent(info.start) { - Some(head) if start != 0 => group.handle().take_head(&head), - _ => None, + // and drops it; one the head does not reach leaves it for a wider arrival. + let adopted = if start == 0 { + state.take_lent(info.start); + None + } else { + let adopted = state + .peek_lent(info.start) + .and_then(|head| group.handle().take_head(&head)); + if adopted.is_some() { + state.take_lent(info.start); + } + adopted }; state.cap(&group.handle())?; state.commit_group(&group.handle()); @@ -16508,6 +16526,36 @@ mod test { drop((late, wider)); } + /// An arrival the head does not reach leaves the loan for a later, wider one. + #[test] + #[cfg(feature = "async")] + fn an_arrival_that_cannot_adopt_leaves_the_loan() { + let producer = track_producer("test", Info::timed()); + let consumer = producer.consume(); + let mut prefix = group::Info { start: 5 }.produce(); + prefix + .write_frame(Timestamp::from_millis(5).unwrap(), bytes::Bytes::new()) + .unwrap(); + consumer.lend_head(5, prefix.consume().handoff().head().unwrap()); + let Received::Group(late) = producer.create_group_at(group::Info { start: 5 }, 2).unwrap() else { + panic!("nothing else holds group 5"); + }; + assert_eq!( + consumer.peek_group(5).expect("cached").first_frame(), + 2, + "one frame does not reach frame 2" + ); + let Received::Group(wider) = producer.create_group_at(group::Info { start: 5 }, 1).unwrap() else { + panic!("a copy from frame 1 displaces one from frame 2"); + }; + assert_eq!( + consumer.peek_group(5).expect("cached").first_frame(), + 0, + "whole once visible" + ); + drop((late, wider)); + } + /// A drop is not final: a group arriving after one still takes the head lent for it. #[test] #[cfg(feature = "async")] From be8a3b5e3390a6c51a8383ff77b00b969b4d67ab Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:24:45 -0800 Subject: [PATCH 21/21] fix(net): fold each reader's resume in as it is read, keeping only the lowest head Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 47 +++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index cfb4c06ada5..c788c7cc20b 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -207,17 +207,17 @@ impl Producer { return (demand, None); }; let mut ranges = Vec::new(); - let mut heads = Vec::with_capacity(readers.len()); + let mut resume = Resume::default(); for reader in &readers { // Ranges and resume under one lock, so a reader moving on between them cannot ask // a range from one group and resume past another. let mut reader = reader.lock().unwrap_or_else(PoisonError::into_inner); let preferences: Vec<_> = reader.subscription.read().ranges().collect(); ranges.extend(reader.resumed(&preferences).into_iter().map(|(_, range)| range)); - heads.push(reader.held_at(reader.newest)); + resume.add(&reader); } demand.ranges = ranges; - let resume = resume_of(heads); + let resume = resume.lowest(); demand.resume = resume.as_ref().map(|(resume, _)| *resume); (demand, resume.map(|(_, head)| head)) } @@ -245,7 +245,7 @@ impl Producer { ..Default::default() }; let mut ranges = Vec::new(); - let mut heads = Vec::with_capacity(readers.len()); + let mut resume = Resume::default(); for reader in &readers { // Ranges and resume under one lock, as `asked` takes them. let mut locked = reader.lock().unwrap_or_else(PoisonError::into_inner); @@ -254,11 +254,11 @@ impl Producer { .into_iter() .map(|(_, range)| range), ); - heads.push(locked.held_at(locked.newest)); + resume.add(&locked); } // Asked whole, it resumes where the readers got to, as a new copy is asked. An edit // keeps that resume, as a reader's own copy does. - let resume = resume_of(heads); + let resume = resume.lowest(); if let Some((resume, head)) = &resume { held.sub.lend_head(resume.group, head.clone()); } @@ -1447,12 +1447,35 @@ impl Candidate { } /// Where every reader resumes the newest group it handed out, past the frames a copy holds -/// of it, the lowest of those and the frames; `None` unless every reader resumes, as in a -/// track's own demand. -fn resume_of(held: Vec)>>) -> Option<(Position, Arc)> { - (held.into_iter()) - .reduce(|a, b| a.zip(b).map(|(a, b)| if b.0 < a.0 { b } else { a })) - .flatten() +/// of it: the lowest of those and the frames, folded in one reader at a time so only the +/// lowest head is kept. `None` unless every reader resumes, as in a track's own demand. +#[derive(Default)] +enum Resume { + #[default] + Empty, + Lowest(Position, Arc), + /// A reader resumes nothing, so neither does the aggregate. + Whole, +} + +impl Resume { + fn add(&mut self, reader: &Reader) { + if matches!(self, Self::Whole) { + return; + } + *self = match (std::mem::take(self), reader.held_at(reader.newest)) { + (_, None) => Self::Whole, + (Self::Lowest(at, head), Some((next, _))) if at <= next => Self::Lowest(at, head), + (_, Some((next, head))) => Self::Lowest(next, head), + }; + } + + fn lowest(self) -> Option<(Position, Arc)> { + match self { + Self::Lowest(at, head) => Some((at, head)), + _ => None, + } + } } /// A subscription on a copy asking for its readers' demand, as [`Producer::subscription`]