From ac566af1e0a187e80569131d05f68f2d57bd1a91 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 20:10:52 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m2/time-addressing/group-end-whole.md Quest-Claim: 4603a13ba5d7adf4 From 9907b09cc2eda4e3a1656f5e62bafcc4869879cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 21:06:45 -0800 Subject: [PATCH 2/6] feat(net)!: lite-07's GROUP_END says only that a group is whole GROUP_END drops its End field and stays a zero Payload Length, so a Group Stream that FINs without it carries a closed, possibly partial group. group::Producer::finish_at(end) becomes finish_whole(), and group::Consumer::end() becomes is_whole(); JS mirrors them as finishWhole() and the whole getter. A whole group with no successor ages on the wall clock from one mean frame interval past its highest frame timestamp. moq-mux and hang finish every group they close whole. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 14 +-- doc/concept/standard.md | 2 +- doc/lib/rs/moq-net.md | 2 +- drafts/draft-lcurley-moq-lite.md | 27 +++-- js/hang/src/container/legacy.test.ts | 20 ++-- js/hang/src/container/legacy.ts | 17 +-- js/net/src/group.ts | 35 +++--- js/net/src/lite/group.test.ts | 26 ++--- js/net/src/lite/group.ts | 28 ++--- js/net/src/lite/publisher.test.ts | 22 ++-- js/net/src/lite/publisher.ts | 23 ++-- js/net/src/lite/subscriber.test.ts | 3 +- js/net/src/lite/subscriber.ts | 24 ++-- js/net/src/track.test.ts | 44 ++++++++ js/net/src/track.ts | 36 +++--- quest/m2/README.md | 2 +- quest/m2/time-addressing/branch-salvage.md | 2 +- quest/m2/time-addressing/group-end-whole.md | 83 -------------- quest/m2/time-addressing/start-placement.md | 6 +- quest/m3/groups-never-overlap.md | 1 - rs/moq-cli/src/play/media.rs | 4 +- rs/moq-e2ee/src/group.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 10 +- rs/moq-mux/src/container/producer.rs | 39 ++----- rs/moq-mux/src/container/ts/recovery_test.rs | 20 ++-- rs/moq-mux/src/start.rs | 9 -- rs/moq-mux/tests/legacy_audio.rs | 2 +- rs/moq-net/src/fuzz/ingest.rs | 23 ++-- rs/moq-net/src/ietf/publisher.rs | 7 +- rs/moq-net/src/lite/publisher.rs | 21 ++-- rs/moq-net/src/lite/subscriber.rs | 71 ++++++------ rs/moq-net/src/model/group.rs | 97 ++++++++-------- rs/moq-net/src/model/track.rs | 104 +++++++++++------- rs/moq-net/src/recv.rs | 32 ++---- rs/moq-net/tests/integration/group_end.rs | 33 +++--- rs/moq-net/tests/integration/partial_group.rs | 10 +- 36 files changed, 400 insertions(+), 501 deletions(-) delete mode 100644 quest/m2/time-addressing/group-end-whole.md diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 38a72e4ff00..47330d6f1e5 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -340,13 +340,13 @@ upstream as that ID, where a start selects the first group at or after it rather than the one containing it. moq-lite 01 and 02 carry no start, so every subscription there starts at the live group. -A group ends one of three ways: finished at the time it ends, so it is whole; -closed, so every frame written arrives but it may have held more, as when a -relay stops sending a group that went stale; or aborted with an error. A -finished group's end also bounds how long it stays fresh, so the last group of -a paused track still ages out. Over the work-in-progress moq-lite 07 each group -also names the start of the group before it, so a reader sees a gap at once; -older versions and moq-transport carry neither the end nor the previous start. +A group ends one of three ways: finished, so it is whole; closed, so every +frame written arrives but it may have held more, as when a relay stops sending +a group that went stale; or aborted with an error. A finished group with no +successor ages from one frame past its highest frame timestamp, so the last +group of a paused track still ages out. Over the work-in-progress moq-lite 07 each group also +names the start of the group before it, so a reader sees a gap at once; older +moq-lite versions say neither whether a group is whole nor the previous start. Order only decides what goes first, never what is skipped, so catching up oldest first needs a max delay that covers the backlog. When a relay serves subscribers of diff --git a/doc/concept/standard.md b/doc/concept/standard.md index dda3509b480..71e43f2e703 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -44,7 +44,7 @@ subset drops. What a peer actually observes against this implementation: - **One `SETUP` credential**, forwarded to the [auth server](/bin/relay/auth#the-contract) unverified. A second `SETUP` token closes the session. We advertise no token cache, so an alias reference in `SETUP` closes it with `PROTOCOL_VIOLATION`, one on a request with `UNKNOWN_AUTH_TOKEN_ALIAS`, and a registration on a request with `AUTH_TOKEN_CACHE_OVERFLOW`. Where the Rust side authorizes the peer or verifies tokens, each distinct request token is verified once while any open request on that session names it, and its grant authorizes only requests naming it, even where that authorization already covers it; it never widens the session. It is refused with `NOT_SUPPORTED` when nothing verifies it, when it is empty, or when its Token Type is anything but 0 (out of band). A side that does neither, JavaScript included, ignores the token. Rust refuses a request carrying more than one distinct token with `NOT_SUPPORTED`, since it verifies one; JavaScript, which verifies none, ignores them all. A token-authorized `SUBSCRIBE` still stays within the grant the peer gave us. A token-authorized `PUBLISH_NAMESPACE` needs a grant covering everything beneath the namespace, and the tracks we subscribe under it rely on that announce instead of our limit, ending with it. An update on a token-authorized request moves it to the token it carries; the old token ends with its last naming request. A request is admitted by the session or by a token for good, so an update naming a token on one the session admitted is refused with `NOT_SUPPORTED`. A token-less `PUBLISH_NAMESPACE` our limit does not cover is refused `UNAUTHORIZED`, and one the limit stops covering ends; the publisher re-announces once it widens. Expiry or revocation ends every request naming that token. The relay reports the lease end to its auth server, and a later request naming the token verifies it again. A refused renewal ends only that request. Rust and JavaScript send the token a route names (`Route::auth`) on its `PUBLISH_NAMESPACE`, and swapping it renews the request with `REQUEST_UPDATE`; the inline `NAMESPACE` that answers a solicited `SUBSCRIBE_NAMESPACE` carries no token. A token added to a reconnecting connection's auth handle rides every session it makes, so the announcement keeps it across a reconnect; one added to a single session's handle ends with that session. Rust honors a token on any supported request; discovery reveals only paths it grants, and a joining `FETCH` keeps following its subscription. A request repeating a token is refused with `MALFORMED_AUTH_TOKEN`. Additional session tokens in band need [moq-lite-07](/concept/moq-lite#authorization); moq-transport has no AUTH exchange. - **Pause delivery.** `FORWARD=0` on a `SUBSCRIBE` or its update drops the subscription, closing its data streams and holding nothing upstream, while the request and its slot stay. `FORWARD=1` subscribes again as a new `SUBSCRIBE` with the same filter would, so nothing carries across the pause: its `REQUEST_OK` names the live edge it joined at. A paused update may change only priority. - **Refused, not fatal.** A legal request this stack does not serve is rejected on its own and the session stays up: `SUBSCRIBE_TRACKS`, any `FETCH` to JavaScript, multi-group `FETCH`, and joining requests reaching before the joined group get `NOT_SUPPORTED`. Range Filters get `INVALID_FILTER` from draft 20 because the advertised filter limit is zero. From draft 20, namespaces whose first tuple field is exactly `.` or `.session` get `DOES_NOT_EXIST` before application lookup; other dot prefixes still reach the application. Subscription updates accept priority and delivery timeouts. Rust also carries range changes, keeps bounded subscriptions updatable, renews request tokens, and judges `NEW_GROUP_REQUEST` as a join (see dynamic groups below). From draft 20, a wider or historical fill fails on its separate fetch stream while its subscription stays open; a largest-group fill hands its head to the live subgroup tail. Other supported-but-unimplemented preferences end that subscription; a `TRACK_PROPERTY_FILTER` there closes the session with `PROTOCOL_VIOLATION`, since it belongs only to `SUBSCRIBE_TRACKS`. A namespace update may change its prefix; suffixes after the acknowledgement are relative to that new prefix. -- **Objects and datagrams.** Only subgroup 0 is accepted: a stream for any other subgroup, whether its header gives the ID or takes it from the first Object ID, is stopped and the session stays up. Rust relays preserve Object IDs, gaps, object properties, FIRST\_OBJECT, and END\_OF\_GROUP status objects between moq-transport peers for subgroup 0, including across a moq-lite-07 hop between two relays that negotiate the [ietf](/draft/moq-ietf) extension. Plain Rust readers and moq-lite peers without the extension see consecutive frames. moq-transport maps a [datagram track](/concept/moq-lite#datagrams) to `SUBSCRIBE` and `OBJECT_DATAGRAM`, with its sequence as the group ID and object ID 0. One upstream subscription serves both kinds of a name. Stream consumers receive only stream objects, and datagram consumers receive only object 0 datagrams, including through relays. An undefined Object Status or malformed datagram framing closes the session. Unreadable object properties drop only that datagram. Our own groups end with an END\_OF\_GROUP status object only when the group finished whole, and never claim their end in the subgroup header. JavaScript has one group end, `finish()`, which counts as whole, so a JavaScript forwarder also claims the end of a group a third party ended with a plain FIN; Rust keeps that group unclaimed. Their object 0 carries a Prior Group ID Gap naming the keys the publisher skipped just below (`append_group` or `skip_groups`), a best-effort hint that a peer may ignore; a fetch or datagram carries none. A received Prior Group ID Gap on a subgroup stream skips those keys on the track, and stays among the object's properties. Property 0x06 stays opaque; it is dropped between draft 18 and later drafts, or refused when immutable, because draft 20 assigns it to SUBGROUP\_DELIVERY\_TIMEOUT. A status object carrying properties is refused. A `FETCH` has no status objects, so its response ends at the last frame. A group cannot resume across route changes by frame ordinal once Object IDs skip ahead of the ordinals on either route; recovery refuses it instead of splicing different objects. +- **Objects and datagrams.** Only subgroup 0 is accepted: a stream for any other subgroup, whether its header gives the ID or takes it from the first Object ID, is stopped and the session stays up. Rust relays preserve Object IDs, gaps, object properties, FIRST\_OBJECT, and END\_OF\_GROUP status objects between moq-transport peers for subgroup 0, including across a moq-lite-07 hop between two relays that negotiate the [ietf](/draft/moq-ietf) extension. Plain Rust readers and moq-lite peers without the extension see consecutive frames. moq-transport maps a [datagram track](/concept/moq-lite#datagrams) to `SUBSCRIBE` and `OBJECT_DATAGRAM`, with its sequence as the group ID and object ID 0. One upstream subscription serves both kinds of a name. Stream consumers receive only stream objects, and datagram consumers receive only object 0 datagrams, including through relays. An undefined Object Status or malformed datagram framing closes the session. Unreadable object properties drop only that datagram. Our own groups end with an END\_OF\_GROUP status object only when the group finished whole, and never claim their end in the subgroup header. JavaScript's moq-transport publisher counts any clean group end as whole, `finish()` as well as `finishWhole()`, so a JavaScript forwarder also claims the end of a group a third party ended with a plain FIN; Rust keeps that group unclaimed. Their object 0 carries a Prior Group ID Gap naming the keys the publisher skipped just below (`append_group` or `skip_groups`), a best-effort hint that a peer may ignore; a fetch or datagram carries none. A received Prior Group ID Gap on a subgroup stream skips those keys on the track, and stays among the object's properties. Property 0x06 stays opaque; it is dropped between draft 18 and later drafts, or refused when immutable, because draft 20 assigns it to SUBGROUP\_DELIVERY\_TIMEOUT. A status object carrying properties is refused. A `FETCH` has no status objects, so its response ends at the last frame. A group cannot resume across route changes by frame ordinal once Object IDs skip ahead of the ordinals on either route; recovery refuses it instead of splicing different objects. - **Dynamic groups.** A track advertises `DYNAMIC_GROUPS` when its publisher sets a refresh ratio, and a track received with `DYNAMIC_GROUPS` gets a ratio of 0. A `NEW_GROUP_REQUEST` counts as one more subscriber joining, which a `SUBSCRIBE` already is: it asks for a new group only when the newest group's bytes after its first frame exceed the ratio times the first frame. Joins coalesce until the asked-for group arrives. A Rust relay sends `NEW_GROUP_REQUEST` upstream, on the `SUBSCRIBE` or in a `REQUEST_UPDATE`, once a join wants a group past the newest it holds; a relay holding nothing yet asks for none. On a track without a ratio, an update carrying one ends the subscription with `UPDATE_FAILED`. A moq-lite-07 hop (`moq-lite-07-wip`, opt-in) carries the same demand natively: the ratio in `TRACK_INFO` and the asked-for group in `SUBSCRIBE_UPDATE`. Earlier moq-lite versions carry neither, so a track received over one never refreshes. - **Priority.** Higher is served first. This stack's priorities run from -128 to 127 with 0 unset, and go out on the IETF wire, which ranks lower first, as `127 - priority`: an unset priority is IETF 127. An omitted subscriber or publisher priority reads as the drafts' default of IETF 128, one step below unset, so a relay forwards it unchanged. A relay's upstream subscription carries the highest priority its subscribers ask for and updates it as they come and go; its group order stays as the `SUBSCRIBE` asked, since no draft lets an update change it. - **Size.** An object extension block larger than 64 KiB ends that subgroup stream. The session stays up. This cap is ours, not the draft's. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 3e36e0a09c4..b15f2819940 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -25,7 +25,7 @@ What you use it for, beyond what the concept page already describes: - **One track per name.** Concurrent subscriptions share one request and producer. A group is keyed by its start, its first frame's timestamp in the track's timescale, and starts only rise. An untimed track has no timescale, so its frames carry `Timestamp::unscaled(key)`, the group's key, and a timestamp's `scale()` is `None`: conversions such as `as_millis()` fail on it, and it compares with no scaled timestamp. Every frame write takes a `Timestamp`, and one that doesn't match its track is refused with `TimestampMismatch`. `append_group(start)` opens the next one at a `Timestamp`, converted into the track's timescale as a frame's is (`Timestamp::unscaled(key)` on an untimed track), declaring the range below it skipped; `create_group` places one without declaring anything. The broadcast keeps nothing of an ended track, and a name always means the same content, so a producer that replaces it starts its groups past its predecessor's. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Keep a track out of listings.** `broadcast.reserve_unlisted_track(name)` reserves a track no track listing names, so only a subscriber who already knows the name reads it. It hides the track from discovery and authorizes nothing: anyone allowed to subscribe to the broadcast can read it by name. A producer you hand out can still be ended on your own shutdown with `producer.abort_handle()`, which aborts the track without keeping it open. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. -- **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track: `finish(self)` consumes the producer and returns a `track::Finished`, and `fail(err)` does the same with an error that readers get after every group below the end. A finished track can still take an older group below the end, be read, or be aborted, but not append. Hold it, and the broadcast stays announced and keeps serving the track until you close it or drop its route; dropping the last handle releases the track, so new readers no longer find it. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. Ending a group returns a `group::Finished`, which can be read or aborted to release its cache: `finish_at(end)` ends it whole where its content stops, so a reader can tell the group starting there continues it, and `finish()` ends it without one. +- **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track: `finish(self)` consumes the producer and returns a `track::Finished`, and `fail(err)` does the same with an error that readers get after every group below the end. A finished track can still take an older group below the end, be read, or be aborted, but not append. Hold it, and the broadcast stays announced and keeps serving the track until you close it or drop its route; dropping the last handle releases the track, so new readers no longer find it. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. Ending a group returns a `group::Finished`, which can be read or aborted to release its cache: `finish_whole()` ends it whole, holding every frame it will ever have, and `finish()` only closes it, as a relay does with a group it holds part of. The last group of a track, finished whole, ages out one frame past its highest frame timestamp, a frame being its mean frame interval; a whole group of one frame waits for a successor. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. An on-demand publisher ends a track nobody wants with `abort_unwanted`, which keeps it for a reader that joined since, rather than by dropping its producer. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. - **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline from the connection's arrival (30 s by default, `with_timeout`), across your transport handshake (`Server::deadline`), SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 69d2ec63a89..0bcda5fbcec 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -182,7 +182,7 @@ A publisher never creates a Group with start 2^64-1, so every span of starts end A Group ends in one of three ways: -- Finished: a [GROUP_END](#group-end) names where it ends, so it is whole. +- Finished: a [GROUP_END](#group-end) says it is whole. - Closed: its stream ends (FIN) without a GROUP_END, so every Frame sent on it arrived, but the Group may have held more. - Aborted: its stream is reset. @@ -810,12 +810,12 @@ A FETCH can additionally carry `Subscriber Max Age`; it can tighten, but never r It is not crucial to aggressively expire groups thanks to [prioritization](#prioritization), but a lower priority group still consumes RAM, bandwidth, and potentially flow control. It is RECOMMENDED that an application set conservative limits and only resort to expiration when data is absolutely no longer needed. -A group is never expired until its **successor** has arrived: the next group by start that has arrived at this hop and was not reset. -The latest group has no successor, so it is never expired. +A group is never expired until its **successor** has arrived, the next group by start that has arrived at this hop and was not reset, or until it finished (see [GROUP_END](#group-end)). +The latest group, while it has not finished, has no successor, so it is never expired. A group is expired once either of two ages meets the relevant `Max Age`: - Its **timestamp age** is the difference between the *newest* frame timestamp of the latest group that has at least one frame, and this group's **reach**, defined below. -- Its **arrival age** is the time elapsed since its successor arrived at this hop. +- Its **arrival age** is the time elapsed since its successor arrived at this hop. A finished group with no successor ages from its own reach instead: the time elapsed since it arrived at this hop, less the span from its first frame timestamp to its reach. On a scale-less Track, timestamp age counts bare units as the milliseconds of a Max Age. @@ -825,10 +825,13 @@ Being behind is not itself a reason to expire anything. [Prioritization](#prioritization) already transmits newer groups first, so an older group consumes only whatever capacity is left over; it therefore closes the gap faster than the live edge advances, and a receiver that is behind can converge without losing content. What cannot be recovered is a group with nothing left worth delivering, and that is what expiration removes. -A group's **reach** is its own end once it finished (see [GROUP_END](#group-end)), and otherwise the first frame timestamp of the next group by start, or unbounded when no later group has presented a frame. -A group cannot present past its end or where its successor begins, so this is the furthest it could still reach. -Its frames alone do not bound it: frame durations are not carried on the wire, so a group's last frame timestamp is where that frame *starts* presenting, not where the group ends. -An unfinished group whose successor has not presented a frame therefore has no timestamp age, because nothing yet proves where it stops. +A group's **reach** is the first frame timestamp of the next group by start, and otherwise, once it finished, one frame past its own highest frame timestamp, or unbounded when neither exists. +A group cannot present past where its successor begins, so this is the furthest it could still reach. +A finished group with no successor holds every frame it will ever have, so its own frames stand in. +Frame durations are not carried on the wire, so its highest frame timestamp is where its last frame *starts* presenting; one frame is its mean interval instead, the span from its first frame timestamp to its highest divided by one less than its frame count. +A finished group whose frames measure no interval, one frame or all at one timestamp, has no reach without a successor, so it never expires before one arrives. +A closed group may have held more frames than arrived, so its frames do not bound it. +An unfinished or closed group whose successor has not presented a frame therefore has no timestamp age, because nothing yet proves where it stops. The group itself needs no timestamp: a zero-frame group (a keep-alive or gap marker) is bounded by its successor's first frame the same way. Reach is an exclusive bound, so a timestamp age **equal** to Max Age already expires the group: the freshest frame it could still hold sits strictly below its reach, and is therefore strictly older than the budget. @@ -2236,15 +2239,11 @@ It shares FRAME's leading field, reserving the `Payload Length` value 0 that a F ~~~ GROUP_END Message { Payload Length (i) = 0 - End (i) } ~~~ -**End**: -The Group's exclusive end: the timestamp where it stops presenting, in the Track's `Timescale`, or a bare number on a scale-less Track. -It is greater than every frame timestamp in the Group and no greater than the next Group's start; a receiver treats an `End` at or below the Group's start as a malformed Track. - A Group that ends with a GROUP_END is whole: every frame it will ever hold has been sent. +It carries no end position: where a Group stops presenting is the application's to say, in its frames. The publisher FINs the stream right after it, and a receiver treats any byte that follows as a malformed Track. A stream that FINs without a GROUP_END carries a closed Group, which may have held more frames than arrived (see [Group](#group)). @@ -2363,7 +2362,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - Added a leading Type to datagrams, 0x0 for a Track Datagram; a receiver discards one of an unknown Type. - FRAME and datagrams always carry a timestamp, a bare number on a scale-less Track. - Replaced GROUP's `Group Sequence` with `Group Start`, added `Previous Start`, the start of the Group before it, so a subscriber sees the gap before a Group without a SUBSCRIBE_SKIP. -- Added GROUP_END, a `Payload Length` of 0 followed by the Group's exclusive end: a Group or Fetch Stream ends with it when its Group finished, a FIN alone ends a closed Group whose frames may not all exist, and a reset aborts. A Group's own end bounds its reach. +- Added GROUP_END, a `Payload Length` of 0: a Group or Fetch Stream ends with it when its Group finished whole, a FIN alone ends a closed Group whose frames may not all exist, and a reset aborts. A finished Group with no successor expires one mean frame interval past its own highest frame timestamp. - Expiration also measures a group's arrival age, the time since its successor arrived at this hop. - A relay closes a Group it holds only the start of, without a GROUP_END. A publisher resets the Group Stream of a group that expires with OLD, even one whose GROUP_END it sent but is not yet acknowledged, and names the group in a SUBSCRIBE_DROP with OLD. - FRAME leads with `Payload Length`, the payload size plus one, in place of the trailing `Message Length`; 0 is malformed. diff --git a/js/hang/src/container/legacy.test.ts b/js/hang/src/container/legacy.test.ts index 90665100868..436b7de25e5 100644 --- a/js/hang/src/container/legacy.test.ts +++ b/js/hang/src/container/legacy.test.ts @@ -18,7 +18,7 @@ function track(scale: Time.Timescale): Track.Producer { // Every group the producer wrote: its start, whole end, and each frame's wire timestamp. async function groups(subscriber: Track.Subscriber) { - const out: { start: bigint; end?: bigint; frames: number[] }[] = []; + const out: { start: bigint; whole: boolean; frames: number[] }[] = []; for (;;) { const group = await subscriber.recvGroup(); if (!group) return out; @@ -28,7 +28,7 @@ async function groups(subscriber: Track.Subscriber) { if (!frame) break; frames.push(Varint.decode(frame.payload)[0]); } - out.push({ start: group.start, end: group.end, frames }); + out.push({ start: group.start, whole: group.whole, frames }); } } @@ -45,11 +45,11 @@ test("Legacy Producer keys groups in a millisecond track's timescale", async () producer.finish(); expect(await groups(subscriber)).toEqual([ - // Its end marker at the next keyframe, which it ends whole at. - { start: 1n, end: 34n, frames: [1_400, 1_900, 34_700] }, - { start: 34n, end: 68n, frames: [34_700, 68_100] }, + // Its end marker at the next keyframe. + { start: 1n, whole: true, frames: [1_400, 1_900, 34_700] }, + { start: 34n, whole: true, frames: [34_700, 68_100] }, // Not below the newest frame's millisecond, so the marker starts there. - { start: 68n, end: undefined, frames: [68_000] }, + { start: 68n, whole: true, frames: [68_000] }, ]); }); @@ -63,10 +63,10 @@ test("Legacy Producer keys groups in a nanosecond track's timescale", async () = producer.finish(); expect(await groups(subscriber)).toEqual([ - { start: 1_000_000n, end: 2_000_000n, frames: [1_000, 2_000] }, - // Its end marker at 3ms, then whole at the marker one nanosecond before it. - { start: 2_000_000n, end: 2_999_999n, frames: [2_000, 3_000] }, - { start: 2_999_999n, end: undefined, frames: [2_999] }, + { start: 1_000_000n, whole: true, frames: [1_000, 2_000] }, + // Its end marker at 3ms, then the marker one nanosecond before it. + { start: 2_000_000n, whole: true, frames: [2_000, 3_000] }, + { start: 2_999_999n, whole: true, frames: [2_999] }, ]); }); diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index 64d8e2faed9..2352e774418 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -244,8 +244,7 @@ export class Producer { // keyOf vouched for the timescale. marker = new Time.Timestamp(Number(start), this.#track.timescale as Time.Timescale); } - // The open group ends whole at the marker, so a reader can tell the marker follows it. - this.#close(end, marker); + this.#close(end); this.#interval = undefined; if (marker === undefined) return; @@ -259,14 +258,14 @@ export class Producer { : encodeFrame(new Uint8Array(), Duration.micros(timestamp)), timestamp: marker, }); - group.finish(); + group.finishWhole(); this.#liveEdge = this.#liveEdge === undefined ? timestamp : Math.max(this.#liveEdge, timestamp); this.#marked = true; } - // Flush and close the current group at the supplied or estimated end timestamp, whole at the - // `whole` marker start when one follows it. - #close(end?: number, whole?: Time.Timestamp) { + // Flush the current group's end marker at the supplied or estimated end timestamp, then finish + // it whole, since it holds every frame written. + #close(end?: number) { if (!this.#group) return; if ( this.#format.kind === "video" && @@ -281,11 +280,7 @@ export class Producer { // A presentation endpoint cannot bound the decode-order tail after reordering. const bound = this.#reordered ? undefined : end; this.#format.finishGroup(this.#group, bound === undefined ? undefined : Duration.micros(bound)); - // A group ends whole where its content stops, which bounds its reach without a successor. - const at = - whole !== undefined ? BigInt(whole.value) : bound === undefined ? undefined : keyOf(this.#track, bound); - if (at !== undefined && at > this.#group.start) this.#group.finishAt(at); - else this.#group.finish(); + this.#group.finishWhole(); this.#group = undefined; if (this.#end !== undefined) { this.#liveEdge = this.#liveEdge === undefined ? this.#end : Math.max(this.#liveEdge, this.#end); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 06422451f92..a6c062fbe46 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -94,9 +94,9 @@ class GroupState { readonly start: bigint; frames = new Signal(new Queue()); closed = new Once(); - // Where the group ends, in its track's timescale, once its producer finished it whole: set - // before a clean `closed`, and undefined when it only closed. - end?: bigint; + // Whether its producer finished the group whole: set before a clean `closed`, and false + // when it only closed. + whole = false; total = new Signal(0); // The total number of frames in the group thus far // Index of the frame at the front of `frames`, advanced by a read: the index the next @@ -466,7 +466,7 @@ export class Producer { if (dst.closed.peek() !== undefined) return makeConsumer(dst); const closed = this.#state.closed.peek(); if (closed !== undefined) { - dst.end = this.#state.end; + dst.whole = this.#state.whole; dst.closed.set(closed); return makeConsumer(dst); } @@ -712,8 +712,10 @@ export class Producer { } /** - * Ends the group without saying where it ends, returning it as {@link Finished}: every frame - * written reaches its readers, but they can't tell the group is whole. Idempotent, as the + * Closes the group without saying it is whole, returning it as {@link Finished}: every frame + * written reaches its readers, but they can't tell whether frames are missing, so failover + * may continue it from another copy. How a relay ends a group that expired or arrived + * partial; a producer that wrote every frame calls {@link finishWhole}. Idempotent, as the * Rust `group::Producer::finish`. */ finish(): Finished { @@ -722,13 +724,12 @@ export class Producer { } /** - * Ends the group whole at `end`, the exclusive end of its presentation in the track's - * timescale, returning it as {@link Finished}, as the Rust `group::Producer::finish_at`. An - * `end` at or before the group's start throws {@link TimestampMismatch}. + * Ends the group whole, returning it as {@link Finished}: it holds every frame it will ever + * have. A whole group with no successor ages from one mean frame interval past its highest + * frame timestamp, as the Rust `group::Producer::finish_whole`. */ - finishAt(end: bigint): Finished { - if (end <= this.start) throw new TimestampMismatch(); - if (this.#state.closed.peek() === undefined) this.#state.end = end; + finishWhole(): Finished { + if (this.#state.closed.peek() === undefined) this.#state.whole = true; this.#end(null); return this; } @@ -780,7 +781,7 @@ export class Producer { if (outcome === undefined) return; for (const mirror of this.#mirrors?.keys() ?? []) { if (mirror.closed.peek() === undefined) { - if (outcome === null) mirror.end = this.#state.end; + if (outcome === null) mirror.whole = this.#state.whole; mirror.closed.set(outcome); } this.#unmirror(mirror); @@ -1022,11 +1023,11 @@ export class Consumer { } /** - * Where the group ends in its track's timescale, once its producer finished it whole with - * {@link Producer.finishAt}'s `end`: undefined while it is open, or when it ended without one. + * Whether its producer finished the group whole with {@link Producer.finishWhole}: false + * while it is open, aborted, or only closed. */ - get end(): bigint | undefined { - return this.#state.closed.peek() === null ? this.#state.end : undefined; + get whole(): boolean { + return this.#state.closed.peek() === null && this.#state.whole; } /** diff --git a/js/net/src/lite/group.test.ts b/js/net/src/lite/group.test.ts index 9b010d96235..a9a6ddb2408 100644 --- a/js/net/src/lite/group.test.ts +++ b/js/net/src/lite/group.test.ts @@ -151,19 +151,19 @@ function lite07(...values: (number | bigint)[]): Uint8Array { } // What reading `bytes` as group `start`'s lite-07 frames resolves with, or the error it throws. -async function readLite07(bytes: Uint8Array, start = 0n, scale = SCALE): Promise { +async function readLite07(bytes: Uint8Array, start = 0n, scale = SCALE): Promise { const producer = new Producer(start); return readFrames(streamOf([bytes], Version.DRAFT_07), producer, { scale, lengthFirst: true }).catch( (err: unknown) => err, ); } -// A lite-07 GROUP_END, a Payload Length of 0 then the End, says where a whole group ends; a stream -// ending without one only closes it. -test("a lite-07 GROUP_END says where the group ends", async () => { +// A lite-07 GROUP_END, a Payload Length of 0, says the group is whole; a stream ending without one +// only closes it. +test("a lite-07 GROUP_END says the group is whole", async () => { const frame = [2, 0, 7]; - expect(await readLite07(lite07(...frame, 0, 500))).toBe(500n); - expect(await readLite07(lite07(...frame))).toBeUndefined(); + expect(await readLite07(lite07(...frame, 0))).toBe(true); + expect(await readLite07(lite07(...frame))).toBe(false); }); // A publisher that stalls between GROUP_END and its FIN can't hold the read open: the group's own @@ -173,7 +173,7 @@ test("a lite-07 GROUP_END still waiting for its FIN stops once the group closes" const stream = new Reader( new ReadableStream({ start(controller) { - controller.enqueue(lite07(2, 0, 7, 0, 500)); + controller.enqueue(lite07(2, 0, 7, 0)); }, }), undefined, @@ -184,16 +184,14 @@ test("a lite-07 GROUP_END still waiting for its FIN stops once the group closes" // The frame and GROUP_END arrive together, so once the frame is read the FIN is all that's left. expect((await consumer.readFrame())?.payload).toEqual(new Uint8Array([7])); producer.abort(new Error("evicted")); - expect(await done).toBeUndefined(); + expect(await done).toBe(false); }); -// An End at or before the group's start, or anything after GROUP_END, is malformed. A malformed -// FRAME ends only its stream, so it is a stream error rather than a session's protocol violation. +// Anything after GROUP_END is malformed. A malformed FRAME ends only its stream, so it is a stream +// error rather than a session's protocol violation. test.each([ - ["an End at the group's start", lite07(0, 5), 5n], - ["an End before the group's start", lite07(0, 4), 5n], - ["a byte after GROUP_END", lite07(0, 9, 0), 5n], - ["a frame after GROUP_END", lite07(0, 9, 2, 0, 7), 5n], + ["a byte after GROUP_END", lite07(0, 0), 5n], + ["a frame after GROUP_END", lite07(0, 2, 0, 7), 5n], ])("a lite-07 GROUP_END with %s is malformed", async (_, bytes, start) => { const err = await readLite07(bytes, start); expect(err).toBeInstanceOf(StreamError); diff --git a/js/net/src/lite/group.ts b/js/net/src/lite/group.ts index 8bfbf07e85f..9a373727a5d 100644 --- a/js/net/src/lite/group.ts +++ b/js/net/src/lite/group.ts @@ -119,9 +119,9 @@ export interface WireFrame extends WirePayload { inflated?: number; } -/** Lite-07 GROUP_END: the group is whole, ending at `end` in the track's timescale. */ +/** Lite-07 GROUP_END: the group is whole. */ export interface GroupEnd { - end: bigint; + whole: true; } /** How {@link frameDecoder} reads each frame's optional fields. */ @@ -162,7 +162,7 @@ export function frameDecoder({ let size = 0; if (lengthFirst) { const length = c.u53(); - if (length === 0) return { end: c.u62() }; + if (length === 0) return { whole: true }; size = length - 1; } const delta = stamped ? unzigzag(c.u62()) : 0n; @@ -182,15 +182,11 @@ export function frameDecoder({ /** * Write a group stream's frames into `producer` until the stream ends or the producer closes. - * Resolves with where a lite-07 GROUP_END said the group ends, or undefined when the stream ended - * without one: the caller finishes the group at that end, and a bare FIN only closes it. Anything - * after GROUP_END, or an End the group refuses (at or before its start), is malformed. + * Resolves true when a lite-07 GROUP_END said the group is whole, or false when the stream ended + * without one: the caller finishes the group whole, and a bare FIN only closes it. Anything after + * GROUP_END is malformed. */ -export async function readFrames( - stream: Reader, - producer: netGroup.Producer, - format: FrameFormat, -): Promise { +export async function readFrames(stream: Reader, producer: netGroup.Producer, format: FrameFormat): Promise { const decode = frameDecoder({ ...format, start: producer.start }); for (;;) { @@ -198,15 +194,13 @@ export async function readFrames( // per batch rather than once per frame. Only the group's own stream ends it: a track // that closes first has already closed (or aborted) this group through its cache. const frame = stream.tryDecode(decode) ?? (await race([stream.decodeMaybe(decode), producer.closed])); - if (!frame || frame instanceof Error) return undefined; - if ("end" in frame) { - if (frame.end <= producer.start) throw malformed(`GROUP_END at ${frame.end}, not past the group's start`); + if (!frame || frame instanceof Error) return false; + if ("whole" in frame) { // The FIN may never come, so the group's own close, such as an abort or eviction, still // ends the wait. const done = await race([stream.done(), producer.closed]); if (done === false) throw malformed("bytes after GROUP_END"); - if (done !== true) return undefined; - return frame.end; + return done === true; } if (frame.payload) hooks.writeGroupFrame( @@ -214,6 +208,6 @@ export async function readFrames( { payload: frame.payload, timestamp: Time.received(frame.timestamp, producer.start) }, frame.inflated, ); - else if (!(await receiveFrame(stream, producer, frame))) return; + else if (!(await receiveFrame(stream, producer, frame))) return false; } } diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 7912e4b62c4..4b6867f040f 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -872,14 +872,14 @@ async function servedSubscription( const header = await GroupMessage.decode(reader, version); const payloads: string[] = []; - let end: bigint | undefined; + let whole: true | undefined; while (!(await reader.done())) { // Each frame is a zigzag timestamp delta, then a length-prefixed payload. Lite-07 - // leads with the size plus one instead, where 0 is GROUP_END and its End. + // leads with the size plus one instead, where 0 is GROUP_END. const lengthFirst = hasLengthFirst(version); const length = lengthFirst ? (await reader.u53()) - 1 : undefined; if (length === -1) { - end = await reader.u62(); + whole = true; continue; } await reader.u62(); @@ -891,7 +891,7 @@ async function servedSubscription( frameStart: header.frameStart, payloads, previous: header.previous, - end, + whole, }; }, // Stop the next group stream the publisher opened with `code`, before reading any of it, @@ -1299,18 +1299,18 @@ async function holesToFinish(reader: Reader, count = Number.POSITIVE_INFINITY): } } -// Lite-07 ends a finished group's stream with GROUP_END, naming where the group ends, and a closed -// one's with a bare FIN. Mirrors the Rust `lite07_ends_a_finished_group_with_group_end`. -test("lite-07 ends a finished group with GROUP_END and a closed one with a bare FIN", async () => { +// Lite-07 ends a whole group's stream with GROUP_END and a closed one's with a bare FIN. Mirrors +// the Rust `lite07_ends_a_whole_group_with_group_end`. +test("lite-07 ends a whole group with GROUP_END and a closed one with a bare FIN", async () => { const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n }); try { const whole = sub.track.createGroup(0n); whole.writeFrame(textFrame("x", Timestamp.fromMillis(0))); - whole.finishAt(500n); - expect(await sub.servedGroup()).toMatchObject({ key: 0n, payloads: ["x"], end: 500n }); + whole.finishWhole(); + expect(await sub.servedGroup()).toMatchObject({ key: 0n, payloads: ["x"], whole: true }); sub.serve(1000n); const closed = await sub.servedGroup(); - expect([closed?.key, closed?.payloads, closed?.end]).toEqual([1000n, ["hello"], undefined]); + expect([closed?.key, closed?.payloads, closed?.whole]).toEqual([1000n, ["hello"], undefined]); } finally { await sub.close(); } @@ -2653,7 +2653,7 @@ test("blocked DROP responses stop opening later groups", async () => { }); /** One group stream the publisher put on the wire. */ -type Served = { key: bigint; frameStart: number; payloads: string[]; previous?: bigint; end?: bigint }; +type Served = { key: bigint; frameStart: number; payloads: string[]; previous?: bigint; whole?: true }; /** * Serves `groups` (frame payloads per group, keyed by sequence) under `bounds`, and diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index da9f0027251..73a6acc649a 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -138,12 +138,9 @@ export function frameHeader( return concat(parts); } -/** - * Lite-07 GROUP_END, after a finished group's last frame: a zero `Payload Length`, where a frame - * would start, then the group's exclusive `end` in the track's timescale. - */ -function groupEnd(version: StreamVersion, end: bigint): Uint8Array { - return concat([encodeVarint(0, version), encodeVarint(end, version)]); +/** Lite-07 GROUP_END, after a whole group's last frame: a zero `Payload Length`, where a frame would start. */ +function groupEnd(version: StreamVersion): Uint8Array { + return encodeVarint(0, version); } /** How a lite hop stamps a track's frames and datagrams. */ @@ -2509,10 +2506,9 @@ export class Publisher { else if (frame.payload.length > 0) await stream.write(frame.payload); }); } - // Lite-07: a response that reached a finished group's end says so before the FIN. - const end = group.end; - if (hasGroupEnd(this.version) && end !== undefined && hooks.groupIndex(group) >= group.frameCount) - await write(() => stream.write(groupEnd(stream.version, end))); + // Lite-07: a response that reached a whole group's end says so before the FIN. + if (hasGroupEnd(this.version) && group.whole && hooks.groupIndex(group) >= group.frameCount) + await write(() => stream.write(groupEnd(stream.version))); } /** @@ -2594,7 +2590,7 @@ export class Publisher { // Whether the cursor ever reached the requested start, which decides how the // end of the group is read below. A received tail may already begin there. let reached = hooks.groupIndex(group) === startFrame; - // Whether every frame went out, which a finished group's GROUP_END then claims. + // Whether every frame went out, which a whole group's GROUP_END then claims. let whole = false; for (;;) { @@ -2641,9 +2637,8 @@ export class Publisher { }); } - // Lite-07 ends a finished group with GROUP_END. - const end = hasGroupEnd(this.version) && whole ? group.end : undefined; - if (end !== undefined) await write(() => stream.write(groupEnd(stream.version, end))); + // Lite-07 ends a whole group with GROUP_END. + if (hasGroupEnd(this.version) && whole && group.whole) await write(() => stream.write(groupEnd(stream.version))); stream.close(); group.close(); // A stream that fails after its FIN may have lost its header, so report it for a DROP. diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index b4012c01a12..830e2666caf 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1748,11 +1748,10 @@ test("a malformed lite-07 group stream ends only its group", async () => { const zero = new TextEncoder().encode("zero"); const first = handStream(version); first.write(await liteFrame(300, zero)); - // A zero Payload Length, the End, then a byte where only the FIN may follow. + // A zero Payload Length, then a byte where only the FIN may follow. first.write( await Writer.collect(version, async (w) => { await w.u53(0); - await w.u62(1000n); await w.u53(0); }), ); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 20f498e71cb..93c8af9e436 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -2048,7 +2048,7 @@ export class Subscriber { try { // One guard covers both header waits and a stalled payload. Abandonment aborts // the group before resetting the stream, releasing the shared frame pump too. - const end = await untilAbandoned( + const whole = await untilAbandoned( group, readFrames(stream.reader, group, { scale: info.timescale ?? 0, @@ -2057,9 +2057,9 @@ export class Subscriber { }), ); - // Whole at a GROUP_END's end, else without one: every frame sent arrived. - if (end === undefined) group.finish(); - else group.finishAt(end); + // Whole after a GROUP_END, else only closed: every frame sent arrived. + if (whole) group.finishWhole(); + else group.finish(); stream.close(); } catch (err: unknown) { const e = error(err); @@ -2396,7 +2396,7 @@ export class Subscriber { try { // Once no reader holds the group or waits to pick it up, cancel upstream and abort the // truncated group rather than cache it whole, as the Rust fetch's ingest does. - const end = await untilAbandoned( + const whole = await untilAbandoned( group, readFrames(stream.reader, group, { scale: framing.timescale, @@ -2404,9 +2404,9 @@ export class Subscriber { lengthFirst: hasLengthFirst(this.version), }), ); - // Whole at a GROUP_END's end, else without one: every frame sent arrived. - if (end === undefined) group.finish(); - else group.finishAt(end); + // Whole after a GROUP_END, else only closed: every frame sent arrived. + if (whole) group.finishWhole(); + else group.finish(); stream.close(); } catch (err: unknown) { const e = error(err); @@ -3033,11 +3033,11 @@ export class Subscriber { const format = { scale: timescale, compress: entry.compress, lengthFirst: hasLengthFirst(this.version) }; const frames = readFrames(stream, producer, format); - const end = await (entry.filling ? untilAbandoned(producer, frames) : frames); + const whole = await (entry.filling ? untilAbandoned(producer, frames) : frames); - // Whole at a GROUP_END's end, else without one: every frame sent arrived. - if (end === undefined) producer.finish(); - else producer.finishAt(end); + // Whole after a GROUP_END, else only closed: every frame sent arrived. + if (whole) producer.finishWhole(); + else producer.finish(); stream.stop(StreamCode.Cancel); } catch (err: unknown) { // A violation ends the session, which the connection does once it escapes; anything diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 904877b6258..f099d38a52b 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2067,6 +2067,50 @@ test("a stalled track serves only its newest group", () => { } }); +// The last group, finished whole, ages from one mean frame interval past its highest frame +// timestamp. One only closed may lack later frames, so it never ages without a successor. Mirrors +// the Rust `a_whole_group_ages_from_its_highest_frame`. +test.each([true, false])("a last group finished whole=%s ages from its highest frame", (whole) => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: Duration.ZERO }); + try { + const group = producer.appendGroup(Timestamp.fromMillis(0)); + // Out of order, as B-frames are: the highest is not the last written. + for (const millis of [0, 40, 20]) group.writeFrame(textFrame("x", Timestamp.fromMillis(millis))); + if (whole) group.finishWhole(); + else group.finish(); + + // Three frames over 40ms: a 20ms mean interval, so it presents until 60ms. + clock.set(10_059); + expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual([0n]); + clock.set(10_060); + expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual(whole ? [] : [0n]); + } finally { + producer.finish(); + producer.close(); + clock.installed.close(); + } +}); + +// A whole group whose frames measure no interval, one frame here, says nothing about how long it +// presents, so like a closed one it never ages without a successor. Mirrors the Rust +// `a_whole_group_of_one_frame_never_ages_alone`. +test("a whole group of one frame never ages alone", () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: Duration.ZERO }); + try { + const group = producer.appendGroup(Timestamp.fromMillis(0)); + group.writeFrame(textFrame("x", Timestamp.fromMillis(0))); + group.finishWhole(); + clock.set(70_000); + expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual([0n]); + } finally { + producer.finish(); + producer.close(); + clock.installed.close(); + } +}); + // A peer whose timestamps barely advance never ages a group out on media time. The wall // clock still does. test("frozen timestamps still age out by wall clock", () => { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index a3af08b4981..ae0471a9b0a 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -54,7 +54,7 @@ import { } from "./internal.ts"; import { grace, Ledger } from "./ledger.ts"; import { rescale } from "./rescale.ts"; -import { floorAt, Timescale, type Timestamp } from "./time.ts"; +import { floorAt, Timescale, Timestamp } from "./time.ts"; import { checkEnd, checkKey, max, min, UNBOUNDED } from "./util/key.ts"; import { checkName } from "./util/utf8.ts"; import { bounded, type Holdings, joinsLive, rangesOf, UNHELD, Window, wireSpan } from "./window.ts"; @@ -3249,7 +3249,7 @@ export class Subscriber { let arrived = this.#arrivals ? this.#keptSupersededAt(key, drift, held) : this.#supersededAt(arrival, drift, held); - // A group finished whole ages from its own end without a successor. + // A group finished whole ages from its own frames without a successor. if (arrived === undefined && !this.#arrivals) arrived = this.#endedAt(key); if (arrived === undefined) return false; // Plain arithmetic: an unenforced budget is infinite, which no duration brand holds. @@ -3284,12 +3284,11 @@ export class Subscriber { return scale == null ? Number(value) : (Number(value) * 1000) / scale; } - // The furthest presentation time, in milliseconds, group `key` could still reach: its own - // end once finished whole, else where its immediate successor below `end` begins, undefined - // when nothing proves where it stops. Mirrors the Rust `TrackState::reach`. + // The furthest presentation time, in milliseconds, group `key` could still reach: where its + // immediate successor below `end` begins, undefined when nothing proves where it stops. + // Without one, no edge sits above the group to convict it in media time; a whole group ages + // on the wall clock instead (see {@link #endedAt}). Mirrors the Rust `TrackState::reach`. #reach(key: bigint, end: bigint | undefined, whole?: boolean): number | undefined { - const own = this.#held(key)?.end; - if (own !== undefined) return this.#presented(own); const successor = this.#successor(key, end, whole); if (!successor) return undefined; const info = this.#state.info.peek(); @@ -3297,15 +3296,24 @@ export class Subscriber { return hooks.groupTimestamp(successor)?.asMillis(); } - // When group `key`, finished whole, stops presenting on the wall clock: its arrival plus - // its span, which stands in for a successor arriving, so the last group of a paused or - // finished track ages out without one. Mirrors the Rust `TrackState::ended_at`. + // When group `key`, finished whole, stops presenting on the wall clock: its arrival plus the + // span from its first frame's timestamp to one frame past its highest, a frame being its mean + // interval, which stands in for a successor arriving, so the last group of a paused or finished + // track ages out without one. Undefined when it is not whole, or its frames measure no interval + // (one frame, or all at one timestamp). Only a whole group's frames prove where it stops, and + // they are the same at every copy. Mirrors the Rust `TrackState::ended_at` and + // `Handle::whole_reach`. #endedAt(key: bigint): Instant | undefined { const group = this.#held(key); - const end = group?.end; - if (group === undefined || end === undefined || end <= key) return undefined; - const span = this.#presented(end - key); - return span === undefined ? undefined : hooks.groupArrived(group).checkedAdd(Duration.millis(span)); + if (!group?.whole || group.frameCount < 2) return undefined; + const start = hooks.groupTimestamp(group); + const latest = hooks.groupLatest(group); + if (start === undefined || latest?.scale === undefined) return undefined; + // In the track's units, as Rust divides them. + const interval = Math.floor((latest.value - start.value) / (group.frameCount - 1)); + if (interval <= 0) return undefined; + const reach = new Timestamp(latest.value + interval, latest.scale); + return hooks.groupArrived(group).checkedAdd(Duration.millis(reach.asMillis() - start.asMillis())); } // When the wall clock starts: the cached `arrival`'s arrival, or that of the one `held` diff --git a/quest/m2/README.md b/quest/m2/README.md index e211eeca350..4456e40b92d 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -47,7 +47,7 @@ removal, a rename, or a wire, catalog, or format change that is free until - **Catalog, drafts, and formats.** hang-04 (done: the `sync` section, a version per entry, no video `warmup`, the TS passthrough under `mpegts`, one strict rule); declared ends, back from m3, which became - [a whole marker](/quest/m2/time-addressing/group-end-whole.md) on 2026-11-19. + [a whole marker](https://github.com/moq-dev/moq/pull/9221) on 2026-11-19. - **Native media and gateways.** [Upstream str0m](https://github.com/moq-dev/moq/pull/8963), which reverses the 2026-10-30 vendoring. - **Bindings.** The C API as `libmoq` (done), reversing 2026-10-25. diff --git a/quest/m2/time-addressing/branch-salvage.md b/quest/m2/time-addressing/branch-salvage.md index 4dd74f5d9f2..e7d6374f086 100644 --- a/quest/m2/time-addressing/branch-salvage.md +++ b/quest/m2/time-addressing/branch-salvage.md @@ -33,7 +33,7 @@ worth keeping. Research on the branch (head `20c13b1308`) found: on the branch that aging is why the WHEP tests changed (`rs/moq-webrtc/src/lib.rs` `flowing!`, and `cd42c40e49`). Recommended: land it only if something still reads the duration once - [GROUP_END has no End](/quest/m2/time-addressing/group-end-whole.md), such + [GROUP_END has no End](https://github.com/moq-dev/moq/pull/9221), such as hang's duration marker or the HLS export, and say in the PR either way. The rest of the branch (its end and loss tests, `read_breaks`, the HLS diff --git a/quest/m2/time-addressing/group-end-whole.md b/quest/m2/time-addressing/group-end-whole.md deleted file mode 100644 index dd0941f52de..00000000000 --- a/quest/m2/time-addressing/group-end-whole.md +++ /dev/null @@ -1,83 +0,0 @@ -# [M] lite-07's GROUP_END says only that a group is whole - -## Goal - -On lite-07-wip, GROUP_END drops its `End` field and stays a zero-length frame -meaning "this group is whole". A Group Stream that FINs without it carries a -closed, possibly partial group that failover may continue. A group's end -position lives only in hang's markers. moq-net ages a finished group with no -successor from its highest frame timestamp, and any other group from the next -group's start. Rust, `@moq/net`, and the moq-lite draft change together. - -## Plan - -Decided 2026-11-19 (maintainer, on closed -[moq-dev/moq#9192](https://github.com/moq-dev/moq/pull/9192) and this plan's -interview): - -- ✅ Drop GROUP_END's timestamp; hang keeps its in-band end markers on every - transport (rejected: readers infer stops from GROUP_END's position, as - #9192 did; keep the position but ignore it). Reason: the timestamp has no - moq-transport equivalent and saves only about a byte over hang's marker. -- ✅ Keep GROUP_END as a bare whole marker (rejected: drop GROUP_END - entirely, with FIN meaning whole). Reason: a FIN cannot tell a partial - group from a whole one. The draft lets a relay whose upstream closed - mid-group deliver the frames it holds and FIN, and failover continues that - group from another publisher; relay caching relies on knowing which copies - are whole. moq-transport's END_OF_GROUP is the same bit, so `ietf` maps it - one to one (`ietf/publisher.rs:7979` already sends END_OF_GROUP only for a - whole group). -- ✅ Aging: a finished group with no successor ages from its highest frame - timestamp; a group with a successor is bounded by the successor's start - (rejected: age from when the last byte arrived; never age it). The rule - ages up to one frame early, since frame durations are not on the wire. A - pause makes the next start overstate a group's reach, which only keeps it - a little longer; hang's markers give the player the exact end. - -Where the timestamp lives today: - -- Rust wire: `buffer_group_end` writes `varint(0)` then `varint(end)` - (`rs/moq-net/src/lite/publisher.rs:6432-6450`; call sites `:3300`, `:3330`, - `:6804`, `:6861`). Decode in `lite/subscriber.rs:1605-1609` and - `:1791-1814`, applied at `:1543` through `recv.rs:111-131` (`finish_at`). - The version gate `has_group_end` (`lite/version.rs:298-306`) also gates - GROUP's Previous Start; keep that. -- JS wire: `frameDecoder` in `js/net/src/lite/group.ts:165` (type at - `:122-125`, `readFrames` at `:189-215`), `groupEnd()` in - `js/net/src/lite/publisher.ts:144-146` (sent at `:2488-2490`, - `:2620-2621`), applied in `lite/subscriber.ts:2062`, `:2409`, `:3039`. -- API: `group::Producer::finish_at(end)` (`rs/moq-net/src/model/group.rs:2266`) - and `group::Consumer::end()` (`:4255`); JS `Producer.finishAt` and the - `Consumer.end` getter (`js/net/src/group.ts:729`, `:1028`, mirrors at - `:469`, `:783`). `finish_at` becomes a timestamp-free whole finish, and - `finish()` stays the closed one. Recommended name: `finish_whole()`, with - `Outcome::Finished` carrying a whole flag instead of an end. -- Callers outside moq-net: `rs/moq-mux/src/container/producer.rs:809` - (`close_group`), `container/consumer.rs:793-801` (feeds only - `presented_end`), `container/ts/recovery_test.rs:497`, `:941`, - `rs/moq-mux/tests/legacy_audio.rs:164`, `js/hang/src/container/legacy.ts:283-288`. -- Aging: Rust `rs/moq-net/src/model/track.rs` `reach` (`:2175-2179`), - `own_end` (`:2203-2205`), `ended_at` (`:2210-2214`), used by `superseded`, - `drifted`, `poll_drifted`, documented at `:2316-2321` and - `model/group.rs:2257-2264`; tests at `track.rs:14674`, `:18642`. JS - `js/net/src/track.ts:3243` (`#endedAt`), `:3252`, `:3280-3281`, `:3293-3295`. -- Draft `drafts/draft-lcurley-moq-lite.md`: group end states (183-187), the - relay's closed delivery (225), Fetch (637, 2016), reach (829-830), expiry - (842-843, 2000), Group Stream (871, 875-876), `Payload Length` 0 (2200), - the GROUP_END message (2220-2237), and the changelog (2354). The TLA model - mentions it in `rs/moq-net/verify/subscribe/SubscribeAccounting.tla` - (lines 18, 44, 175, 252). - -Tests: the GROUP_END codec round trip and fuzz (`subscriber.rs:4632`, -`:4651`, `publisher.rs:7285`, `fuzz/ingest.rs:388`) without the field; a -closed group still reads as partial and a whole one as whole across a relay; -the last group of a paused track ages out from its highest frame timestamp. -Run `just test interop --all`. - -Public API: breaking in Rust and JS (`finish_at` and `end()` on groups). -Wire: lite-07-wip only, which is unpublished. - -## Related - -- [Start placement](/quest/m2/time-addressing/start-placement.md) - the by-start conversions the old consumers quest also held -- [Salvage from #9192](/quest/m2/time-addressing/branch-salvage.md) - pieces of the closed branch that stand alone diff --git a/quest/m2/time-addressing/start-placement.md b/quest/m2/time-addressing/start-placement.md index a7101288d10..783ae652ef6 100644 --- a/quest/m2/time-addressing/start-placement.md +++ b/quest/m2/time-addressing/start-placement.md @@ -17,7 +17,7 @@ its in-band group end markers (empty duration frame, discontinuity marker group) on every transport (rejected: readers infer stops from lite-07's GROUP_END position). This reverses the 2026-11-16 decision that dropped the markers before the cut, and GROUP_END's timestamp goes instead (see -[group-end-whole](/quest/m2/time-addressing/group-end-whole.md)). So +[moq-dev/moq#9221](https://github.com/moq-dev/moq/pull/9221)). So `Read::End`, `Producer::discontinuity`, `Container::end`, and `Container::finish_group` keep their meaning, and the hang draft keeps its marker rules. @@ -53,7 +53,3 @@ reads them as continuous where the media is; an HLS export of a track whose first frame sits above its group start lands in the same segment as before. Public API: none expected; a renamed parameter is not a break. Wire: none. - -## Related - -- [GROUP_END is a bare whole marker](/quest/m2/time-addressing/group-end-whole.md) - the wire half of the same re-plan diff --git a/quest/m3/groups-never-overlap.md b/quest/m3/groups-never-overlap.md index f3b0312c189..3a0082170c7 100644 --- a/quest/m3/groups-never-overlap.md +++ b/quest/m3/groups-never-overlap.md @@ -60,5 +60,4 @@ Public API: none expected. Wire: none; the draft's group rule tightens. ## Related -- [GROUP_END is a bare whole marker](/quest/m2/time-addressing/group-end-whole.md) - a finished group with no successor ages from its highest frame timestamp, which this rule keeps below the next start - [An fMP4 group start never rewinds the decode timeline](/quest/m3/fmp4-group-start-tfdt.md) - a keyframe repeating its predecessor's timestamp would now continue the group; check which of its cases remain diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index b514d66dcb6..6c009463538 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -2738,7 +2738,9 @@ mod tests { .unwrap(); } } - let _finished = producer.finish().unwrap(); + // Left open, as a live burst is: finished whole, its wall-clock expiry would arm a timer + // that lets the paused clock run past the instant this measures. + let _producer = producer; let catalog = hang::catalog::VideoConfig::new(hang::catalog::H264 { inline: true, profile: 0x42, diff --git a/rs/moq-e2ee/src/group.rs b/rs/moq-e2ee/src/group.rs index 52f00f1ff97..97f55763aa0 100644 --- a/rs/moq-e2ee/src/group.rs +++ b/rs/moq-e2ee/src/group.rs @@ -91,7 +91,7 @@ impl Producer { Ok(()) } - /// Finish the group without a known end. No more frames will be written. + /// Close the group without saying it is whole. No more frames will be written. /// /// # Errors /// diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 55209132670..d17e327bb62 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -789,15 +789,7 @@ impl Consumer { self.group = group.group.start; return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: true }))); } - // A group its producer finished whole presents up to its end, which can pass - // what its frames reach when they carry no duration. - let whole = group - .group - .end() - .zip(group.group.timescale()) - .and_then(|(end, scale)| Timestamp::new(end, scale).ok()) - .and_then(time); - if let Some(end) = group.max_end.max(whole) { + if let Some(end) = group.max_end { self.presented_end = Some(end); } self.pending.pop_front(); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index dd077aeee83..c707286e978 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -738,20 +738,12 @@ where /// returns [`InvalidEnd`](super::InvalidEnd) without flushing or closing the group. pub fn cut(&mut self, end: Option) -> crate::Result<()> { let marker_at = end.or_else(|| self.estimated_end()); - self.close(end, marker_at, None) + self.close(end, marker_at) } - /// Close the current group, ending a video track's group with a duration marker at `marker_at`. - /// - /// The group ends whole at `whole`, else at a bound `end` the caller knows, such as the next - /// keyframe, else where a media track's duration marker goes, so a reader can tell the group - /// starting there continues it. - fn close( - &mut self, - end: Option, - marker_at: Option, - whole: Option, - ) -> crate::Result<()> { + /// Close the current group whole, ending a video track's group with a duration marker at + /// `marker_at`. + fn close(&mut self, end: Option, marker_at: Option) -> crate::Result<()> { if self.container.kind() == Kind::Video && !self.reordered && let Some((end, previous)) = end.zip(self.previous_timestamp) @@ -777,10 +769,7 @@ where self.previous_timestamp = None; let tail_end = marker_at.filter(|_| !reordered); - // Media ends whole where its last frame does, as the duration marker says. A data track's - // items have no cadence to estimate that from. - let estimated = tail_end.filter(|_| self.container.kind() != Kind::Data); - if let Err(err) = self.close_group(whole.or(end).or(estimated), tail_end, reached) { + if let Err(err) = self.close_group(tail_end, reached) { // A group that fails to close fails with the cause, rather than staying open for the next // write to continue. self.buffer.clear(); @@ -792,11 +781,10 @@ where self.publish_estimate() } - /// Flush the buffer into the open group and finish it, raising the live edge to `reached`. With - /// a `bound`, the group ends whole there rather than only closing. + /// Flush the buffer into the open group and finish it whole, since it holds every frame + /// written, raising the live edge to `reached`. fn close_group( &mut self, - bound: Option, tail_end: Option, reached: Option, ) -> crate::Result<()> { @@ -805,10 +793,7 @@ where self.container.finish_group(group, tail_end)?; } if let Some(group) = self.group.take() { - match bound.and_then(|bound| crate::start::end(&self.inner, group.start, bound)) { - Some(end) => group.finish_at(end)?, - None => group.finish()?, - }; + group.finish_whole()?; // Only a closed group reaches anything; a cut with none open names a point nothing reached. if let Some(end) = reached { self.raise_live_edge(end); @@ -918,8 +903,7 @@ where // The marker starts one unit of the track's timescale before the live edge, so media // resuming right at the edge still starts after it, but not below the latest frame, which // it would otherwise claim to end before. Or one unit past the newest group when that - // doesn't pass the group's start. The open group ends whole there, so a reader can tell - // the marker follows it. + // doesn't pass the group's start. let marker = match (self.pending_start, edge) { // A seek keyed the next group. Its marker is stamped at the live edge, or one unit past // the newest group when the edge is unknown or doesn't pass it, since starts strictly @@ -940,8 +924,7 @@ where Some((start, timestamp, true)) } }; - let whole = marker.and_then(|(_, timestamp, appended)| appended.then_some(timestamp)); - self.close(None, None, whole)?; + self.close(None, None)?; // Nothing is measured across the break: the frames still open on this side have no end, and // the gap to the far side is not a frame duration. self.estimator.discontinuity(); @@ -966,7 +949,7 @@ where }], )?, } - group.finish()?; + group.finish_whole()?; Ok(()) } diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 9a3da4fdede..319f8e48dda 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -446,8 +446,8 @@ struct Recorded { name: String, video: bool, groups: Vec<(u64, Vec)>, - /// Where each group ended whole, when the import said. - ends: Vec>, + /// Whether the import finished each group whole. + wholes: Vec, } /// What the import of a clip published. @@ -484,7 +484,7 @@ async fn record(profile: Profile) -> Recording { .await .unwrap(); let mut groups = Vec::new(); - let mut ends = Vec::new(); + let mut wholes = Vec::new(); while let Some(mut group) = kio::clock::timeout(Duration::from_secs(1), track.recv_group()) .await .unwrap() @@ -494,7 +494,7 @@ async fn record(profile: Profile) -> Recording { while let Some(frame) = group.read_frame().await.unwrap() { frames.push(frame); } - ends.push(group.end()); + wholes.push(group.is_whole()); groups.push((group.start, frames)); } // The import appends each group, so nothing lies below its first or between two. @@ -505,7 +505,7 @@ async fn record(profile: Profile) -> Recording { name, video, groups, - ends, + wholes, }); } assert_eq!( @@ -905,7 +905,7 @@ async fn run(recording: &Recording, arm: &Arm) -> Run { let consumer = broadcast.consume(); let groups: Vec)>> = recording.tracks.iter().map(|track| track.groups.clone()).collect(); - let ends: Vec>> = recording.tracks.iter().map(|track| track.ends.clone()).collect(); + let wholes: Vec> = recording.tracks.iter().map(|track| track.wholes.clone()).collect(); // The import's skips, which a relay forwards with the first group past each: a group the path // omits is not skipped, so the export sees it missing. let mut skips: Vec<_> = (recording.tracks.iter()) @@ -936,11 +936,11 @@ async fn run(recording: &Recording, arm: &Arm) -> Run { let (sequence, frames) = &groups[t][g]; let Some(f) = step else { let group = open[t].take().unwrap(); - match (kept[t] == frames.len(), ends[t][g]) { - (true, Some(end)) => { - group.finish_at(end).unwrap(); + match (kept[t] == frames.len(), wholes[t][g]) { + (true, true) => { + group.finish_whole().unwrap(); } - (true, None) => { + (true, false) => { group.finish().unwrap(); } (false, _) => group.abort(moq_net::Error::Old).unwrap(), diff --git a/rs/moq-mux/src/start.rs b/rs/moq-mux/src/start.rs index 2a274630a05..e29dc1bdde6 100644 --- a/rs/moq-mux/src/start.rs +++ b/rs/moq-mux/src/start.rs @@ -74,12 +74,3 @@ pub(crate) fn after(track: &moq_net::track::Producer, timestamp: moq_net::Timest } } -/// Where a group ends whole on `track` when its content stops at `timestamp`: `timestamp` in the -/// track's timescale, or `None` when that doesn't pass `start` or won't convert. -pub(crate) fn end(track: &moq_net::track::Producer, start: u64, timestamp: moq_net::Timestamp) -> Option { - let end = track.info().rescale(timestamp).ok()?; - // A scale-less track's key is no time. - end.scale()?; - let end = end.value(); - (end > start).then_some(end) -} diff --git a/rs/moq-mux/tests/legacy_audio.rs b/rs/moq-mux/tests/legacy_audio.rs index 6a9b6c188cd..5ddb4965c48 100644 --- a/rs/moq-mux/tests/legacy_audio.rs +++ b/rs/moq-mux/tests/legacy_audio.rs @@ -161,7 +161,7 @@ async fn a_dropped_run_waits_for_the_next_groups_first_packet() { duration: None, }; container.write(&mut group, &[packet]).unwrap(); - group.finish_at(start + 20_000).unwrap(); + group.finish_whole().unwrap(); let read = kio::clock::timeout(TIMEOUT, reader.read()) .await .unwrap_or_else(|_| panic!("{version}: packet {n} never read")) diff --git a/rs/moq-net/src/fuzz/ingest.rs b/rs/moq-net/src/fuzz/ingest.rs index cd8d9fc71e3..9fe824fb92b 100644 --- a/rs/moq-net/src/fuzz/ingest.rs +++ b/rs/moq-net/src/fuzz/ingest.rs @@ -216,7 +216,7 @@ fn run(session: Session, info: &track::Info, body: &[u8], chunk: usize) -> Optio let broadcast = broadcast.produce(); let track = broadcast.create_track("ingest", info.clone())?; - let (end, fin, group) = match session { + let (end, whole, group) = match session { Session::Lite { version, records, @@ -234,9 +234,9 @@ fn run(session: Session, info: &track::Info, body: &[u8], chunk: usize) -> Optio keep_up(&mut viewer); res }); - let fin = end.as_ref().ok().copied().flatten(); - let end = end.map(|end| end.map_or_else(|| "fin".to_string(), |end| format!("end {end}"))); - (end, fin, (group, consumer)) + let whole = end.as_ref().is_ok_and(|whole| *whole); + let end = end.map(|whole| if whole { "whole" } else { "fin" }.to_string()); + (end, whole, (group, consumer)) } Session::Ietf { version } => { let (header, used) = ietf::GroupHeader::decode_slice(body, version).ok()?; @@ -262,15 +262,17 @@ fn run(session: Session, info: &track::Info, body: &[u8], chunk: usize) -> Optio keep_up(&mut viewer); res }); - (end.map(|ended| format!("{ended:?}")), None, (group, consumer)) + (end.map(|ended| format!("{ended:?}")), false, (group, consumer)) } }; let (group, consumer) = group; - // Settle the group as the session does, so the reader below never waits. A GROUP_END - // the group refuses, such as one before a frame it holds, fails the stream there too. + // Settle the group as the session does, so the reader below never waits. let end = match end { - Ok(said) => group.end(fin).map(|()| said), + Ok(said) => { + group.end(whole); + Ok(said) + } Err(err) => { let _ = group.abort(err.clone()); Err(err) @@ -386,10 +388,9 @@ pub(super) fn seeds() -> Vec { data.extend_from_slice(payload); } if version.has_group_end() { - // A zero length, then the End: past the last frame's 2ms, or the start's next - // unit when untimed. + // A zero length: GROUP_END, the group is whole. let mut whole = data.clone(); - whole.extend_from_slice(&[0, if timed { 3 } else { 1 }]); + whole.push(0); push(whole); } push(data); diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index ea8b3809cf9..81a8ce8126c 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -7976,7 +7976,7 @@ impl GroupServe { if let Some(end) = group.end_object() { return Some(status_fields(end)); } - if group.wire().is_some() || group.end().is_none() { + if group.wire().is_some() || !group.is_whole() { return None; } let id = group.index(); @@ -17127,10 +17127,7 @@ mod serve_tests { group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } match whole { - true => { - let end = group.start + 1; - group.finish_at(end).unwrap() - } + true => group.finish_whole().unwrap(), false => group.finish().unwrap(), }; // The track goes on past the range, so no END_OF_TRACK marks the group's end. diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 113bc66af57..86231b77f7c 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -6529,23 +6529,18 @@ fn poll_given_up(group: &mut group::Consumer, waiter: &kio::Waiter) -> Option( writer: &mut Writer, version: Version, records: bool, group: &group::Consumer, ) -> Result { - if !version.has_group_end() || records { + if !version.has_group_end() || records || !group.is_whole() || group.index() < group.frame_count() as u64 { return Ok(false); } - let Some(end) = group.end().filter(|_| group.index() >= group.frame_count() as u64) else { - return Ok(false); - }; writer.buffer_varint(0)?; - writer.buffer_varint(end)?; Ok(true) } @@ -7469,10 +7464,10 @@ mod serve_group_test { (bytes, fin) } - /// Lite-07 ends a finished group's stream with GROUP_END, a zero `Payload Length` then the - /// End, before the FIN; a closed group's ends with the FIN alone. + /// Lite-07 ends a whole group's stream with GROUP_END, a zero `Payload Length`, before the + /// FIN; a closed group's ends with the FIN alone. #[kio::test] - async fn lite07_ends_a_finished_group_with_group_end() { + async fn lite07_ends_a_whole_group_with_group_end() { for finished in [true, false] { let log = Log::default(); let subscription = lite07_subscription(SinkSession::new(log.clone())); @@ -7486,7 +7481,7 @@ mod serve_group_test { .write_frame(Timestamp::from_millis(2).unwrap(), b"x".as_slice()) .unwrap(); match finished { - true => group.finish_at(5).unwrap(), + true => group.finish_whole().unwrap(), false => group.finish().unwrap(), }; let handle = subscription.priority.insert(Priority::new(0, 0, 0)); @@ -7499,7 +7494,7 @@ mod serve_group_test { assert!(fin, "finished={finished}: no FIN"); // The frame: its length plus one, the timestamp delta, and the payload. let tail: &[u8] = match finished { - true => &[2, 4, b'x', 0, 5], + true => &[2, 4, b'x', 0], false => &[2, 4, b'x'], }; assert!(bytes.ends_with(tail), "finished={finished}: {bytes:?}"); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a681670af07..6a0294f1bc8 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1539,8 +1539,11 @@ impl GroupRecv { }); let res = match res { - // Whole at a GROUP_END's end, else closed: every frame sent arrived. - Ok(end) => group.end(end), + // Whole after a GROUP_END, else closed: every frame sent arrived. + Ok(whole) => { + group.end(whole); + Ok(()) + } Err(err @ (Error::Cancel | Error::Stream(crate::StreamError::Cancel))) => { let _ = group.abort(err); Ok(()) @@ -1603,10 +1606,8 @@ enum IngestPhase { /// Lite-07: reading the `Payload Length`, the payload size plus one, which leads the /// frame, or 0 for GROUP_END. Stream end here means the group closed incomplete. Length, - /// Lite-07: reading GROUP_END's `End`, the group's exclusive end. - GroupEnd, - /// Lite-07: GROUP_END said where the group ends, so only the stream end may follow. - Finished { end: u64 }, + /// Lite-07: GROUP_END said the group is whole, so only the stream end may follow. + Whole, /// Reading the timestamp delta (skipped when frames carry none), after the size on lite-07. /// Stream end here before lite-07 means the group has no more frames. Timing { size: Option }, @@ -1707,15 +1708,15 @@ impl FrameIngest { } } - /// `Ready(Ok(end))` once the stream FINs on a frame boundary, with the group's end when a - /// GROUP_END preceded the FIN. The caller finishes, closes, or aborts the group; a frame - /// cut short mid-payload was already aborted here with the reason. + /// `Ready(Ok(whole))` once the stream FINs on a frame boundary, whole when a GROUP_END + /// preceded the FIN. The caller finishes, closes, or aborts the group; a frame cut short + /// mid-payload was already aborted here with the reason. pub(crate) fn poll( &mut self, reader: &mut Reader, group: &mut crate::recv::Group, waiter: &kio::Waiter, - ) -> Poll, Error>> { + ) -> Poll> { let mut budget = kio::coop::Budget::new(32); loop { match &mut self.phase { @@ -1737,7 +1738,7 @@ impl FrameIngest { } IngestPhase::Object => { let Some(code) = ready!(reader.poll_varint_maybe(waiter))? else { - return Poll::Ready(Ok(None)); + return Poll::Ready(Ok(false)); }; let records = self.records.as_mut().expect("the Group State came first"); let (id, end) = lite::ietf::decode_object(records.prior, code)?; @@ -1784,11 +1785,11 @@ impl FrameIngest { if ready!(reader.poll_has_more(waiter))? { return Poll::Ready(Err(Error::ProtocolViolation)); } - return Poll::Ready(Ok(None)); + return Poll::Ready(Ok(false)); } IngestPhase::Length => { let Some(length) = ready!(self.poll_header_varint(reader, true, waiter))? else { - return Poll::Ready(Ok(None)); + return Poll::Ready(Ok(false)); }; // 0 is GROUP_END, where a frame would start. Under object records a record // precedes every FRAME, and the group's end is the stream's. @@ -1796,21 +1797,16 @@ impl FrameIngest { if self.records.is_some() { return Poll::Ready(Err(Error::ProtocolViolation)); } - self.phase = IngestPhase::GroupEnd; + self.phase = IngestPhase::Whole; continue; }; self.phase = IngestPhase::Timing { size: Some(size) }; } - IngestPhase::GroupEnd => { - let end = ready!(reader.poll_varint(waiter))?; - self.phase = IngestPhase::Finished { end }; - } - IngestPhase::Finished { end } => { - let end = *end; + IngestPhase::Whole => { if ready!(reader.poll_has_more(waiter))? { return Poll::Ready(Err(Error::MalformedTrack)); } - return Poll::Ready(Ok(Some(end))); + return Poll::Ready(Ok(true)); } IngestPhase::Timing { size } => { let size = *size; @@ -1820,7 +1816,7 @@ impl FrameIngest { } // Before lite-07 the timestamp delta doubles as the per-frame sentinel. let Some(zz) = ready!(self.poll_header_varint(reader, size.is_none(), waiter))? else { - return Poll::Ready(Ok(None)); + return Poll::Ready(Ok(false)); }; let next: u64 = (self.prev_ts as i128 + crate::coding::varint::unzigzag(zz) as i128) .try_into() @@ -1844,7 +1840,7 @@ impl FrameIngest { true => { let boundary = size.is_none() && !self.framing.stamped(); let Some(inflated) = ready!(self.poll_header_varint(reader, boundary, waiter))? else { - return Poll::Ready(Ok(None)); + return Poll::Ready(Ok(false)); }; Some(inflated) } @@ -1862,7 +1858,7 @@ impl FrameIngest { let (timestamp, inflated) = (*timestamp, *inflated); let boundary = !self.framing.stamped() && !group.compressed(); let Some(size) = ready!(self.poll_header_varint(reader, boundary, waiter))? else { - return Poll::Ready(Ok(None)); + return Poll::Ready(Ok(false)); }; let header = self.header(group, timestamp, inflated, size)?; self.phase = IngestPhase::Admit { header }; @@ -4626,12 +4622,12 @@ mod tests { } } - /// GROUP_END finishes the group whole at its End, and a bare FIN closes it with every frame - /// sent but no end: each frame is its `Payload Length`, a timestamp delta, and the payload. + /// GROUP_END finishes the group whole, and a bare FIN closes it with every frame sent but + /// not whole: each frame is its `Payload Length`, a timestamp delta, and the payload. #[kio::test] async fn group_end_finishes_and_a_bare_fin_closes() { let x = u64::from(b'x'); - for (body, end) in [(&[2, 0, x, 0, 5][..], Some(5)), (&[2, 0, x][..], None)] { + for (body, whole) in [(&[2, 0, x, 0][..], true), (&[2, 0, x][..], false)] { let (res, track, _) = receive_lite07(lite07(Some(Timescale::MILLI)), false, lite07_header(0, None), body).await; res.unwrap(); @@ -4641,28 +4637,25 @@ mod tests { b"x"[..] ); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_none()); - assert_eq!(group.end(), end, "{body:?}"); + assert_eq!(group.is_whole(), whole, "{body:?}"); } } - /// Anything after GROUP_END, an End the group refuses (at or before its start), or a - /// zero length under object records, where a record would come first, is refused. + /// Anything after GROUP_END, or a zero length under object records, where a record would + /// come first, is refused. #[kio::test] async fn group_end_refuses_what_cannot_end_a_group() { let x = u64::from(b'x'); for (key, body, reason) in [ - (0, &[2, 0, x, 0, 5, 0][..], "a byte after GROUP_END"), - (0, &[2, 0, x, 0, 5, 2, 0, x], "a frame after GROUP_END"), - (5, &[2, 5, x, 0, 5], "an End at the group's start"), - (5, &[2, 5, x, 0, 4], "an End before the group's start"), - (0, &[0], "a stream ending inside GROUP_END"), + (0, &[2, 0, x, 0, 0][..], "a byte after GROUP_END"), + (0, &[2, 0, x, 0, 2, 0, x], "a frame after GROUP_END"), ] { let (res, track, _) = receive_lite07(lite07(Some(Timescale::MILLI)), false, lite07_header(key, None), body).await; assert!(res.is_err(), "accepted {reason}"); if let Some(mut group) = track.consume().peek_group(key) { while let Some(Ok(Some(_))) = group.read_frame().now_or_never() {} - assert!(group.end().is_none(), "{reason}: the group finished"); + assert!(!group.is_whole(), "{reason}: the group finished whole"); } } let (res, ..) = receive_ietf(draft18(None), &[0, 0]).await; @@ -13561,10 +13554,8 @@ impl kio::Task for FetchServeRun { unreachable!() }; match res { - // Whole at a GROUP_END's end, else closed; an end the group refuses aborts it. - Ok(end) => { - let _ = producer.end(end); - } + // Whole after a GROUP_END, else closed. + Ok(whole) => producer.end(whole), Err(err) => { let _ = producer.abort(err); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 40c06bf6cc8..9e70d6d074b 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -290,9 +290,9 @@ pub(super) enum Outcome { /// Frames may still be written. #[default] Open, - /// Finished with this many frames, and where it ends in its track's timescale when its - /// producer said it is whole: `None` when it finished without an end (see [`Producer::finish`]). - Finished(usize, Option), + /// Finished with this many frames, and whether its producer said it is whole: `false` when + /// it only closed, possibly short of its frames (see [`Producer::finish`]). + Finished(usize, bool), /// Finished with this many frames, then aborted to free its cache: a reader that took /// every frame got the whole group, and one short of it gets the error. Released(usize, Error), @@ -2281,48 +2281,33 @@ impl Producer { let _ = self.handle.close_aborted(err); } - /// End the group whole at `end`, the exclusive end of its presentation in its track's - /// timescale, returning the [`Finished`] group: no more frames can be written, but it can - /// still be read or aborted. + /// End the group whole, returning the [`Finished`] group: it holds every frame it will + /// ever have, so no more can be written, but it can still be read or aborted. /// - /// A finished group's own end bounds how long it stays fresh, so the last group of a - /// paused or finished track ages out without a successor (see - /// [`track::Info::max_age`]). An `end` at or before the group's start is - /// [`Error::TimestampMismatch`], which aborts the group. Fails with the abort error if the - /// group was already aborted. - pub fn finish_at(self, end: u64) -> Result { - if let Err(err) = self.check_end(end) { - let _ = self.handle.close_aborted(err.clone()); - return Err(err); - } - self.end(Some(end)) - } - - /// Whether [`Self::finish_at`] takes `end`: [`Error::TimestampMismatch`] at or before the - /// group's start. - pub(crate) fn check_end(&self, end: u64) -> Result<()> { - match end <= self.start { - true => Err(Error::TimestampMismatch), - false => Ok(()), - } + /// A whole group with no successor ages from one mean frame interval past its highest frame + /// timestamp, so the last group of a paused or finished track ages out (see + /// [`track::Info::max_age`]). Fails + /// with the abort error if the group was already aborted. + pub fn finish_whole(self) -> Result { + self.end(true) } - /// End the group without saying where it ends, returning the [`Finished`] group: every - /// frame written still reaches its readers, but they can't tell the group is whole, since - /// only [`Self::finish_at`] declares the end a media reader checks against. + /// Close the group without saying it is whole, returning the [`Finished`] group: every + /// frame written still reaches its readers, but they can't tell whether frames are + /// missing, so failover may continue it from another copy. /// - /// What a producer that doesn't know its group's end calls, and how a relay ends a - /// group that expired or arrived partial. Fails with the abort error if the group was + /// How a relay ends a group that expired or arrived partial. A producer that wrote every + /// frame calls [`Self::finish_whole`] instead. Fails with the abort error if the group was /// already aborted. pub fn finish(self) -> Result { - self.end(None) + self.end(false) } - fn end(self, end: Option) -> Result { + fn end(self, whole: bool) -> Result { let mut state = modify(&self.handle.state)?; debug_assert!(state.partial.is_none(), "a frame is already open"); let fin = state.next_index; - state.outcome = Outcome::Finished(fin, end); + state.outcome = Outcome::Finished(fin, whole); // No frame follows to compress. let original_fin = state.range_end.is_none().then_some(fin); if let Some(flate) = &mut state.flate { @@ -3023,15 +3008,6 @@ impl Handle { self.state.read().outcome.frames().is_some() } - /// Where the group ends in its track's timescale, once its producer finished it whole - /// (see [`Producer::finish_at`]). - pub(crate) fn end(&self) -> Option { - match self.state.read().outcome { - Outcome::Finished(_, end) => end, - _ => None, - } - } - /// Whether the group was finished: it holds every frame it will ever have. pub(crate) fn is_finished(&self) -> bool { { @@ -3147,6 +3123,30 @@ impl Handle { } } + /// How far a group finished whole presents: one frame past its highest frame timestamp, a + /// frame being its mean interval from its first frame's timestamp to its highest. `None` + /// when it is not whole, or when its frames measure no interval (one frame, or all at one + /// timestamp). Only a whole group's frames prove where it stops, and they are the same at + /// every copy, so its reach never depends on which copy judges it. + pub(super) fn whole_reach(&self) -> Option { + let state = self.state.read(); + let Outcome::Finished(frames, true) = state.outcome else { + return None; + }; + let Timeline::Timed { start, latest } = state.timeline else { + return None; + }; + if state.origin == Origin::Unverified { + return None; + } + let gaps = u64::try_from(frames.checked_sub(1)?).ok()?; + let interval = latest.value().checked_sub(start.value())?.checked_div(gaps)?; + if interval == 0 { + return None; + } + latest.checked_add(Timestamp::new(interval, latest.scale()?).ok()?).ok() + } + /// The group's full cached footprint (payload plus fixed overhead, and a compressed /// group's inflated copy as far as it is built), used by the track to size this group /// as an eviction victim. @@ -4293,13 +4293,10 @@ impl Consumer { state.outcome.frames().unwrap_or(state.next_index) } - /// Where the group ends in its track's timescale, once its producer finished it whole - /// with [`Producer::finish_at`]: `None` while it is open, or when it finished without an end. - pub fn end(&self) -> Option { - match self.cursor.state.read().outcome { - Outcome::Finished(_, end) => end, - _ => None, - } + /// Whether its producer finished the group whole with [`Producer::finish_whole`]: `false` + /// while it is open, aborted, or only closed. + pub fn is_whole(&self) -> bool { + matches!(self.cursor.state.read().outcome, Outcome::Finished(_, true)) } /// The next frame, as a consumer for chunked reading. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ead2ef5aac9..92d1d5e1f97 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2281,15 +2281,13 @@ impl TrackState { /// its immediate successor begins, or `None` when nothing proves where it stops. /// /// An upper bound, deliberately. A frame's duration is not on the wire, so a group's - /// own last timestamp says where it *starts* presenting, not where it ends; only its - /// successor's start proves it cannot run past it. The bound is the *immediate* + /// own last timestamp says where its last frame *starts* presenting, not where it ends; + /// only its successor's start proves it cannot run past it. The bound is the *immediate* /// servable successor's start, so an unstamped successor leaves the reach unbounded /// until it presents its first frame, and so does having no successor in `servable` - /// at all. + /// at all. Without one, no edge sits above the group to convict it in media time; a + /// whole group ages on the wall clock instead (see [`Self::ended_at`]). fn reach(&self, key: u64, servable: &Servable) -> Option { - if let Some(end) = self.own_end(key) { - return Some(end); - } self.started_at(self.successor(key, servable)?) } @@ -2312,18 +2310,13 @@ impl TrackState { } } - /// Where group `key` ends, once its producer finished it whole: what bounds its reach - /// without a successor (see [`group::Producer::finish_at`]). - fn own_end(&self, key: u64) -> Option { - self.presented(self.lookup.get(&key)?.group.end()?) - } - /// When group `key`, finished whole, stops presenting on the wall clock: its arrival plus - /// its span, which stands in for a successor arriving, so the last group of a paused or - /// finished track ages out without one. + /// the span from its first frame's timestamp to how far its frames present, which stands + /// in for a successor arriving, so the last group of a paused or finished track ages out + /// without one (see [`group::Producer::finish_whole`]). fn ended_at(&self, key: u64) -> Option { let slot = self.lookup.get(&key)?; - let span = self.presented(slot.group.end()?.checked_sub(key)?)?; + let span = slot.group.whole_reach()?.checked_sub(slot.group.timestamp()?).ok()?; slot.arrived.checked_add(age(span)?) } @@ -2423,16 +2416,15 @@ impl TrackState { /// delivering, so a group is abandoned only once everything it could still present /// falls outside the budget. /// - /// A group's reach is bounded by its own end once finished whole, else by its immediate - /// successor (see [`Self::reach`]): it cannot present past where the next group begins. - /// The candidate itself needs no timestamp: an empty group is bounded by its stamped - /// successor the same way. + /// A group's reach is bounded by its immediate successor (see [`Self::reach`]): it cannot + /// present past where the next group begins. The candidate itself needs no timestamp: an + /// empty group is bounded by its stamped successor the same way. /// /// The wall clock judges it too: a group is also stale once its successor arrived a /// full `budget` ago (see [`Self::superseded`]), so a peer whose timestamps stop /// advancing or an empty successor that never presents cannot pin a superseded group. /// Media time still convicts first when a burst arrives at once. A group finished whole - /// ages from its arrival plus its span instead when no successor arrived, so the last + /// ages from its arrival plus its frames' span instead when no successor arrived, so the last /// group of a paused or finished track ages out; one with neither is never convicted. /// /// The bound is exclusive, so the comparison is `>=` rather than `>`: the freshest frame @@ -2494,7 +2486,7 @@ impl TrackState { // is observed or wakes us, and armed for whichever successor this pass selected, so // re-selecting re-arms it. let arrival = self.watch_abort(waiter, |state| state.arrival(key, servable)); - // A group finished whole ages from its own end without one; see [`Self::ended_at`]. + // A group finished whole ages from its own frames without one; see [`Self::ended_at`]. let Some(arrived) = self .superseded_at(arrival, servable, superseded) .or_else(|| self.ended_at(key)) @@ -2508,9 +2500,6 @@ impl TrackState { return true; } let reach = loop { - if let Some(end) = self.own_end(key) { - break end; - } // An abort can change reach even after the successor is stamped. Unbounded // until a group shows above, or the successor presents its first frame. let Some(successor) = self.watch_abort(waiter, |state| state.successor(key, servable)) else { @@ -14864,26 +14853,59 @@ mod test { assert!(!subscriber.is_skipped(20..30)); } - /// A group finished whole ages from its own end, which a zero budget waits for too: a - /// fetch before the end still gets it, and one after is told it is old. + /// The last group, finished whole, ages from one mean frame interval past its highest frame + /// timestamp, which a zero budget waits for too: a fetch before it still gets the group, and + /// one after is told it is old. A group only closed may lack later frames, so it never ages + /// without a successor. #[kio::test] #[cfg(feature = "async")] - async fn a_whole_group_ages_only_once_it_ends() { + async fn a_whole_group_ages_from_its_highest_frame() { + for whole in [true, false] { + let producer = track_producer("test", Info::timed().with_max_age(Duration::ZERO)); + elapse(&producer, Duration::ZERO); + let mut group = producer.append_group(Timestamp::from_millis(0).unwrap()).unwrap(); + // Out of order, as B-frames are: the highest is not the last written. + for at in [0, 40, 20] { + group + .write_frame(Timestamp::from_millis(at).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + } + match whole { + true => group.finish_whole().unwrap(), + false => group.finish().unwrap(), + }; + + let consumer = producer.consume(); + // Three frames over 40ms: a 20ms mean interval, so it presents until 60ms. + elapse(&producer, Duration::from_millis(59)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(fetched.is_ok(), "whole={whole}, before its end: {:?}", fetched.err()); + elapse(&producer, Duration::from_millis(1)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + match whole { + true => assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()), + false => assert!(fetched.is_ok(), "a closed group aged: {:?}", fetched.err()), + } + } + } + + /// A whole group whose frames measure no interval, one frame here, says nothing about how + /// long it presents, so like a closed one it never ages without a successor: a reader still + /// subscribed reads it however late, rather than losing it the moment it finishes. + #[kio::test] + #[cfg(feature = "async")] + async fn a_whole_group_of_one_frame_never_ages_alone() { let producer = track_producer("test", Info::timed().with_max_age(Duration::ZERO)); elapse(&producer, Duration::ZERO); let mut group = producer.append_group(Timestamp::from_millis(0).unwrap()).unwrap(); group .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) .unwrap(); - group.finish_at(40).unwrap(); + group.finish_whole().unwrap(); - let consumer = producer.consume(); - elapse(&producer, Duration::from_millis(39)); - let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); - assert!(fetched.is_ok(), "before its end: {:?}", fetched.err()); - elapse(&producer, Duration::from_millis(1)); - let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); - assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()); + elapse(&producer, Duration::from_secs(60)); + let fetched = producer.consume().fetch_group(0, None).now_or_never().unwrap(); + assert!(fetched.is_ok(), "{:?}", fetched.err()); } /// A successor whose header landed but whose first frame never does still starts the @@ -19131,11 +19153,11 @@ mod test { )); } - /// A group ended by `finish` reads whole but says nothing about where it ends; `finish_at` - /// declares the end a media reader checks against the next group. + /// A group ended by `finish` reads every frame but does not say it is whole; + /// `finish_whole` does. #[kio::test] #[cfg(feature = "async")] - async fn a_finished_group_declares_no_end() { + async fn a_finished_group_is_not_whole() { let producer = track_producer("test", Info::timed()); let mut open = producer.append_next().unwrap(); open.write_frame(Timestamp::from_millis(0).unwrap(), &b"a"[..]).unwrap(); @@ -19146,15 +19168,15 @@ mod test { ended .write_frame(Timestamp::from_millis(20).unwrap(), &b"c"[..]) .unwrap(); - ended.finish_at(30).unwrap(); + ended.finish_whole().unwrap(); let mut reader = producer.subscribe(replay().with_range(0..)); let mut first = reader.assert_group(); assert_eq!(first.read_frame().await.unwrap().unwrap().payload, &b"a"[..]); assert_eq!(first.read_frame().await.unwrap().unwrap().payload, &b"b"[..]); assert!(first.read_frame().await.unwrap().is_none(), "every frame is read"); - assert_eq!(first.end(), None); - assert_eq!(reader.assert_group().end(), Some(30)); + assert!(!first.is_whole()); + assert!(reader.assert_group().is_whole()); } #[kio::test] diff --git a/rs/moq-net/src/recv.rs b/rs/moq-net/src/recv.rs index 02cc0582736..ff7fa2cc71f 100644 --- a/rs/moq-net/src/recv.rs +++ b/rs/moq-net/src/recv.rs @@ -97,7 +97,7 @@ impl Group { self.writer.take().expect("group owned until consumed") } - /// Finish receiving a group ended without saying where it ends, as a bare FIN does, + /// Finish receiving a group closed without saying it is whole, as a bare FIN does, /// without running cancellation cleanup. pub fn finish(mut self) -> Result<(), Error> { self.writer.take().expect("group owned until consumed").finish()?; @@ -105,29 +105,15 @@ impl Group { Ok(()) } - /// Finish receiving a whole group that ends at `end`, as GROUP_END says, without running - /// cancellation cleanup. An `end` the group refuses (see [`group::Producer::finish_at`]) is - /// [`Error::MalformedTrack`], which aborts the group; one already aborted stays so. - pub fn finish_at(mut self, end: u64) -> Result<(), Error> { - if self.check_end(end).is_err() { - let _ = self.abort(Error::MalformedTrack); - return Err(Error::MalformedTrack); - } - let _ = self.writer.take().expect("group owned until consumed").finish_at(end); + /// Finish receiving as the stream ended: whole after a GROUP_END, else only closed. A group + /// already aborted stays so. + pub fn end(mut self, whole: bool) { + let writer = self.writer.take().expect("group owned until consumed"); self.armed = false; - Ok(()) - } - - /// Finish receiving as the stream ended: whole at a GROUP_END's `end`, else without one. - /// Only an `end` the group refuses fails (see [`Self::finish_at`]). - pub fn end(self, end: Option) -> Result<(), Error> { - match end { - Some(end) => self.finish_at(end), - None => { - let _ = self.finish(); - Ok(()) - } - } + let _ = match whole { + true => writer.finish_whole(), + false => writer.finish(), + }; } /// Abort with the receive error instead of cancellation. diff --git a/rs/moq-net/tests/integration/group_end.rs b/rs/moq-net/tests/integration/group_end.rs index 487f84aa66a..0aa3ed1c097 100644 --- a/rs/moq-net/tests/integration/group_end.rs +++ b/rs/moq-net/tests/integration/group_end.rs @@ -1,8 +1,8 @@ #![cfg(feature = "async")] -//! A group finished whole reaches the next hop with its end, over a subscription and over a -//! fetch, and a closed one without it. Lite-07 says which with GROUP_END; earlier versions -//! end every group with a bare FIN, so each arrives closed. +//! A group finished whole reaches the next hop whole, over a subscription and over a fetch, +//! and a closed one only closed. Lite-07 says which with GROUP_END; earlier versions end every +//! group with a bare FIN, so each arrives closed. use crate::support; @@ -23,7 +23,7 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -/// Group 0 finished at 500ms and group 1000 closed, on a live track that keeps both. +/// Group 0 finished whole and group 1000 closed, on a live track that keeps both. fn publish(origin: &moq_net::origin::Producer) -> (moq_net::origin::Route, moq_net::track::Producer) { let broadcast = origin.route("room", moq_net::origin::RouteOptions::hidden()).unwrap(); let track = broadcast @@ -34,31 +34,31 @@ fn publish(origin: &moq_net::origin::Producer) -> (moq_net::origin::Route, moq_n ) .unwrap(); broadcast.announce().unwrap(); - for (start, end) in [(0, Some(500)), (1000, None)] { + for (start, whole) in [(0, true), (1000, false)] { let mut group = track.append_group(Timestamp::from_millis(start).unwrap()).unwrap(); group .write_frame(Timestamp::from_millis(start).unwrap(), b"x".as_slice()) .unwrap(); - match end { - Some(end) => group.finish_at(end).unwrap(), - None => group.finish().unwrap(), + match whole { + true => group.finish_whole().unwrap(), + false => group.finish().unwrap(), }; } (broadcast, track) } -/// Read `group` to its end, returning where it says it ends. -async fn read_to_end(mut group: moq_net::group::Consumer) -> Option { +/// Read `group` to its end, returning whether it says it is whole. +async fn read_to_end(mut group: moq_net::group::Consumer) -> bool { while kio::clock::timeout(TIMEOUT, group.read_frame()) .await .expect("read timeout") .unwrap() .is_some() {} - group.end() + group.is_whole() } -async fn round(version: &str) -> (Vec<(u64, Option)>, Option) { +async fn round(version: &str) -> (Vec<(u64, bool)>, bool) { let publisher = produce_origin(1); let (_broadcast, _track) = publish(&publisher); let client = produce_origin(2); @@ -120,14 +120,11 @@ async fn round(version: &str) -> (Vec<(u64, Option)>, Option) { } #[kio::test] -async fn a_finished_group_arrives_with_its_end() { +async fn a_whole_group_arrives_whole() { for version in VERSIONS { let (subscribed, fetched) = round(version).await; - let whole = match *version { - "moq-lite-07-wip" => Some(500), - _ => None, - }; - assert_eq!(subscribed, [(0, whole), (1000, None)], "{version}: subscribed"); + let whole = *version == "moq-lite-07-wip"; + assert_eq!(subscribed, [(0, whole), (1000, false)], "{version}: subscribed"); assert_eq!(fetched, whole, "{version}: fetched"); } } diff --git a/rs/moq-net/tests/integration/partial_group.rs b/rs/moq-net/tests/integration/partial_group.rs index 1b7c7c9b6a5..2edcd98369e 100644 --- a/rs/moq-net/tests/integration/partial_group.rs +++ b/rs/moq-net/tests/integration/partial_group.rs @@ -29,8 +29,8 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -/// How a reader ended a group: the read's outcome, and the end the group was finished at. -type Ended = (Result, moq_net::Error>, Option); +/// How a reader ended a group: the read's outcome, and whether the group was finished whole. +type Ended = (Result, moq_net::Error>, bool); /// Publish an unfinished group 0 that stalls past the budget after its only frame, and /// return how a reader without a budget of its own ends group 0 at the downstream hop: @@ -125,7 +125,7 @@ async fn round(version: &str) -> Option { } }; stalled.abort(moq_net::Error::Cancel).unwrap(); - Some((end, first.end())) + Some((end, first.is_whole())) } #[kio::test] @@ -133,11 +133,11 @@ async fn a_stalled_group_is_never_cached_downstream_as_finished() { let mut failures = Vec::new(); for version in VERSIONS { let ended = round(version).await; - // Lite-07 says a group is whole with its end, which a closed one never has; earlier + // Lite-07 says a group is whole with GROUP_END, which a closed one never has; earlier // versions can't tell a closed group from a whole one, so only an error is safe. let finished = matches!( (&ended, version == &"moq-lite-07-wip"), - (Some((_, Some(_))), _) | (Some((Ok(_), None)), false) + (Some((_, true)), _) | (Some((Ok(_), false)), false) ); if finished { failures.push(format!("{version}: the downstream hop finished group 0: {ended:?}")); From 33037e0059b2996f390f438e452f5764a9dc6423 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 21:12:58 -0800 Subject: [PATCH 3/6] chore: format and refresh the @moq/net API report Co-Authored-By: Claude Opus 5.5 --- js/net/api/index.api.md | 4 ++-- js/net/src/lite/publisher.ts | 3 ++- rs/moq-mux/src/start.rs | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 096ceaada06..ae58bc83a71 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -400,7 +400,6 @@ export namespace Group { close(): void; get closed(): Signals.GetPromise; get done(): boolean; - get end(): bigint | undefined; get frameCount(): number; nextFrame(): Promise; readFrame(): Promise; @@ -415,6 +414,7 @@ export namespace Group { tryReadFrameIndex(): { index: number; } & Group.Frame | undefined; + get whole(): boolean; } export interface Finished { readonly closed: Signals.GetPromise; @@ -436,7 +436,7 @@ export namespace Group { consume(): Group.Consumer; createFrame(info: Frame.Info): Frame.Producer; finish(): Group.Finished; - finishAt(end: bigint): Group.Finished; + finishWhole(): Group.Finished; readonly start: bigint; writeFrame(frame: Group.Frame): void; } diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 73a6acc649a..c4f86da7efa 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -2638,7 +2638,8 @@ export class Publisher { } // Lite-07 ends a whole group with GROUP_END. - if (hasGroupEnd(this.version) && whole && group.whole) await write(() => stream.write(groupEnd(stream.version))); + if (hasGroupEnd(this.version) && whole && group.whole) + await write(() => stream.write(groupEnd(stream.version))); stream.close(); group.close(); // A stream that fails after its FIN may have lost its header, so report it for a DROP. diff --git a/rs/moq-mux/src/start.rs b/rs/moq-mux/src/start.rs index e29dc1bdde6..e4c39773fb3 100644 --- a/rs/moq-mux/src/start.rs +++ b/rs/moq-mux/src/start.rs @@ -73,4 +73,3 @@ pub(crate) fn after(track: &moq_net::track::Producer, timestamp: moq_net::Timest (start, _, _) => Ok(start.value()), } } - From d81ddb1a3015f678379ea0ba34f97ade32cf31ae Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 21:20:15 -0800 Subject: [PATCH 4/6] test(interop): the lite varint fixture tells GROUP_END by its whole marker Co-Authored-By: Claude Opus 5.5 --- test/interop/lite-varint.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/interop/lite-varint.ts b/test/interop/lite-varint.ts index 6240888baec..be6df677748 100644 --- a/test/interop/lite-varint.ts +++ b/test/interop/lite-varint.ts @@ -82,7 +82,7 @@ for (;;) { const frame = await g.decodeMaybe(decode); if (!frame) break; // Only a length-first frame can be a GROUP_END, and the fixture's frames are not. - if ("end" in frame) throw new Error("a fixture frame decoded as GROUP_END"); + if ("whole" in frame) throw new Error("a fixture frame decoded as GROUP_END"); if (!frame.timestamp) throw new Error("a lite-05+ frame decoded without its timestamp"); // The fixture's frames fit one segment, so each payload decodes with its header. if (!frame.payload) throw new Error("a fixture frame streamed past its header"); @@ -109,7 +109,7 @@ if (untimed) { for (;;) { const frame = await u.decodeMaybe(frameDecoder({ scale: 0 })); if (!frame) break; - if ("end" in frame) throw new Error("a fixture frame decoded as GROUP_END"); + if ("whole" in frame) throw new Error("a fixture frame decoded as GROUP_END"); assert.equal(frame.timestamp, undefined); if (!frame.payload) throw new Error("a fixture frame streamed past its header"); untimedFrames.push(frame.payload); From c89a4b4616a48ca4cadea9466217e06dab16add1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 21:56:23 -0800 Subject: [PATCH 5/6] fix(net): only a copy held from frame 0 ages from its own frames, and JS measures the span without building the reach Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.test.ts | 2 +- js/net/src/track.ts | 12 +++++++----- rs/moq-net/src/model/group.rs | 10 ++++++---- rs/moq-net/src/model/track.rs | 27 ++++++++++++++++++++++++++- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index f099d38a52b..a01e1147546 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2102,7 +2102,7 @@ test("a whole group of one frame never ages alone", () => { const group = producer.appendGroup(Timestamp.fromMillis(0)); group.writeFrame(textFrame("x", Timestamp.fromMillis(0))); group.finishWhole(); - clock.set(70_000); + clock.set(11_000); expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual([0n]); } finally { producer.finish(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index ae0471a9b0a..00082ce126b 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -54,7 +54,7 @@ import { } from "./internal.ts"; import { grace, Ledger } from "./ledger.ts"; import { rescale } from "./rescale.ts"; -import { floorAt, Timescale, Timestamp } from "./time.ts"; +import { floorAt, Timescale, type Timestamp } from "./time.ts"; import { checkEnd, checkKey, max, min, UNBOUNDED } from "./util/key.ts"; import { checkName } from "./util/utf8.ts"; import { bounded, type Holdings, joinsLive, rangesOf, UNHELD, Window, wireSpan } from "./window.ts"; @@ -3306,14 +3306,16 @@ export class Subscriber { #endedAt(key: bigint): Instant | undefined { const group = this.#held(key); if (!group?.whole || group.frameCount < 2) return undefined; + // Undefined for a copy not held from frame 0: a tail's first timestamp is not the group's. const start = hooks.groupTimestamp(group); const latest = hooks.groupLatest(group); - if (start === undefined || latest?.scale === undefined) return undefined; - // In the track's units, as Rust divides them. + const scale = latest?.scale; + if (start === undefined || latest === undefined || scale === undefined) return undefined; + // In the track's units, as Rust divides them, and as a span: the reach itself may not fit. const interval = Math.floor((latest.value - start.value) / (group.frameCount - 1)); if (interval <= 0) return undefined; - const reach = new Timestamp(latest.value + interval, latest.scale); - return hooks.groupArrived(group).checkedAdd(Duration.millis(reach.asMillis() - start.asMillis())); + const span = ((latest.value - start.value + interval) * 1000) / scale; + return hooks.groupArrived(group).checkedAdd(Duration.millis(span)); } // When the wall clock starts: the cached `arrival`'s arrival, or that of the one `held` diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 9e70d6d074b..9360aac9632 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3125,9 +3125,11 @@ impl Handle { /// How far a group finished whole presents: one frame past its highest frame timestamp, a /// frame being its mean interval from its first frame's timestamp to its highest. `None` - /// when it is not whole, or when its frames measure no interval (one frame, or all at one - /// timestamp). Only a whole group's frames prove where it stops, and they are the same at - /// every copy, so its reach never depends on which copy judges it. + /// when it is not whole, when this copy does not hold it from frame 0 (a tail's first + /// timestamp is not the group's), or when its frames measure no interval (one frame, or + /// all at one timestamp). Only a whole group's frames prove where it stops, and they are + /// the same at every copy holding them all, so its reach never depends on which copy + /// judges it. pub(super) fn whole_reach(&self) -> Option { let state = self.state.read(); let Outcome::Finished(frames, true) = state.outcome else { @@ -3136,7 +3138,7 @@ impl Handle { let Timeline::Timed { start, latest } = state.timeline else { return None; }; - if state.origin == Origin::Unverified { + if state.origin == Origin::Unverified || state.offset != 0 { return None; } let gaps = u64::try_from(frames.checked_sub(1)?).ok()?; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 92d1d5e1f97..d367132d5a9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -14903,11 +14903,36 @@ mod test { .unwrap(); group.finish_whole().unwrap(); - elapse(&producer, Duration::from_secs(60)); + elapse(&producer, Duration::from_secs(1)); let fetched = producer.consume().fetch_group(0, None).now_or_never().unwrap(); assert!(fetched.is_ok(), "{:?}", fetched.err()); } + /// A whole copy received from past frame 0 holds only its tail, whose first timestamp is + /// not the group's, so it measures no interval and never ages without a successor. + #[kio::test] + #[cfg(feature = "async")] + async fn a_whole_tail_never_ages_alone() { + let producer = track_producer("test", Info::timed().with_max_age(Duration::ZERO)); + elapse(&producer, Duration::ZERO); + let Received::Group(mut tail) = producer.create_group_at(group::Info { start: 0 }, 2).unwrap() else { + panic!("nothing else holds group 0"); + }; + for at in [20, 30] { + tail.write_frame(Timestamp::from_millis(at).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + } + tail.finish_whole().unwrap(); + + elapse(&producer, Duration::from_secs(1)); + let from = group::Fetch { + frame_start: 2, + ..Default::default() + }; + let fetched = producer.consume().fetch_group(0, from).now_or_never().unwrap(); + assert!(fetched.is_ok(), "{:?}", fetched.err()); + } + /// A successor whose header landed but whose first frame never does still starts the /// wall clock, so a relay ages the group before it as the origin does. #[kio::test] From d93249075521d8eae7ace3dc80c5aa14142ae8f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 04:12:22 -0800 Subject: [PATCH 6/6] fix(net): a whole group ages from any copy holding it, rounds its interval up, and stays aged - A whole group's own span comes from whichever copy holds all its frames, so a fetched head beside a live tail ages the group. - The mean frame interval rounds up to a whole unit, so a sub-unit mean still counts. - A whole group ages from the earlier of its successor's arrival and its own end, so a late successor never revives a group its frames already aged out. - The draft says a group's own frames set only its own end (arrival age), never its reach, and lists when a group has no own end. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 8 ++- drafts/draft-lcurley-moq-lite.md | 23 +++--- js/net/src/group.ts | 4 +- js/net/src/ordered.test.ts | 5 +- js/net/src/track.test.ts | 46 ++++++++++++ js/net/src/track.ts | 21 +++--- rs/moq-net/src/model/group.rs | 29 ++++---- rs/moq-net/src/model/track.rs | 119 +++++++++++++++++++++++++++---- 8 files changed, 202 insertions(+), 53 deletions(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 47330d6f1e5..0596fa6fb17 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -342,9 +342,11 @@ subscription there starts at the live group. A group ends one of three ways: finished, so it is whole; closed, so every frame written arrives but it may have held more, as when a relay stops sending -a group that went stale; or aborted with an error. A finished group with no -successor ages from one frame past its highest frame timestamp, so the last -group of a paused track still ages out. Over the work-in-progress moq-lite 07 each group also +a group that went stale; or aborted with an error. A finished group also +ages from one frame past its highest frame timestamp, a frame being its mean +interval, so the last group of a paused track still ages out. That needs a +timed track, at least two frame timestamps apart, and the group held from its +first frame; otherwise it ages only once a successor arrives. Over the work-in-progress moq-lite 07 each group also names the start of the group before it, so a reader sees a gap at once; older moq-lite versions say neither whether a group is whole nor the previous start. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 39dde26cdca..eb4b81522c0 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -820,7 +820,7 @@ The latest group, while it has not finished, has no successor, so it is never ex A group is expired once either of two ages meets the relevant `Max Age`: - Its **timestamp age** is the difference between the *newest* frame timestamp of the latest group that has at least one frame, and this group's **reach**, defined below. -- Its **arrival age** is the time elapsed since its successor arrived at this hop. A finished group with no successor ages from its own reach instead: the time elapsed since it arrived at this hop, less the span from its first frame timestamp to its reach. +- Its **arrival age** is the time elapsed since its successor arrived at this hop. A finished group also ages from its **own end**, defined below: the time elapsed since it arrived at this hop, less the span from its first frame timestamp to its own end. When both exist, the greater age counts, so a successor arriving late never makes an expired group fresh again. On a scale-less Track, timestamp age counts bare units as the milliseconds of a Max Age. @@ -830,13 +830,20 @@ Being behind is not itself a reason to expire anything. [Prioritization](#prioritization) already transmits newer groups first, so an older group consumes only whatever capacity is left over; it therefore closes the gap faster than the live edge advances, and a receiver that is behind can converge without losing content. What cannot be recovered is a group with nothing left worth delivering, and that is what expiration removes. -A group's **reach** is the first frame timestamp of the next group by start, and otherwise, once it finished, one frame past its own highest frame timestamp, or unbounded when neither exists. +A group's **reach** is the first frame timestamp of the next group by start, or unbounded when there is none. A group cannot present past where its successor begins, so this is the furthest it could still reach. -A finished group with no successor holds every frame it will ever have, so its own frames stand in. -Frame durations are not carried on the wire, so its highest frame timestamp is where its last frame *starts* presenting; one frame is its mean interval instead, the span from its first frame timestamp to its highest divided by one less than its frame count. -A finished group whose frames measure no interval, one frame or all at one timestamp, has no reach without a successor, so it never expires before one arrives. -A closed group may have held more frames than arrived, so its frames do not bound it. -An unfinished or closed group whose successor has not presented a frame therefore has no timestamp age, because nothing yet proves where it stops. +A group whose successor has not presented a frame therefore has no timestamp age, because nothing yet proves where it stops. +Its own frames never set its reach: without a successor, no newer frame exists to measure its timestamp age against. + +A finished group's **own end** is one frame past its highest frame timestamp. +It holds every frame it will ever have, so its frames prove where it stops. +Frame durations are not carried on the wire, so its highest frame timestamp is where its last frame *starts* presenting; one frame is its mean interval instead, the span from its first frame timestamp to its highest divided by one less than its frame count, rounded up to a whole unit of the Track's timescale. +A group has no own end, and so ages only from a successor, when any of these holds: + +- It is closed rather than finished: it may have held more frames than arrived. +- Its frames span no time: one frame, or all at one timestamp. +- This hop holds it only from a later frame, as after a resumed subscription: that copy's first frame timestamp is not the group's. +- Its Track has no timescale, so its frames carry no timestamps. The group itself needs no timestamp: a zero-frame group (a keep-alive or gap marker) is bounded by its successor's first frame the same way. Reach is an exclusive bound, so a timestamp age **equal** to Max Age already expires the group: the freshest frame it could still hold sits strictly below its reach, and is therefore strictly older than the budget. @@ -2368,7 +2375,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - Added a leading Type to datagrams, 0x0 for a Track Datagram; a receiver discards one of an unknown Type. - FRAME and datagrams always carry a timestamp, a bare number on a scale-less Track. - Replaced GROUP's `Group Sequence` with `Group Start`, added `Previous Start`, the start of the Group before it, so a subscriber sees the gap before a Group without a SUBSCRIBE_SKIP. -- Added GROUP_END, a `Payload Length` of 0: a Group or Fetch Stream ends with it when its Group finished whole, a FIN alone ends a closed Group whose frames may not all exist, and a reset aborts. A finished Group with no successor expires one mean frame interval past its own highest frame timestamp. +- Added GROUP_END, a `Payload Length` of 0: a Group or Fetch Stream ends with it when its Group finished whole, a FIN alone ends a closed Group whose frames may not all exist, and a reset aborts. A finished Group also ages from its own end, one mean frame interval past its highest frame timestamp, whether or not a successor has arrived. - Expiration also measures a group's arrival age, the time since its successor arrived at this hop. - A relay closes a Group it holds only the start of, without a GROUP_END. A publisher resets the Group Stream of a group that expires with OLD, even one whose GROUP_END it sent but is not yet acknowledged, and names the group in a SUBSCRIBE_DROP with OLD. - FRAME leads with `Payload Length`, the payload size plus one, in place of the trailing `Message Length`; 0 is malformed. diff --git a/js/net/src/group.ts b/js/net/src/group.ts index a6c062fbe46..b0b4b999dfc 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -725,8 +725,8 @@ export class Producer { /** * Ends the group whole, returning it as {@link Finished}: it holds every frame it will ever - * have. A whole group with no successor ages from one mean frame interval past its highest - * frame timestamp, as the Rust `group::Producer::finish_whole`. + * have. A whole group also ages from one mean frame interval past its highest frame + * timestamp, so it ages out without a successor, as the Rust `group::Producer::finish_whole`. */ finishWhole(): Finished { if (this.#state.closed.peek() === undefined) this.#state.whole = true; diff --git a/js/net/src/ordered.test.ts b/js/net/src/ordered.test.ts index 4b7072b8043..780193c7d47 100644 --- a/js/net/src/ordered.test.ts +++ b/js/net/src/ordered.test.ts @@ -501,8 +501,9 @@ test("an ordered read whose window ends past a group it passed as stale reports withClock((clock) => { const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); const first = producer.createGroup(0n); - first.writeFrame(textFrame("x", Timestamp.fromMillis(0))); - first.finishAt(10n); + // Two frames 5ms apart: whole, it presents until 10ms. + for (const millis of [0, 5]) first.writeFrame(textFrame("x", Timestamp.fromMillis(millis))); + first.finishWhole(); clock.jump(Duration.millis(100)); put(producer, 100n); const ordered = producer diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index a01e1147546..13c0e13809d 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2092,6 +2092,52 @@ test.each([true, false])("a last group finished whole=%s ages from its highest f } }); +// A mean frame interval below one unit rounds up rather than to zero: three frames over one second, +// on a track counting whole seconds, still present until 2s. Mirrors the Rust +// `a_whole_group_rounds_a_sub_unit_interval_up`. +test("a whole group rounds a sub-unit interval up", () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.SECOND, maxAge: Duration.ZERO }); + try { + const group = producer.appendGroup(new Timestamp(0, Timescale.SECOND)); + for (const secs of [0, 0, 1]) group.writeFrame(textFrame("x", new Timestamp(secs, Timescale.SECOND))); + group.finishWhole(); + + clock.set(11_999); + expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual([0n]); + clock.set(12_000); + expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual([]); + } finally { + producer.finish(); + producer.close(); + clock.installed.close(); + } +}); + +// A successor landing after a whole group aged out on its own frames does not restart its clock: +// the group stays old rather than coming back for another budget. Mirrors the Rust +// `a_late_successor_never_revives_a_whole_group`. +test("a late successor never revives a whole group", () => { + const clock = manualTime(10_000); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: Duration.millis(10) }); + try { + const group = producer.appendGroup(Timestamp.fromMillis(0)); + for (const millis of [0, 20, 40]) group.writeFrame(textFrame("x", Timestamp.fromMillis(millis))); + group.finishWhole(); + + // It presents until 60ms, so a 10ms budget ages it at 70ms; the successor lands later. + clock.set(10_100); + const next = producer.appendGroup(Timestamp.fromMillis(100)); + next.writeFrame(textFrame("x", Timestamp.fromMillis(100))); + next.finish(); + expect(drain(producer.subscribe({ delay: { max: Duration.millis(1000) } }))).toEqual([100n]); + } finally { + producer.finish(); + producer.close(); + clock.installed.close(); + } +}); + // A whole group whose frames measure no interval, one frame here, says nothing about how long it // presents, so like a closed one it never ages without a successor. Mirrors the Rust // `a_whole_group_of_one_frame_never_ages_alone`. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index f6b30d484f8..35321e1ec61 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -3277,8 +3277,10 @@ export class Subscriber { let arrived = this.#arrivals ? this.#keptSupersededAt(key, drift, held) : this.#supersededAt(arrival, drift, held); - // A group finished whole ages from its own frames without a successor. - if (arrived === undefined && !this.#arrivals) arrived = this.#endedAt(key); + // A group finished whole ages from its own frames, or its successor's arrival if earlier: a + // late successor never revives a group its frames already aged out. + const ended = this.#arrivals ? undefined : this.#endedAt(key); + if (ended !== undefined) arrived = arrived?.min(ended) ?? ended; if (arrived === undefined) return false; // Plain arithmetic: an unenforced budget is infinite, which no duration brand holds. const deadline = arrived.since(Instant.ZERO).asNanos() + drift.budget * 1_000_000; @@ -3326,11 +3328,11 @@ export class Subscriber { // When group `key`, finished whole, stops presenting on the wall clock: its arrival plus the // span from its first frame's timestamp to one frame past its highest, a frame being its mean - // interval, which stands in for a successor arriving, so the last group of a paused or finished - // track ages out without one. Undefined when it is not whole, or its frames measure no interval - // (one frame, or all at one timestamp). Only a whole group's frames prove where it stops, and + // interval rounded up to a whole unit, which stands in for a successor arriving, so the last + // group of a paused or finished track ages out without one. Undefined when it is not whole, or + // its frames span no time (one frame, or all at one timestamp). Only a whole group's frames prove where it stops, and // they are the same at every copy. Mirrors the Rust `TrackState::ended_at` and - // `Handle::whole_reach`. + // `Handle::whole_span`. #endedAt(key: bigint): Instant | undefined { const group = this.#held(key); if (!group?.whole || group.frameCount < 2) return undefined; @@ -3340,9 +3342,10 @@ export class Subscriber { const scale = latest?.scale; if (start === undefined || latest === undefined || scale === undefined) return undefined; // In the track's units, as Rust divides them, and as a span: the reach itself may not fit. - const interval = Math.floor((latest.value - start.value) / (group.frameCount - 1)); - if (interval <= 0) return undefined; - const span = ((latest.value - start.value + interval) * 1000) / scale; + // Rounded up, so a mean below one unit still counts and the bound stays an upper one. + const frames = latest.value - start.value; + if (frames <= 0) return undefined; + const span = ((frames + Math.ceil(frames / (group.frameCount - 1))) * 1000) / scale; return hooks.groupArrived(group).checkedAdd(Duration.millis(span)); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 9360aac9632..1ac7b9ca686 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -2284,10 +2284,9 @@ impl Producer { /// End the group whole, returning the [`Finished`] group: it holds every frame it will /// ever have, so no more can be written, but it can still be read or aborted. /// - /// A whole group with no successor ages from one mean frame interval past its highest frame - /// timestamp, so the last group of a paused or finished track ages out (see - /// [`track::Info::max_age`]). Fails - /// with the abort error if the group was already aborted. + /// A whole group also ages from one mean frame interval past its highest frame timestamp, + /// so the last group of a paused or finished track ages out without a successor (see + /// [`track::Info::max_age`]). Fails with the abort error if the group was already aborted. pub fn finish_whole(self) -> Result { self.end(true) } @@ -3123,14 +3122,13 @@ impl Handle { } } - /// How far a group finished whole presents: one frame past its highest frame timestamp, a - /// frame being its mean interval from its first frame's timestamp to its highest. `None` - /// when it is not whole, when this copy does not hold it from frame 0 (a tail's first - /// timestamp is not the group's), or when its frames measure no interval (one frame, or - /// all at one timestamp). Only a whole group's frames prove where it stops, and they are - /// the same at every copy holding them all, so its reach never depends on which copy - /// judges it. - pub(super) fn whole_reach(&self) -> Option { + /// How long a group finished whole presents, from its first frame's timestamp to one frame + /// past its highest, a frame being its mean interval rounded up to a whole unit. `None` when + /// it is not whole, when this copy does not hold it from frame 0 (a tail's first timestamp + /// is not the group's), or when its frames span no time (one frame, or all at one + /// timestamp). Only a whole group's frames prove where it stops, and they are the same at + /// every copy holding them all, so its span never depends on which copy judges it. + pub(super) fn whole_span(&self) -> Option { let state = self.state.read(); let Outcome::Finished(frames, true) = state.outcome else { return None; @@ -3142,11 +3140,12 @@ impl Handle { return None; } let gaps = u64::try_from(frames.checked_sub(1)?).ok()?; - let interval = latest.value().checked_sub(start.value())?.checked_div(gaps)?; - if interval == 0 { + let span = latest.value().checked_sub(start.value())?; + if gaps == 0 || span == 0 { return None; } - latest.checked_add(Timestamp::new(interval, latest.scale()?).ok()?).ok() + // Rounded up, so a mean below one unit still counts and the bound stays an upper one. + Timestamp::new(span.checked_add(span.div_ceil(gaps))?, latest.scale()?).ok() } /// The group's full cached footprint (payload plus fixed overhead, and a compressed diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 97a139cdec2..a2a2e92080c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2323,12 +2323,13 @@ impl TrackState { } /// When group `key`, finished whole, stops presenting on the wall clock: its arrival plus - /// the span from its first frame's timestamp to how far its frames present, which stands + /// how long a copy holding all its frames presents (a fetched head may, beside a live tail), + /// which stands /// in for a successor arriving, so the last group of a paused or finished track ages out /// without one (see [`group::Producer::finish_whole`]). fn ended_at(&self, key: u64) -> Option { let slot = self.lookup.get(&key)?; - let span = slot.group.whole_reach()?.checked_sub(slot.group.timestamp()?).ok()?; + let span = slot.copies().find_map(group::Handle::whole_span)?; slot.arrived.checked_add(age(span)?) } @@ -2370,13 +2371,16 @@ impl TrackState { /// /// The wall-clock twin of [`Self::reach`], re-read the same way rather than latched: /// a group that lands between the two restarts the clock, and an aborted successor - /// hands it to the next one. The clock is read only once there is a successor, - /// so judging the newest group, the common case, costs no clock read. + /// hands it to the next one. A group finished whole also ages from its own frames, and + /// the earlier of the two counts. The clock is read only once either exists, so judging + /// the newest unfinished group, the common case, costs no clock read. fn superseded(&self, key: u64, servable: &Servable, budget: Duration) -> bool { - match self.arrival(key, servable) { - Some(successor) => self.aged(successor.arrived, budget), - None => self.ended_at(key).is_some_and(|ended| self.aged(ended, budget)), - } + let successor = self.arrival(key, servable).map(|slot| slot.arrived); + successor + .into_iter() + .chain(self.ended_at(key)) + .min() + .is_some_and(|arrived| self.aged(arrived, budget)) } /// Whether a full `budget` passed since a successor `arrived`, or since a group finished @@ -2498,11 +2502,10 @@ impl TrackState { // is observed or wakes us, and armed for whichever successor this pass selected, so // re-selecting re-arms it. let arrival = self.watch_abort(waiter, |state| state.arrival(key, servable)); - // A group finished whole ages from its own frames without one; see [`Self::ended_at`]. - let Some(arrived) = self - .superseded_at(arrival, servable, superseded) - .or_else(|| self.ended_at(key)) - else { + // A group finished whole ages from its own frames, or its successor's arrival if earlier, + // so a late successor never revives it; see [`Self::ended_at`]. + let successor = self.superseded_at(arrival, servable, superseded); + let Some(arrived) = successor.into_iter().chain(self.ended_at(key)).min() else { return false; }; if let Some(deadline) = arrived.checked_add(budget) { @@ -15088,6 +15091,89 @@ mod test { assert!(fetched.is_ok(), "{:?}", fetched.err()); } + /// A successor landing after a whole group aged out on its own frames does not restart + /// its clock: the group stays old rather than coming back for another budget. + #[kio::test] + #[cfg(feature = "async")] + async fn a_late_successor_never_revives_a_whole_group() { + let producer = track_producer("test", Info::timed().with_max_age(Duration::from_millis(10))); + elapse(&producer, Duration::ZERO); + let mut group = producer.append_group(Timestamp::ZERO).unwrap(); + for at in [0, 20, 40] { + group + .write_frame(Timestamp::from_millis(at).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + } + group.finish_whole().unwrap(); + + // It presents until 60ms, so a 10ms budget ages it at 70ms; the successor lands later. + elapse(&producer, Duration::from_millis(100)); + put(&producer, 100, Some(100)); + let fetched = producer.consume().fetch_group(0, None).now_or_never().unwrap(); + assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()); + } + + /// A mean frame interval below one unit rounds up rather than to zero: three frames over + /// one second, on a track counting whole seconds, still present until 2s. + #[kio::test] + #[cfg(feature = "async")] + async fn a_whole_group_rounds_a_sub_unit_interval_up() { + let info = Info::default() + .with_timescale(Timescale::SECOND) + .with_max_age(Duration::ZERO); + let producer = track_producer("test", info); + elapse(&producer, Duration::ZERO); + let mut group = producer.append_group(Timestamp::ZERO).unwrap(); + for at in [0, 0, 1] { + group + .write_frame(Timestamp::from_secs(at).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + } + group.finish_whole().unwrap(); + + let consumer = producer.consume(); + elapse(&producer, Duration::from_millis(1999)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(fetched.is_ok(), "before its end: {:?}", fetched.err()); + elapse(&producer, Duration::from_millis(1)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()); + } + + /// A live copy fed from frame 1 measures nothing, but a fetched head holding the whole + /// group does, so the group ages from the head's frames as the origin's copy would. + #[kio::test] + #[cfg(feature = "async")] + async fn a_whole_fetched_head_ages_beside_a_live_tail() { + let info = Info::timed().with_max_age(Duration::ZERO); + let producer = track_producer("test", info.clone()); + elapse(&producer, Duration::ZERO); + let dynamic = producer.requests(); + let consumer = producer.consume(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); + live.start_at(1).unwrap(); + live.write_frame(Timestamp::from_millis(20).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + + let fetch = consumer.fetch_group(0, None); + let request = dynamic.requested_fill_group().await.unwrap(); + let mut head = request.accept(info).unwrap().expect("a fresh group"); + for at in [0, 20, 40] { + head.write_frame(Timestamp::from_millis(at).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + } + head.finish_whole().unwrap(); + assert_eq!(fetch.await.unwrap().index(), 0); + + // Three frames over 40ms: a 20ms mean interval, so it presents until 60ms. + elapse(&producer, Duration::from_millis(59)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(fetched.is_ok(), "before its end: {:?}", fetched.err()); + elapse(&producer, Duration::from_millis(1)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()); + } + /// A whole copy received from past frame 0 holds only its tail, whose first timestamp is /// not the group's, so it measures no interval and never ages without a successor. #[kio::test] @@ -21167,7 +21253,12 @@ mod test { let producer = track_producer("test", Info::timed()); elapse(&producer, Duration::ZERO); let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); - group.write_frame(Timestamp::ZERO, &b"x"[..]).unwrap(); + // Two frames 5ms apart: whole, it presents until 10ms. + for at in [0, 5] { + group + .write_frame(Timestamp::from_millis(at).unwrap(), &b"x"[..]) + .unwrap(); + } group.finish_whole().unwrap(); elapse(&producer, Duration::from_millis(100)); put(&producer, 100, Some(100));