diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 0e3d8f638e7..f504ddcd66f 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -14381,6 +14381,7 @@ abstract class MoqTrackProducerInterface { void finishAt({required int finalSequence}); MoqPosition? largest(); MoqTrackRequests requests(); + void skipGroups({required int start, required int end}); void writeFrame({required MoqFrame frame}); } @@ -14518,6 +14519,17 @@ class MoqTrackProducer implements MoqTrackProducerInterface { ); } + void skipGroups({required int start, required int end}) { + return rustCall((status) { + uniffi_moq_ffi_fn_method_moqtrackproducer_skip_groups( + uniffiClonePointer(), + FfiConverterUInt64.lower(start), + FfiConverterUInt64.lower(end), + status, + ); + }, moqExceptionErrorHandler); + } + void writeFrame({required MoqFrame frame}) { return rustCall((status) { uniffi_moq_ffi_fn_method_moqtrackproducer_write_frame( @@ -21940,6 +21952,16 @@ external Pointer uniffi_moq_ffi_fn_method_moqtrackproducer_requests( Pointer uniffiStatus, ); +@Native, Uint64, Uint64, Pointer)>( + assetId: _uniffiAssetId, +) +external void uniffi_moq_ffi_fn_method_moqtrackproducer_skip_groups( + Pointer ptr, + int start, + int end, + Pointer uniffiStatus, +); + @Native, RustBuffer, Pointer)>( assetId: _uniffiAssetId, ) @@ -23748,6 +23770,9 @@ external int uniffi_moq_ffi_checksum_method_moqtrackproducer_largest(); @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqtrackproducer_requests(); +@Native(assetId: _uniffiAssetId) +external int uniffi_moq_ffi_checksum_method_moqtrackproducer_skip_groups(); + @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqtrackproducer_write_frame(); @@ -24635,6 +24660,9 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqtrackproducer_requests() != 60252) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } + if (uniffi_moq_ffi_checksum_method_moqtrackproducer_skip_groups() != 25592) { + throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); + } if (uniffi_moq_ffi_checksum_method_moqtrackproducer_write_frame() != 46891) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 246d26ddd5a..f00ca15f760 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -344,9 +344,9 @@ Sent when resetting a stream (RESET_STREAM), or when refusing to receive one (ST | ------- | ------------- | ----------- | | 0x32 | GROUP_TOO_LARGE | The group grew past the publisher's cache budget and was aborted. | | ------- | ------------- | ----------- | -| 0x33 | NOT_FOUND | The requested group, track, or broadcast is not here. | +| 0x33 | NOT_FOUND | The requested group, track, or broadcast is not here. A sequence its publisher declared it will never produce is named by SUBSCRIBE_SKIP instead; any other sequence a publisher never had, including a gap it never declared, is named with NOT_FOUND unless it is past the subscriber's budget (see [SUBSCRIBE_DROP](#subscribe-drop)). | | ------- | ------------- | ----------- | -| 0x34 | OLD | The group was superseded by a newer group and dropped. | +| 0x34 | OLD | The group was superseded by a newer group and dropped. In SUBSCRIBE_DROP, the next SUBSCRIBE_UPDATE reopens it: the publisher re-offers the group or drops it again. | | ------- | ------------- | ----------- | | 0x35 | EVICTED | The group was dropped under memory pressure. Unlike OLD it was still current, so it can be re-fetched. | | ------- | ------------- | ----------- | @@ -566,16 +566,16 @@ The subscriber needs the track's TRACK_INFO (notably its timescale) to interpret The publisher sends SUBSCRIBE_OK once the start is resolved, and SUBSCRIBE_FINISH as soon as the exclusive final group is known (see [SUBSCRIBE_OK](#subscribe-ok) and [SUBSCRIBE_FINISH](#subscribe-finish)). Neither closes the stream: the publisher keeps reading SUBSCRIBE_UPDATE messages, so an update can still widen the Ranges, including into an ended Track's history, and the publisher serves or drops what it newly covers. -The subscriber judges a Range complete from what it observed: the Groups it received, the SUBSCRIBE_DROP messages, the Range Starts in SUBSCRIBE_OK and SUBSCRIBE_STARTED, and SUBSCRIBE_FINISH. +The subscriber judges a Range complete from what it observed: the Groups it received, the SUBSCRIBE_SKIP and SUBSCRIBE_DROP messages, the Range Starts in SUBSCRIBE_OK and SUBSCRIBE_STARTED, and SUBSCRIBE_FINISH. Drops and Groups name absolute sequences, so the subscriber judges them against its latest Ranges, and an update still in flight changes nothing it has counted. -The publisher never closes its side first: a subscription whose every Range has an `End` other than None and whose every Group has been delivered or dropped stays open for an update to widen it. +The publisher never closes its side first: a subscription whose every Range has an `End` other than None and whose every Group has been delivered, skipped, or dropped stays open for an update to widen it. A subscriber fills Groups it misses by asking for them as Ranges, adding them to an open subscription with SUBSCRIBE_UPDATE or opening a bounded SUBSCRIBE when none is open, rather than with a FETCH of each Group; FETCH remains for one Group, or a slice of one. -A publisher never sends a Group twice on one subscription, so a Group the subscription already delivered or named in SUBSCRIBE_DROP is asked for with a SUBSCRIBE of its own. -The subscriber asks that way, once, for a Group a subscription dropped with OLD that it still wants, with a `Subscriber Max Age` its age does not reach, and never for a Group that may still arrive on the existing subscription. +A publisher never sends a Group twice on one subscription, except to re-offer one whose stream it reset, so a Group the subscription already delivered or named in a final SUBSCRIBE_DROP is asked for with a SUBSCRIBE of its own. +A Group dropped with OLD that the subscriber still wants needs no new SUBSCRIBE: a SUBSCRIBE_UPDATE with a `Subscriber Max Age` its age does not reach reopens it on the same subscription (see [SUBSCRIBE_UPDATE](#subscribe-update)). The subscriber closes its side (FIN) once it wants nothing more from the subscription, and the publisher then closes its side (FIN), cancelling any Group Stream still open. -After SUBSCRIBE_FINISH, the subscriber SHOULD close its side once its tail accounting has settled: every received Group Stream has been read to its FIN, reset, or dropped, and every sequence in range has arrived, been named by SUBSCRIBE_DROP, or exhausted the subscriber's grace period. +After SUBSCRIBE_FINISH, the subscriber SHOULD close its side once its tail accounting has settled: every received Group Stream has been read to its FIN, reset, or dropped, and every sequence in range has arrived, been named by SUBSCRIBE_SKIP or SUBSCRIBE_DROP, or exhausted the subscriber's grace period. A publisher closing its session gracefully MUST wait for the subscriber's FIN or reset on every Subscribe Stream before closing the connection; a transport acknowledgement alone does not confirm the subscriber read the tail. Either endpoint MAY reset the stream at any time. @@ -830,7 +830,8 @@ A frame MAY contain an empty payload, potentially indicating a gap in the group. The GROUP message's `Frame Start` gives the index of the first FRAME on the stream, so a stream carrying only part of a Group is self-describing. It is redundant with the subscription's own start Position in the steady state, and carried anyway because a SUBSCRIBE_UPDATE that moves the bound races the Group Streams already in flight: the two travel on different streams with no ordering between them, so the receiver would otherwise have to guess which bound a given Group Stream was opened under. -A publisher MUST NOT send two Group Streams for the same Group Sequence on one subscription. +A publisher MUST NOT send two Group Streams for the same Group Sequence on one subscription, except to re-offer a Group whose stream it reset (see [SUBSCRIBE_DROP](#subscribe-drop)). +The re-offer can arrive before that reset: a subscriber holds a Group Stream for a sequence whose earlier stream is still arriving unread until that stream ends, then discards it if the earlier stream completed the Group, or reads it if the earlier stream was reset. A subscriber assembling a Group from more than one publisher does so across separate subscriptions, and is responsible for concatenating the runs in index order and for ignoring any frame it has already received. Both the publisher and subscriber MAY reset the stream at any time. @@ -1439,6 +1440,7 @@ SUBSCRIBE Message { Subscriber Min Age (i) Range Count (i) Range (..) ... + Update (i) } Range { @@ -1492,7 +1494,7 @@ A value of 0 is a PROTOCOL_VIOLATION, since a subscription that asks for nothing **Range**: A span of the Track to deliver, from its `Start` up to its `End`. -The subscription is the union of its Ranges: the publisher delivers each Group inside any of them once, in `Group Order` across all of them, and accounts with SUBSCRIBE_DROP for every Group it will not deliver. +The subscription is the union of its Ranges: the publisher delivers each Group inside any of them once, in `Group Order` across all of them, and accounts with SUBSCRIBE_SKIP or SUBSCRIBE_DROP for every Group it will not deliver. Each Range implicitly takes the next Range ID on the stream, starting at 0, which a [SUBSCRIBE_UPDATE](#subscribe-update) uses to retire it. **Anchor Type**: @@ -1507,6 +1509,10 @@ Any other value is a PROTOCOL_VIOLATION. The publisher resolves a Relative or Time Anchor against the Track as it holds it. A Range resolves once: an update that leaves it live does not resolve it again. +**Update**: +The number of the subscription's terms this message sets: 0 on SUBSCRIBE, and on each SUBSCRIBE_UPDATE a larger value than the one before it. +A [SUBSCRIBE_DROP](#subscribe-drop) echoes the number in effect when its groups were given up, so the subscriber can tell a stale verdict on its current terms from one an update has since replaced. + A `Start` is a floor, not a request; `Subscriber Max Age` is the only thing that asks for data. The publisher SHOULD start at the oldest group at or above the floor that [Expiration](#expiration) has not expired, and MUST NOT deliver an older one. A `Subscriber Max Age` of 0 therefore starts at the latest group, since every older group is already stale. @@ -1524,6 +1530,10 @@ A subscriber MAY send multiple SUBSCRIBE_UPDATE messages to update the subscript Every field but the Ranges restates the subscription, and the latest one applies. The Ranges are an edit, growing or shrinking the subscription: the update retires the live Ranges it names by Range ID, then adds its own, each taking the next Range ID. Lowering a `Start` covers the groups below the old one that the publisher still holds, not only those that arrive later; a group already sent is not sent again. +Growing the subscription asks for what it newly covers: the publisher MUST serve the newly covered Groups it holds and never delivered on this subscription. +Every SUBSCRIBE_UPDATE, whatever it changes, reopens every stale (OLD) SUBSCRIBE_DROP sent before it: the publisher MUST serve each Group named in one that it still holds, that the new terms cover, and that is no longer stale under them, and MUST name each that is still stale in a SUBSCRIBE_DROP carrying the new `Update`. +A SUBSCRIBE_DROP with any other Error Code is final, and no update reopens it. +A SUBSCRIBE_UPDATE never resets the subscription, since a reset would race Group Streams already in flight, and shrinking it sends no SUBSCRIBE_SKIP or SUBSCRIBE_DROP. ~~~ SUBSCRIBE_UPDATE Message { @@ -1536,11 +1546,13 @@ SUBSCRIBE_UPDATE Message { Range ID (i) ... Range Count (i) Range (..) ... + Update (i) Wanted Group (i) } ~~~ See [SUBSCRIBE](#subscribe) for information about each field. +An `Update` no larger than the previous one only confuses the subscriber's own accounting of stale drops; the publisher echoes the latest value it read. Moving a `Start` forward within a group that is already being delivered does not retract frames the publisher has already sent; it only bounds what the publisher sends from here on. **Remove Count** and **Range ID**: @@ -1839,7 +1851,9 @@ A Range retired before its start resolved gets none. ## SUBSCRIBE_DROP {#subscribe-drop} -A publisher names groups it will not deliver with SUBSCRIBE_DROP on the Subscribe Stream. +A publisher names groups that existed but that it will not deliver with SUBSCRIBE_DROP on the Subscribe Stream: an aborted or reset group, one evicted or expired with no copy left to serve it, or one past the subscriber's budget (OLD). +Sequences the publisher declared it will never produce are named by [SUBSCRIBE_SKIP](#subscribe-skip) instead. +A sequence the publisher never had but never declared skipped, such as a gap between the Groups it created or one a relay never received, might have existed, so it is named in SUBSCRIBE_DROP: with OLD when it is past the subscriber's budget, otherwise with NOT_FOUND. ~~~ SUBSCRIBE_DROP Message { @@ -1848,18 +1862,71 @@ SUBSCRIBE_DROP Message { Group Start (i) Group End (i) Error Code (i) + Update (i) } ~~~ **Group Start** and **Group End** are the inclusive range of group sequences the publisher gave up. `Group Start = Group End + 1` is an empty range; a larger Group Start is a PROTOCOL_VIOLATION. -**Error Code** is a [stream error code](#error-codes) saying why the groups were given up, as a reset retaining the Group Stream header would: for example NOT_FOUND for a sequence the publisher never had, or OLD for one past its budget. + +**Error Code** is a [stream error code](#error-codes) saying why the groups were given up, as a reset retaining the Group Stream header would: for example OLD for one past the subscriber's budget, EVICTED for one the publisher no longer holds, or NOT_FOUND for one a relay never received. A subscriber MUST end the readers of a dropped group it holds with that error, including a group whose stream is still arriving, rather than end it cleanly. +A stale (OLD) drop is the exception: it ends no group whose Group Stream arrived, since that stream wins over it (see below), and a publisher that gives up a stream it opened resets it. + +**Update**: +The [SUBSCRIBE](#subscribe) or [SUBSCRIBE_UPDATE](#subscribe-update) `Update` in effect when the publisher gave the groups up. +A subscriber MUST treat an `Update` larger than the last one it sent as a PROTOCOL_VIOLATION. + +A drop whose Error Code is OLD is a stale verdict, which holds only under the update it names. +The subscriber judges it against the `Subscriber Max Age` it sent with that update, from its own record: the drop does not carry the budget. +A stale drop whose `Update` a later update has replaced is no verdict, since the update may have moved the Range or budget it was judged against: the subscriber still owes those groups, and a relay asks for such a group again rather than drop it for its own readers. +The next SUBSCRIBE_UPDATE reopens every stale drop the subscriber accounted for, which it owes again until the publisher serves the group or drops it again under the new `Update`, as described in [SUBSCRIBE_UPDATE](#subscribe-update). +A relay that received a stale drop settles with it only a reader whose own budget is no wider than the one the relay sent with that update and whose Ranges include every group the relay's own Ranges name above the dropped sequence, since the verdict on a group was judged against the groups above it. + +A drop with any other Error Code is final, whatever its `Update`, and replaces a stale drop of the same sequences, so no update reopens them. + +A later [SUBSCRIBE_SKIP](#subscribe-skip) for the same sequences overrides a drop, whatever its Error Code: the subscriber records them as skipped from then on, and a relay forwards the skip even over a drop it already forwarded. +A subscriber that already ended readers on the drop does not deliver anything again: those readers stay ended. +A drop that follows a skip of the same sequences changes nothing. -For every sequence in the subscription's range, the publisher MUST send a Group Stream or name it in SUBSCRIBE_DROP. -This includes sequences it skipped and streams reset before their headers were delivered. +A Group Stream for a sequence the subscriber already saw dropped, as after such a re-offer, wins over the drop: the subscriber delivers it. +A Group Stream also wins over a stale drop of its sequence that arrives after it, since the stream can overtake a drop the publisher sent first. +Either way the Group Stream accounts for the sequence, and the next SUBSCRIBE_UPDATE does not reopen it: the publisher, having served it, neither serves it again nor names it in another drop. + +For every sequence in the subscription's range, the publisher MUST send a Group Stream or name it in SUBSCRIBE_SKIP or SUBSCRIBE_DROP. +This includes streams reset before their headers were delivered. A reliably delivered reset retaining the Group Stream header accounts for that group instead. -The publisher SHOULD send SUBSCRIBE_DROP as soon as it gives a sequence up, without waiting for SUBSCRIBE_FINISH. +The publisher SHOULD name a sequence as soon as it gives it up, without waiting for SUBSCRIBE_FINISH. + +Before moq-lite-07, SUBSCRIBE_DROP carries no `Update`, its Error Code is application-specific, and it also names the sequences moq-lite-07 names in SUBSCRIBE_SKIP; every such drop is final. + +## SUBSCRIBE_SKIP {#subscribe-skip} + +A publisher names sequences it declared it will never produce with SUBSCRIBE_SKIP on the Subscribe Stream. + +~~~ +SUBSCRIBE_SKIP Message { + Type (i) = 0x3 + Message Length (i) + Group Start (i) + Group End (i) +} +~~~ + +**Group Start** and **Group End** are the inclusive range of sequences that will never hold a group. +`Group Start = Group End + 1` is an empty range; a larger Group Start is a PROTOCOL_VIOLATION. + +A skip names only sequences the original publisher declared it will never produce, and it is final: no Group will ever exist at those sequences, whatever the terms. +A publisher MUST NOT later create a Group at a sequence it skipped, and a subscriber MUST NOT deliver a Group Stream for one. +It carries no error and ends no reader, since there is nothing to read; a subscriber stops waiting for those sequences. +A skip overrides an earlier SUBSCRIBE_DROP of the same sequences, whatever its Error Code, since never produced is the most authoritative account of a sequence: no update reopens them, and a relay forwards the skip downstream even over a drop it already forwarded. +A subscriber that already ended readers on that drop does not deliver anything again: those readers stay ended. +A skip never overrides a Group that arrived, since that Group was produced; a subscriber ignores a skip for such a sequence, which is a protocol error the publisher made, and MAY log it. + +A gap in the sequence numbers says nothing on its own: a publisher may create Groups out of order, or create again a sequence whose Group it aborted, so it never skips a sequence merely because a higher Group exists or the Track ended without it. +Only the application producing the Track knows a sequence will never exist, for example when it numbers Groups sparsely, or when it moves on to the next Group and so declares that it will not go back to fill the sequences below. +A publisher SHOULD send the skip as soon as it declares it. +A relay forwards a skip it received unchanged, and never skips a sequence on its own: one it never received is named in SUBSCRIBE_DROP (see [SUBSCRIBE_DROP](#subscribe-drop)). ## FETCH FETCH is sent by a subscriber to request a single group from a track. @@ -2199,6 +2266,8 @@ The `Message Length` describes the payload size on the wire. - Restored SUBSCRIBE_DROP (type 0x2), replacing Stream Count on SUBSCRIBE_FINISH. Every sequence is accounted for by a Group Stream or a drop; a publisher SHOULD send a drop as soon as it gives a sequence up. - SUBSCRIBE_DROP's Error Code is a stream error code, which ends the dropped groups' readers, including one still arriving. - Refuse a SUBSCRIBE_DROP whose Group Start is greater than Group End plus one on every version that carries it. +- Added SUBSCRIBE_SKIP (type 0x3) for sequences the publisher declared it will never produce, which ends no reader and is final: a Group is never created at a skipped sequence. A skip overrides an earlier drop of the same sequences, whatever its Error Code, without re-delivering to readers the drop already ended, and never overrides a Group that arrived. A gap in the sequence numbers is not a skip, since Groups may be created out of order; a sequence never had and never declared skipped is named in SUBSCRIBE_DROP. SUBSCRIBE_DROP no longer names declared skips. +- Added `Update` to SUBSCRIBE, SUBSCRIBE_UPDATE, and SUBSCRIBE_DROP: the number of the terms in effect. A stale (OLD) drop holds only under the update it names, judged against the budget the subscriber sent with it; every later SUBSCRIBE_UPDATE reopens it, and the publisher re-offers the group or drops it again under the new `Update`. Any other drop is final. A Group Stream wins over its earlier drop, and over a stale drop that arrives after it; no update reopens a sequence its Group Stream accounted for. A re-offered Group Stream that overtakes the reset of the stream it replaces waits, unread, until that stream ends, and is read only if it was reset. - SUBSCRIBE_FINISH declares the final boundary while lower groups can still be in flight, and no longer closes the Subscribe Stream: the publisher keeps reading SUBSCRIBE_UPDATE, so an update can reach an ended Track's history. - A covered subscription, whose every Range has an End and whose every Group was sent or dropped, also stays open for an update; the publisher never FINs first. - A publisher that withheld SUBSCRIBE_OK sends it, then the drops it withheld, once no group can resolve the start: every Group of a bounded subscription was given up, or the Track finished. diff --git a/go/wrapper/publish.go b/go/wrapper/publish.go index 07282c691d9..b506335d357 100644 --- a/go/wrapper/publish.go +++ b/go/wrapper/publish.go @@ -205,6 +205,12 @@ func (t *TrackProducer) CreateGroup(sequence uint64) (*GroupProducer, error) { return &GroupProducer{inner: inner}, nil } +// SkipGroups declares that groups start up to the exclusive end will never be +// produced, so readers move past them. Creating a group there then fails. +func (t *TrackProducer) SkipGroups(start, end uint64) error { + return t.inner.SkipGroups(start, end) +} + // WriteFrame writes frame as a single-frame group. func (t *TrackProducer) WriteFrame(frame Frame) error { return t.inner.WriteFrame(frame) diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 5f9f8a73074..b9bb6b83b4e 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -345,6 +345,10 @@ export namespace Error { }); readonly code: SessionCode; } + export class Skipped extends Error { + constructor(sequence: bigint); + readonly sequence: bigint; + } export class Stream extends Error { constructor(code: StreamCode, options?: Error.StreamOptions); readonly code: StreamCode; @@ -751,6 +755,7 @@ export namespace Track { readonly name: string; get priority(): number; requests(): Track.Requests; + skipGroups(start: bigint, end: bigint): void; subscribe(options?: Track.Subscription): Track.Subscriber; get subscription(): Signals.Getter; writeFrame(frame: Group.Frame): void; diff --git a/js/net/src/error.ts b/js/net/src/error.ts index db0ba2469e3..9a5f3ae0c31 100644 --- a/js/net/src/error.ts +++ b/js/net/src/error.ts @@ -405,6 +405,25 @@ function localStreamCode(err: unknown): StreamCode { return StreamCode.Internal; } +/** + * A group was created at a sequence its track declared skipped. + * + * A skip says the sequence will never be produced, so it is final. See the track producer's + * `skipGroups`. Mirrors the Rust `Error::Skipped`. + * + * @public + */ +export class Skipped extends Error { + /** The sequence that was refused. */ + readonly sequence: bigint; + + constructor(sequence: bigint) { + super(`sequence ${sequence} was skipped`); + this.name = "Skipped"; + this.sequence = sequence; + } +} + /** * The {@link StreamCode | StreamCode.ControlTimeout} error for a control request the peer never answered. * diff --git a/js/net/src/errors.ts b/js/net/src/errors.ts index 585dd0472e0..3e9157c1cec 100644 --- a/js/net/src/errors.ts +++ b/js/net/src/errors.ts @@ -11,6 +11,7 @@ export { ProtocolViolation, RefusedRedirect, Session, + Skipped, Stream, type StreamOptions, TimestampMismatch, diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index ae66a7ea732..ac9e8a627f4 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -4,7 +4,7 @@ import { Producer as BroadcastProducer } from "./broadcast.ts"; import { accept as acceptSession, connect as connectSession } from "./connection/index.ts"; import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; -import { hooks } from "./internal.ts"; +import { hooks, staleHole } from "./internal.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; import { Producer as OriginProducer } from "./origin.ts"; @@ -336,6 +336,30 @@ test("fill: a fill names the most urgent priority and loosest budget among its r broadcast.close(); }); +// A stale verdict settles only the readers it covers, so a fetch it does not settle keeps its fill. +// Rust records a stale run without answering any fill. +test("fill: a stale verdict leaves an outstanding fill owed", async () => { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { maxAge: null }); + const requests = track.requests(); + track.createGroup(3n).finish(); + const fetching = wireOf(broadcast).fetchGroup("video", 2n); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + expect(fill.wanted()).toBe(2n); + + hooks.dropTrackGroups(track, 2n, 3n, staleHole(500)); + expect(fill.wanted()).toBe(2n); + const group = fill.accept(2n); + group.writeFrame({ payload: enc.encode("filled") }); + group.finish(); + expect(await frames(await fetching)).toEqual(["filled"]); + + fill.close(); + requests.close(); + broadcast.close(); +}); + test("fill: a group arriving live answers the fill", async () => { const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: null }); @@ -720,7 +744,7 @@ test("fill: a group the track gives up elsewhere answers the fill", async () => // An upstream SUBSCRIBE_DROP, say: the handler is no longer asked for it. const cut = new StreamError(StreamCode.Cancel, { message: "cut upstream" }); - hooks.dropTrackGroups(track, 2n, 3n, cut); + hooks.dropTrackGroups(track, 2n, 3n, { error: cut }); expect(fill.wanted()).toBeUndefined(); await expect(fetching).rejects.toBe(cut); @@ -729,6 +753,106 @@ test("fill: a group the track gives up elsewhere answers the fill", async () => broadcast.close(); }); +// A received stale run that does not settle a reader is unknown to it, so its fill is still asked: +// here the upstream judged it under an open range, while the reader is capped. Mirrors the Rust +// `Misses::unclaimed`. +test("fill: a stale run that does not settle the reader is still asked for", async () => { + const { broadcast, track } = resumed(2n); + const requests = track.requests(); + hooks.dropTrackGroups(track, 0n, 2n, staleHole(60_000)); + const reader = track.subscribe({ + ranges: [{ start: { absolute: 0n }, end: { absolute: 2n } }], + delay: { max: Milli(500) }, + }); + void drain(reader); + const fill = await Promise.race([requests.requestedFill(), yieldNow().then(() => undefined)]); + expect(fill && [fill.start, fill.end]).toEqual([0n, 2n]); + + fill?.close(); + reader.close(); + requests.close(); + broadcast.close(); +}); + +// The upstream judged 0..2 and 4..6 under 500ms, and gave up both of them as stale. +function judgedTwice() { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { maxAge: null }); + hooks.receiveTrack(track); + const requests = track.requests(); + hooks.supersedeStale(track, [ + { start: 0n, end: 2n }, + { start: 4n, end: 6n }, + ]); + hooks.dropTrackGroups(track, 0n, 2n, staleHole(500)); + hooks.dropTrackGroups(track, 4n, 6n, staleHole(500)); + const reader = (ranges: [bigint, bigint][]) => + track.subscribe({ + ranges: ranges.map(([start, end]) => ({ start: { absolute: start }, end: { absolute: end } })), + delay: { max: Milli(500) }, + }); + return { broadcast, requests, reader }; +} + +// A stale verdict settles a reader that reads every range the upstream judged it against, however +// many there are, with nothing received and no fill asked. Mirrors Rust. +test("fill: a stale verdict settles a reader of every range it judged", async () => { + const { broadcast, requests, reader: subscribe } = judgedTwice(); + const reader = subscribe([ + [0n, 2n], + [4n, 6n], + ]); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 0n, end: 2n, stale: 500 }); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 4n, end: 6n, stale: 500 }); + expect(hooks.tryRecvGroup(reader, false)).toEqual({ kind: "done" }); + const fill = await Promise.race([requests.requestedFill(), yieldNow().then(() => undefined)]); + expect(fill).toBeUndefined(); + + reader.close(); + requests.close(); + broadcast.close(); +}); + +// A reader whose ranges only partly overlap the ones a stale verdict was judged against is settled +// where it reads every judged group above the run, and asks a fill for the rest, even with nothing +// received: no update brings it, since its ranges add nothing to the upstream's terms. Mirrors Rust. +test("fill: a stale verdict settles a partly overlapping reader only where it reads above", async () => { + const { broadcast, requests, reader: subscribe } = judgedTwice(); + // Reads every judged group above each run, and more past them: settled. + const wider = subscribe([ + [1n, 2n], + [4n, 8n], + ]); + expect(hooks.tryDrop(wider)).toMatchObject({ start: 1n, end: 2n, stale: 500 }); + expect(hooks.tryDrop(wider)).toMatchObject({ start: 4n, end: 6n, stale: 500 }); + expect(hooks.tryDrop(wider)).toBeUndefined(); + expect(await Promise.race([requests.requestedFill(), yieldNow().then(() => undefined)])).toBeUndefined(); + // Stops at 4, which the upstream judged against 5: its own newest, which no verdict convicts. + // Both runs are unknown to it, and asked for. + const narrower = subscribe([ + [0n, 2n], + [4n, 5n], + ]); + expect(hooks.tryRecvGroup(narrower, false)).toEqual({ kind: "idle" }); + expect(hooks.tryDrop(narrower)).toBeUndefined(); + const asked: bigint[][] = []; + for (;;) { + const fill = await Promise.race([requests.requestedFill(), yieldNow().then(() => undefined)]); + if (!fill) break; + asked.push([fill.start, fill.end]); + } + asked.sort((a, b) => Number(a[0] - b[0])); + expect(asked).toEqual([ + [0n, 2n], + [4n, 5n], + ]); + + wider.close(); + narrower.close(); + requests.close(); + broadcast.close(); +}); + test("fill: a cache-miss fetch of a dynamic track serves the one track the broadcast already has", async () => { const broadcast = new BroadcastProducer(); // Pulled first, so the broadcast serves tracks on demand. diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index b3bbf302cc0..0368a398cf7 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -1907,12 +1907,9 @@ export class Subscriber { }; // A group that will never arrive whole is dropped, so readers stop owing it. const dropHeadless = () => - hooks.dropTrackGroups( - track, - group.groupId, - group.groupId + 1n, - new StreamError(StreamCode.NotFound, { message: "the group's head was lost" }), - ); + hooks.dropTrackGroups(track, group.groupId, group.groupId + 1n, { + error: new StreamError(StreamCode.NotFound, { message: "the group's head was lost" }), + }); // After an END_OF_GROUP or END_OF_TRACK status only the stream's end may follow: an // object past the group's final one makes the track malformed. const last = async () => { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index f44c02f8cbe..2992a7386ce 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -10,7 +10,7 @@ import type { Dispose, GetPromise, Getter } from "@moq/signals"; import type { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; import type * as datagram from "./datagram.ts"; import type { Delivered } from "./delivered.ts"; -import { toStreamCode } from "./error.ts"; +import { StreamCode, StreamError, toStreamCode } from "./error.ts"; import type { Frame, FrameConsumer, Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { isAnonymous, type WireRoute } from "./hop.ts"; import type * as Instance from "./instance.ts"; @@ -189,26 +189,70 @@ export function removeRun(runs: readonly R[], start: bigint, end: ); } -/** A run of sequences given up, with the error they were given up with. */ -export interface DropRun extends Run { +/** + * Why a group sequence will not be delivered. Mirrors the Rust `track::Hole`. + * + * lite-07 carries the two kinds in their own messages, SUBSCRIBE_SKIP and SUBSCRIBE_DROP; + * earlier versions name both in SUBSCRIBE_DROP. + */ +export interface Hole { + /** The error a reader waiting on the sequence ends with: NOT_FOUND for a skip. */ error: Error; + /** Never produced, such as a skipped sequence number: final, and it ends no reader. */ + skip?: boolean; + /** + * The budget in milliseconds a staleness verdict was judged under, undefined for a drop that + * does not depend on it. Only a widening past it reverses the verdict. + */ + stale?: number; +} + +/** A sequence never produced; see {@link Hole.skip}. */ +export function skipHole(): Hole { + return { error: new StreamError(StreamCode.NotFound, { message: "group never produced" }), skip: true }; +} + +/** A group judged stale under `budget` milliseconds. */ +export function staleHole(budget: number): Hole { + return { error: new StreamError(StreamCode.Old, { message: "group is past the budget" }), stale: budget }; +} + +/** A run of sequences given up, with why. */ +export interface DropRun extends Run, Hole { + /** Copied into a cursor from the track's runs, which it folds in afresh on each change. */ + received?: boolean; +} + +/** Whether two runs say the same thing, so they may merge. */ +function sameHole(a: DropRun, b: DropRun): boolean { + return ( + toStreamCode(a.error) === toStreamCode(b.error) && + !a.skip === !b.skip && + a.stale === b.stale && + !a.received === !b.received + ); } /** - * Sorted, disjoint `drops` with `[start, end)` given up with `error`, which replaces whatever - * it overlaps. Neighbours given up with the same stream code merge, so a wide DROP stays one - * run and a peer naming one sequence per DROP cannot grow the runs. + * Sorted, disjoint `drops` with `[start, end)` given up as `hole` says, which replaces whatever + * it overlaps. Neighbours given up alike merge, so a wide DROP stays one run and a peer naming + * one sequence per DROP cannot grow the runs. */ -export function insertDrop(drops: readonly DropRun[], start: bigint, end: bigint, error: Error): DropRun[] { +export function insertDrop( + drops: readonly DropRun[], + start: bigint, + end: bigint, + hole: Hole & { received?: boolean }, +): DropRun[] { if (start >= end) return [...drops]; const next = removeRun(drops, start, end); let at = 0; while (at < next.length && next[at].start < start) at++; - next.splice(at, 0, { start, end, error }); + next.splice(at, 0, { start, end, error: hole.error, skip: hole.skip, stale: hole.stale, received: hole.received }); const merged: DropRun[] = []; for (const run of next) { const last = merged[merged.length - 1]; - if (last && last.end === run.start && toStreamCode(last.error) === toStreamCode(run.error)) last.end = run.end; + if (last && last.end === run.start && sameHole(last, run)) last.end = run.end; else merged.push({ ...run }); } return merged; @@ -240,7 +284,7 @@ export type Recv = /** A group to serve. It has already left the buffer, so dropping this is dropping the group. */ | { kind: "group"; group: GroupConsumer } /** Sequences `[start, end)` this cursor gave up without serving, and why. */ - | { kind: "drop"; start: bigint; end: bigint; error: Error } + | ({ kind: "drop"; start: bigint; end: bigint } & Hole) /** Nothing readable, but the track is live and may produce more. */ | { kind: "idle" } /** The producer finished, yet groups above the cap are still held: raising it releases them. */ @@ -397,6 +441,13 @@ export interface ResumeSource { fork(options?: Subscription): Subscriber; /** A fresh reader for a group still cached by one of this reader's copies. */ cachedGroup(sequence: bigint, from?: number): GroupConsumer | undefined; + /** See {@link hooks.reofferGroups}, answered by the serving copy. */ + reofferGroups(start: bigint, end: bigint): { groups: GroupConsumer[]; givenUp: Run[] }; + /** + * See {@link hooks.reofferStale}: the stale runs queued here, those this reader reported, which a + * route change keeps, and those the serving copy reported. + */ + reofferStale(): Run[]; /** Let go of every copy. */ close(abort?: Error): void; } @@ -491,9 +542,22 @@ export const hooks: { * Record that the publisher gave up the groups `[start, end)` with `error`: none will * arrive, so a reader waiting on one moves past it, a group still arriving in them ends with * `error`, the declared end does not wait for them, and a relay serving this track forwards - * the drop. Mirrors the Rust `drop_groups`; assigned by `track.ts`. + * the drop. A skip ends no reader, and replaces any drop; a drop leaves a skip in place. + * Returns the runs recorded. Mirrors the Rust `give_up`; assigned by `track.ts`. + */ + dropTrackGroups: (producer: Producer, start: bigint, end: bigint, hole: Hole) => Run[]; + /** + * `Producer.skipGroups`, returning the cached sequences it left alone, which a skip naming a + * group that arrived contradicts. Mirrors the Rust `skip_received`; assigned by `track.ts`. + */ + skipTrackGroups: (producer: Producer, start: bigint, end: bigint) => Run[]; + /** + * Forget every run a received copy holds as stale: the terms its upstream judged them under + * were replaced, so they are unknown again. `judged` is every group the new terms' ranges name, + * undefined when unknown. Returns the runs forgotten: a group that arrived since its stale drop + * replaced it, so it is not among them. Mirrors the Rust `supersede_stale`. */ - dropTrackGroups: (producer: Producer, start: bigint, end: bigint, error: Error) => void; + supersedeStale: (producer: Producer, judged: readonly Run[] | undefined) => Run[]; /** * Mark a track as fed by a wire subscription, which owes missing groups. `starts` says how * that wire's start bounds the fills its readers ask below it. Assigned by `track.ts`. @@ -501,6 +565,8 @@ export const hooks: { receiveTrack: (producer: Producer, starts?: FillStarts) => void; /** Create a received group at its wire offset before publishing it to readers. Assigned by `track.ts`. */ receiveGroup: (producer: Producer, group: { sequence: bigint; start: number }) => GroupProducer; + /** How the copy of `sequence` still arriving ends, if one is, for a duplicate to wait on. Assigned by `track.ts`. */ + arrivingGroup: (producer: Producer, sequence: bigint) => GetPromise | undefined; /** The producer's largest reported location and its subsequent changes. */ producerLargest: (producer: Producer) => Getter; @@ -634,6 +700,20 @@ export const hooks: { * below where SUBSCRIBE_START resolved never goes out, so a wider budget may still serve it. */ unreport: (subscriber: Subscriber, start: bigint, end: bigint) => void; + /** + * Re-offer what a subscriber gave up as stale through dispositions, as a wire whose every update + * reopens its stale drops does: each run is judged again under the new terms. Returns the runs + * taken back, which are owed again. Mirrors the Rust crate-private + * `track::Subscriber::reoffer_stale`. + */ + reofferStale: (subscriber: Subscriber) => Run[]; + /** + * Fresh reads of the groups in `[start, end)` a subscriber's track still caches, its range still + * admits, and its current budget no longer judges stale, each expiring against that budget, and + * the runs still given up: for a wire whose update re-offers a group whose stream it gave up as + * stale. Mirrors the Rust crate-private `track::Subscriber::reoffer_groups`. + */ + reofferGroups: (subscriber: Subscriber, start: bigint, end: bigint) => { groups: GroupConsumer[]; givenUp: Run[] }; /** Whether this subscriber's budget gives up on `sequence`, judged by its successor; see {@link expireGroup}. */ staleAt: (subscriber: Subscriber, sequence: bigint) => Expiry; /** A group the subscriber's track still caches, as a fresh read handle. */ @@ -642,6 +722,11 @@ export const hooks: { holdsGroup: (subscriber: Subscriber, sequence: bigint) => boolean; /** The error the subscriber's track gave `sequence` up with, if it did, and what changes it. */ givenUp: (subscriber: Subscriber, sequence: bigint) => { error?: Error; changed: Getter }; + /** + * The parts of `[start, end)` the subscriber's track still holds as a received stale run: one + * new terms superseded is gone. Mirrors the Rust crate-private `track::Subscriber::stale_held`. + */ + staleHeld: (subscriber: Subscriber, start: bigint, end: bigint) => Run[]; /** Whether a group may still land on the live feed of the subscriber's track; see {@link Land}. */ mayLand: (subscriber: Subscriber, land: Land) => boolean; /** A subscriber's largest location now, and whether it can still advance. */ @@ -811,6 +896,9 @@ export const hooks: { dropTrackGroups: () => { throw new Error("track.ts not loaded"); }, + skipTrackGroups: () => { + throw new Error("track.ts not loaded"); + }, producerLargest: () => { throw new Error("track.ts not loaded"); }, @@ -898,6 +986,15 @@ export const hooks: { unreport: () => { throw new Error("track.ts not loaded"); }, + supersedeStale: () => { + throw new Error("track.ts not loaded"); + }, + reofferStale: () => { + throw new Error("track.ts not loaded"); + }, + reofferGroups: () => { + throw new Error("track.ts not loaded"); + }, deliveredRuns: () => { throw new Error("track.ts not loaded"); }, @@ -928,6 +1025,9 @@ export const hooks: { holdsGroup: () => { throw new Error("track.ts not loaded"); }, + staleHeld: () => { + throw new Error("track.ts not loaded"); + }, givenUp: () => { throw new Error("track.ts not loaded"); }, @@ -973,6 +1073,9 @@ export const hooks: { receiveGroup: () => { throw new Error("track.ts not loaded"); }, + arrivingGroup: () => { + throw new Error("track.ts not loaded"); + }, groupOffset: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/ledger.test.ts b/js/net/src/ledger.test.ts index d413dbcad26..fa14f9ffba0 100644 --- a/js/net/src/ledger.test.ts +++ b/js/net/src/ledger.test.ts @@ -111,3 +111,30 @@ test("shuffled arrivals and bulk merges agree with the accounted sequence set", expect(ledger.hasGaps()).toBe(false); expect(ledger.covers(0n, 4128n)).toBe(true); }); + +test("unaccount reopens a gap for a fresh grace", () => { + const ledger = new Ledger(); + const grace = Milli(1000); + ledger.account(0n, 10n, at(0)); + ledger.unaccount(3n, 5n, at(1000)); + expect(ledger.covers(0n, 3n) && ledger.covers(5n, 10n)).toBe(true); + expect(ledger.covers(3n, 5n)).toBe(false); + // Its grace runs from the reversal. + ledger.expire(at(1500), grace); + expect(ledger.covers(3n, 5n)).toBe(false); + ledger.expire(at(2000), grace); + expect(ledger.covers(0n, 10n)).toBe(true); +}); + +test("unaccount restarts the older gap it joins", () => { + const ledger = new Ledger(); + const grace = Milli(1000); + ledger.account(0n, 2n, at(0)); + ledger.account(4n, 6n, at(0)); + ledger.unaccount(1n, 2n, at(900)); + // Its grace runs from the reversal, not from when the older gap opened. + ledger.expire(at(1000), grace); + expect(ledger.covers(1n, 2n)).toBe(false); + ledger.expire(at(1900), grace); + expect(ledger.covers(0n, 6n)).toBe(true); +}); diff --git a/js/net/src/ledger.ts b/js/net/src/ledger.ts index 6ed51d28ec5..ddf7617ecc9 100644 --- a/js/net/src/ledger.ts +++ b/js/net/src/ledger.ts @@ -154,6 +154,39 @@ export class Ledger { return gaps; } + /** + * Take `[start, end)` back out of what is accounted for, owed again from `at`: a verdict that + * accounted for it was reversed. The gap it leaves opens at `at`, so it is waited for a fresh + * grace, even where it joins an older gap. Mirrors the Rust `Ledger::unaccount`. + */ + unaccount(start: bigint, end: bigint, at: Instant): void { + if (start >= end) return; + const lower = this.#runs.getPairOrNextLower(start)?.[1]; + const overlapping: Run[] = []; + for (const [, run] of this.#runs.entries(lower?.start ?? start)) { + if (run.start >= end) break; + if (run.end > start) overlapping.push(run); + } + if (overlapping.length === 0) return; + for (const run of overlapping) { + this.#remove(run); + if (this.#last === run) this.#last = undefined; + if (run.start < start) { + const below = { start: run.start, end: start, since: run.since }; + this.#runs.set(below.start, below); + this.#ages.set(below, undefined); + } + if (end < run.end) { + const above = { start: end, end: run.end, since: at }; + this.#runs.set(above.start, above); + this.#ages.set(above, undefined); + } + } + const last = this.#runs.maxKey(); + this.#last = last === undefined ? undefined : this.#runs.get(last); + this.demand(start, end, at); + } + /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ demand(start: bigint, end: bigint, at: Instant): void { if (start >= end) return; diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 27a6d4495a6..a331a845aef 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -7,7 +7,7 @@ import { Producer as BroadcastProducer } from "../broadcast.ts"; import { fromTransport, SessionCode, StreamCode, StreamError, toStreamCode } from "../error.ts"; import type { Producer as GroupProducer } from "../group.ts"; import { randomHop } from "../hop.ts"; -import { hooks } from "../internal.ts"; +import { hooks, staleHole } from "../internal.ts"; import { createMockTransportPair, textFrame } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; @@ -16,6 +16,7 @@ import { Front } from "../resume.ts"; import { Reader, Stream, Writer } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import { Subscriber as TrackSubscriber } from "../track.ts"; +import { Budget } from "../util/yield.ts"; import { wireOf } from "../wire.ts"; import { type AnnounceBroadcast, AnnounceOk, AnnounceRequest, decodeAnnounceBroadcast } from "./announce.ts"; import { Fetch } from "./fetch.ts"; @@ -1097,6 +1098,8 @@ test("lite-07 serves a time range from the on-demand track its lookup resolved", // deliver it; a sequence cursor would skip it permanently. test("lite draft-05: a late-arriving older group is still served", async () => { const sub = await servedSubscription({ startGroup: 1n }); + // A received copy: a local publisher's gap is skipped at once, so its late group is not. + hooks.receiveTrack(sub.track); try { // Serve one at a time so arrival order at the publisher is the order given. sub.serve(1n); @@ -1178,7 +1181,7 @@ for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { expect(await sub.servedSequence()).toBe(0n); expect("start" in (await decodeSubscribeResponse(sub.client.reader, version))).toBe(true); - hooks.dropTrackGroups(sub.track, 1n, 3n, new StreamError(StreamCode(70), { message: "cut" })); + hooks.dropTrackGroups(sub.track, 1n, 3n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); const resp = await decodeSubscribeResponse(sub.client.reader, version); if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); expect([resp.drop.start, resp.drop.end, resp.drop.error]).toEqual([ @@ -1191,32 +1194,135 @@ for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { } }); - test(`lite ${version}: the end sweep's SUBSCRIBE_DROP carries ${version === Version.DRAFT_07 ? "NOT_FOUND" : "no code"}`, async () => { - const sub = await servedSubscription({ version, startGroup: 5n }); + // A skip replaces a drop, so a relay forwards it over a drop it already forwarded; lite-06, + // which names a skip as a drop, sends nothing more. Mirrors Rust. + test(`lite ${version}: a skip goes out over a forwarded drop`, async () => { + const sub = await servedSubscription({ version, startGroup: 0n, routed: true }); try { - sub.serve(5n); - expect(await sub.servedSequence()).toBe(5n); - await replayUpdate({ priority: 0, startGroup: 2n, ...restate(version) }).encode(sub.client.writer, version); - await flush(); + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, version))).toBe(true); + + hooks.dropTrackGroups(sub.track, 1n, 3n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); + const resp = await decodeSubscribeResponse(sub.client.reader, version); + if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); + expect([resp.drop.start, resp.drop.end]).toEqual([1n, 2n]); + + sub.track.skipGroups(1n, 3n); + if (version === Version.DRAFT_07) { + const skip = await decodeSubscribeResponse(sub.client.reader, version); + expect(skip).toMatchObject({ skip: { start: 1n, end: 2n } }); + } sub.track.finish(); sub.track.close(); - const codes: number[] = []; + const holes: unknown[] = []; for (;;) { - const resp = await decodeSubscribeResponseMaybe(sub.client.reader, version); - if (!resp) break; - if ("drop" in resp) codes.push(resp.drop.error); - // Lite-07 keeps the stream open past SUBSCRIBE_FINISH, after the sweep's drops. - if ("finish" in resp && keepsSubscribeOpen(version)) break; + const next = await decodeSubscribeResponseMaybe(sub.client.reader, version); + if (!next) break; + if ("drop" in next) holes.push(["drop", next.drop.start, next.drop.end]); + if ("skip" in next) holes.push(["skip", next.skip.start, next.skip.end]); + if ("finish" in next && keepsSubscribeOpen(version)) break; } - expect(codes).toEqual([version === Version.DRAFT_07 ? StreamCode.NotFound : 0]); + expect(holes).toEqual([]); + } finally { + await sub.close(); + } + }); + + // A lite-07 stale (OLD) drop is no final verdict, so a group handed out after it is served, as + // when a front switches to a route that delivers what the old route reported stale, before the + // update that reopens it. Earlier versions' drops are final. Mirrors Rust. + test(`lite ${version}: a group ${version === Version.DRAFT_07 ? "replaces" : "stays dropped after"} a stale drop`, async () => { + const sub = await servedSubscription({ version, startGroup: 0n }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, version))).toBe(true); + + hooks.dropTrackGroups(sub.track, 1n, 2n, staleHole(60_000)); + const resp = await decodeSubscribeResponse(sub.client.reader, version); + if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); + expect([resp.drop.start, resp.drop.end]).toEqual([1n, 1n]); + + sub.serve(1n); + sub.serve(2n); + const next = await sub.servedSequence(); + expect(next).toBe(version === Version.DRAFT_07 ? 1n : 2n); } finally { await sub.close(); } }); + + for (const declared of [false, true]) { + const named = declared && version === Version.DRAFT_07 ? "in SUBSCRIBE_SKIP" : "in SUBSCRIBE_DROP"; + test(`lite ${version}: the end sweep names a ${declared ? "declared skip" : "gap never declared"} ${named}`, async () => { + const sub = await servedSubscription({ version, startGroup: 5n }); + try { + sub.serve(5n); + expect(await sub.servedSequence()).toBe(5n); + await replayUpdate({ priority: 0, startGroup: 2n, ...restate(version) }).encode( + sub.client.writer, + version, + ); + await flush(); + // Only a declared skip says groups 2 through 4 were never produced; a gap alone + // might have held them, so it is not found. + if (declared) sub.track.skipGroups(2n, 6n); + sub.track.finish(); + sub.track.close(); + const codes: number[] = []; + const skips: bigint[][] = []; + for (;;) { + const resp = await decodeSubscribeResponseMaybe(sub.client.reader, version); + if (!resp) break; + if ("drop" in resp) codes.push(resp.drop.error); + if ("skip" in resp) skips.push([resp.skip.start, resp.skip.end]); + // Lite-07 keeps the stream open past SUBSCRIBE_FINISH, after the sweep's drops. + if ("finish" in resp && keepsSubscribeOpen(version)) break; + } + if (version !== Version.DRAFT_07) { + expect([codes, skips]).toEqual([[0], []]); + } else if (declared) { + expect([codes, skips]).toEqual([[], [[2n, 4n]]]); + } else { + expect([codes, skips]).toEqual([[StreamCode.NotFound], []]); + } + } finally { + await sub.close(); + } + }); + } } -// A drop describes what was missing then; a late copy of the same group is still served. -test("lite draft-06: a reported cut still allows a late group", async () => { +// A huge declared skip goes out as one SUBSCRIBE_SKIP, at a cost independent of its length. +// Mirrors Rust. +test("lite-07 skips a huge run at once", async () => { + const FAR = 1n << 61n; + const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + sub.track.skipGroups(1n, FAR); + sub.track.finish(); + sub.track.close(); + const codes: number[] = []; + const skips: bigint[][] = []; + for (;;) { + const resp = await decodeSubscribeResponseMaybe(sub.client.reader, Version.DRAFT_07); + if (!resp) break; + if ("drop" in resp) codes.push(resp.drop.error); + if ("skip" in resp) skips.push([resp.skip.start, resp.skip.end]); + if ("finish" in resp) break; + } + expect([codes, skips]).toEqual([[], [[1n, FAR - 1n]]]); + } finally { + await sub.close(); + } +}); + +// A cut this subscription already dropped stays dropped, as in Rust: the subscriber accounted +// for it, so a group the publisher re-creates under it afterwards is not served. +test("lite draft-06: a reported cut stays dropped when re-created", async () => { const sub = await servedSubscription({ version: Version.DRAFT_06, startGroup: 0n }); try { sub.serve(0n); @@ -1231,9 +1337,7 @@ test("lite draft-06: a reported cut still allows a late group", async () => { sub.serve(1n); sub.serve(2n); - // Opened together, so the newer group may take the first stream. - const served = [await sub.servedSequence(), await sub.servedSequence()]; - expect(served.sort()).toEqual([1n, 2n]); + expect(await sub.servedSequence()).toBe(2n); sub.track.finish(); sub.track.close(); expect(await drops(sub.client.reader, Version.DRAFT_06)).toEqual([]); @@ -1283,6 +1387,232 @@ test("lite draft-06: a group stream that never completes is reset once the group } }); +// Lite-07 reopens every stale drop on an update, including one for a stream the publisher reset +// as stale: a group still stale under the new terms is dropped again under the new update's +// number, so the subscriber settles at once. Mirrors the Rust +// `an_update_reoffers_a_cached_expired_stream_on_lite07`. +test("lite draft-07: an update drops an expired stream again while it is still stale", async () => { + const clock = new Manual(); + const installed = clock.install(); + const budget = Milli(1_000); + const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n, maxAge: budget }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07))).toBe(true); + + const reset = sub.stall(); + sub.serve(1n); + expect(await sub.servedHeader()).toBe(1n); + await flush(); + await clock.advance(Nano.fromMilli(Milli(budget * 10))); + sub.serve(2n); + expect(await sub.servedSequence()).toBe(2n); + await clock.advance(Nano.fromMilli(budget)); + expect(toStreamCode(fromTransport(await reset))).toBe(StreamCode.Old); + const first = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in first)) throw new Error("expected SUBSCRIBE_DROP"); + expect([first.drop.start, first.drop.end, first.drop.update]).toEqual([1n, 1n, 0n]); + + // The track's max age still caps the budget, so group 1 is still stale. + await replayUpdate({ priority: 1, startGroup: 0n, update: 1n, ...restate(Version.DRAFT_07) }).encode( + sub.client.writer, + Version.DRAFT_07, + ); + const again = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in again)) throw new Error("expected SUBSCRIBE_DROP"); + expect([again.drop.start, again.drop.end, again.drop.update]).toEqual([1n, 1n, 1n]); + } finally { + await sub.close(); + installed.close(); + } +}); + +// The drops and skips on a lite-07 subscribe stream up to its SUBSCRIBE_FINISH, with the update each +// drop names. +async function holesToEnd(reader: Reader): Promise { + const holes: unknown[] = []; + for (;;) { + const next = await decodeSubscribeResponseMaybe(reader, Version.DRAFT_07); + if (!next) return holes; + if ("drop" in next) holes.push(["drop", next.drop.start, next.drop.end, next.drop.update]); + if ("skip" in next) holes.push(["skip", next.skip.start, next.skip.end]); + if ("finish" in next) return holes; + } +} + +// A group that wins over a lite-07 stale drop already sent retires it: the next update neither +// sends the drop again nor reopens the served group, and the end names nothing for it. Here a +// relay's upstream reported group 1 stale, then delivered it, as a front that switched routes +// does. Mirrors the Rust `a_group_served_over_a_stale_drop_retires_it`. +test("lite draft-07: a group served over a stale drop retires it", async () => { + const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07))).toBe(true); + + hooks.dropTrackGroups(sub.track, 1n, 2n, staleHole(60_000)); + const resp = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); + expect([resp.drop.start, resp.drop.end]).toEqual([1n, 1n]); + sub.serve(1n); + expect(await sub.servedSequence()).toBe(1n); + + await replayUpdate({ priority: 1, startGroup: 0n, update: 1n, ...restate(Version.DRAFT_07) }).encode( + sub.client.writer, + Version.DRAFT_07, + ); + await flush(); + sub.serve(2n); + expect(await sub.servedSequence()).toBe(2n); + sub.track.finish(); + sub.track.close(); + // No second OLD, and no NOT_FOUND at the end. + expect(await holesToEnd(sub.client.reader)).toEqual([]); + } finally { + await sub.close(); + } +}); + +// A lite-07 stream that expired as stale is a stale drop like any other: a final drop the reader +// reaches for it later replaces it, and no update drops it again then. Mirrors the Rust +// `a_final_drop_replaces_an_expired_stream_on_lite07`. +test("lite draft-07: a final drop replaces an expired stream", async () => { + const clock = new Manual(); + const installed = clock.install(); + const budget = Milli(1_000); + const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n, maxAge: budget }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07))).toBe(true); + + const reset = sub.stall(); + sub.serve(1n); + expect(await sub.servedHeader()).toBe(1n); + await flush(); + await clock.advance(Nano.fromMilli(Milli(budget * 10))); + sub.serve(2n); + expect(await sub.servedSequence()).toBe(2n); + await clock.advance(Nano.fromMilli(budget)); + expect(toStreamCode(fromTransport(await reset))).toBe(StreamCode.Old); + const first = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in first)) throw new Error("expected SUBSCRIBE_DROP"); + expect([first.drop.start, first.drop.end, first.drop.update]).toEqual([1n, 1n, 0n]); + + hooks.dropTrackGroups(sub.track, 1n, 2n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); + const final = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in final)) throw new Error("expected SUBSCRIBE_DROP"); + expect([final.drop.start, final.drop.end, final.drop.error]).toEqual([1n, 1n, 70]); + + await replayUpdate({ priority: 1, startGroup: 0n, update: 1n, ...restate(Version.DRAFT_07) }).encode( + sub.client.writer, + Version.DRAFT_07, + ); + await flush(); + sub.track.finish(); + sub.track.close(); + expect(await holesToEnd(sub.client.reader)).toEqual([]); + } finally { + await sub.close(); + installed.close(); + } +}); + +// A queued re-offer that a later update drops again as stale counts as given up: the track's end +// leaves that stale drop standing, rather than replacing it with a final one. Mirrors the Rust +// `a_redropped_reoffer_stays_stale_at_the_end_on_lite07`. +test("lite draft-07: a re-offer dropped again before it opens stays stale at the end", async () => { + const clock = new Manual(); + const installed = clock.install(); + // A write's budget yields on host CPU time, which could run the publisher's post-update task + // before the test writes update 3. + const poll = spyOn(Budget.prototype, "poll").mockReturnValue(undefined); + // Well inside the cache window, so the expired group is still cached to re-offer. + const budget = Milli(1_000); + const update = (update: bigint, maxDelay: Milli) => + new SubscribeUpdate({ + priority: 1, + startGroup: 0n, + update, + maxDelay, + ...restate(Version.DRAFT_07, update - 1n), + }).encode(sub.client.writer, Version.DRAFT_07); + const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07))).toBe(true); + await update(1n, budget); + // The restated range resolves at once. + expect("started" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07))).toBe(true); + + const reset = sub.stall(); + sub.serve(1n); + expect(await sub.servedHeader()).toBe(1n); + await flush(); + await clock.advance(Nano.fromMilli(Milli(budget * 10))); + sub.serve(2n); + expect(await sub.servedSequence()).toBe(2n); + await clock.advance(Nano.fromMilli(budget)); + expect(toStreamCode(fromTransport(await reset))).toBe(StreamCode.Old); + const first = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in first)) throw new Error("expected SUBSCRIBE_DROP"); + expect([first.drop.start, first.drop.end, first.drop.update]).toEqual([1n, 1n, 1n]); + + // Update 2 widens the budget, so group 1 is re-offered. Update 3 restores it within the + // task the publisher yields after an update, before that re-offer opens. + await update(2n, Milli(budget * 100)); + await flush(); + await update(3n, budget); + await flush(); + sub.track.finish(); + sub.track.close(); + expect(await holesToEnd(sub.client.reader)).toEqual([["drop", 1n, 1n, 3n]]); + } finally { + await sub.close(); + installed.close(); + poll.mockRestore(); + } +}); + +// A drop the reader gave up as final with OLD, as one relayed from a lite-06 upstream is, goes out as +// a stale drop on lite-07, which the subscriber reopens on its next update, so each update drops it +// again rather than leave it owed. Mirrors the Rust +// `a_final_old_drop_is_dropped_again_by_each_update_on_lite07`. +test("lite draft-07: a final OLD drop is dropped again by each update", async () => { + const sub = await servedSubscription({ version: Version.DRAFT_07, startGroup: 0n }); + try { + sub.serve(0n); + expect(await sub.servedSequence()).toBe(0n); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07))).toBe(true); + + hooks.dropTrackGroups(sub.track, 1n, 2n, { error: new StreamError(StreamCode.Old) }); + const first = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_07); + if (!("drop" in first)) throw new Error("expected SUBSCRIBE_DROP"); + expect([first.drop.start, first.drop.end, first.drop.update]).toEqual([1n, 1n, 0n]); + + for (const update of [1n, 2n]) { + await replayUpdate({ + priority: 1, + startGroup: 0n, + update, + ...restate(Version.DRAFT_07, update - 1n), + }).encode(sub.client.writer, Version.DRAFT_07); + await flush(); + } + sub.track.finish(); + sub.track.close(); + expect(await holesToEnd(sub.client.reader)).toEqual([ + ["drop", 1n, 1n, 1n], + ["drop", 1n, 1n, 2n], + ]); + } finally { + await sub.close(); + } +}); + // An update that lowers the start below SUBSCRIBE_START asks for those groups too, so the // end sweep drops them, as the Rust publisher does. test("lite draft-06: an update lowering the start is swept at the end", async () => { @@ -1444,7 +1774,7 @@ for (const skipped of ["failed", "stale"] as const) { group.writeFrame(textFrame("old", Timestamp.fromMillis(5))); if (skipped === "failed") group.abort(new Error("cut")); else group.finish(); - if (skipped === "failed") hooks.dropTrackGroups(sub.track, 7n, 8n, new Error("cut")); + if (skipped === "failed") hooks.dropTrackGroups(sub.track, 7n, 8n, { error: new Error("cut") }); sub.serve(6n); const response = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_06); if (!("start" in response)) throw new Error("expected SUBSCRIBE_START"); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index e8e919b8587..e76ae79f119 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -21,7 +21,18 @@ import { } from "../error.ts"; import type * as group from "../group.ts"; import { Cost, type Hop, routesEqual, type WireRoute } from "../hop.ts"; -import { finalSequence, hiddenBelow, hooks, insertRun, presented, type Run, runGaps } from "../internal.ts"; +import { + type DropRun, + finalSequence, + type Hole, + hiddenBelow, + hooks, + insertRun, + presented, + type Run, + removeRun, + runGaps, +} from "../internal.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import * as Path from "../path.ts"; import { readSize, sendPayload } from "../payload.ts"; @@ -54,6 +65,8 @@ import { SubscribeDrop, SubscribeFinish, SubscribeOk, + type SubscribeResponse, + SubscribeSkip, SubscribeStart, SubscribeStarted, SubscribeUpdate, @@ -74,6 +87,7 @@ import { hasProbeRtt, hasRanges, hasRouteCost, + hasSkip, hasSynced, hasTrackStream, keepsSubscribeOpen, @@ -1155,6 +1169,7 @@ export class Publisher { served, floor, bounds, + update: msg.update, initial: BigInt(asked.length), }); @@ -1325,11 +1340,14 @@ export class Publisher { // The read cursor's floor; see {@link cursorFloor}. floor: bigint; bounds: FrameBounds; + /** The SUBSCRIBE's update number (0), which a drop echoes until an update replaces it. */ + update: bigint; // How many ranges the SUBSCRIBE carried, whose starts SUBSCRIBE_START reports. initial: bigint; }, ): Promise { const { sub, broadcast, stamp, publisher, compress, served, bounds, initial } = serving; + let updateNumber = serving.update; // The read cursor's floor; the subscription's ranges cap it. let floor = serving.floor; const moveCursor = () => track.withRange({ start: { included: floor } }); @@ -1344,7 +1362,7 @@ export class Publisher { const emitRange = hasTrackStream(this.version); let startSent = false; // Dispositions before START refine its boundary; later ones must be named as drops. - const skipped: { start: bigint; end: bigint; error: Error }[] = []; + const skipped: DropRun[] = []; // Where SUBSCRIBE_START resolved, lowered by an update that asks for groups below it. let startSequence: bigint | undefined; // The first group and the edge seen before it was popped, held until the source @@ -1401,19 +1419,84 @@ export class Publisher { return true; }; - // Drop whatever part of `[start, end)` this subscription has not been served or dropped - // yet, given up with `err`. - const drop = async (start: bigint, end: bigint, err: Error): Promise => { + // The message naming `[start, end]` given up as `hole` says: lite-07 skips what was never + // produced, and earlier versions drop it as not found. + const holeMessage = (start: bigint, end: bigint, hole: Hole): SubscribeResponse => + hole.skip && hasSkip(this.version) + ? { skip: new SubscribeSkip({ start, end }) } + : { drop: SubscribeDrop.given(start, end, hole.error, this.version, updateNumber) }; + + // Whether lite-07 names `hole` stale: an OLD drop, which the next update reopens. + const sentStale = (hole: Hole) => !hole.skip && toStreamCode(hole.error) === StreamCode.Old; + + // Lite-07: the part of `served` named in SUBSCRIBE_DROP, never served. A skip replaces a + // drop, so it is sent over one of these, but never over a group served. + let dropped: Run[] = []; + // Lite-07: the part of `dropped` named stale (OLD), which a final drop or a group replaces, + // as one a route the front switched to delivers after the update that reopens it. + let droppedStale: Run[] = []; + // Lite-07 groups whose stream expired as stale, which every update drops again under its + // number: the subscriber reopens each stale drop then, and the stream already reset. + const expired: Run[] = []; + + // Give up, as `hole` says, whatever part of `[start, end)` this subscription has not been + // served or given up yet. + const drop = async (start: bigint, end: bigint, hole: Hole): Promise => { if (!served) return true; - for (const gap of runGaps(served, start, end)) { + const gaps = runGaps(served, start, end); + if (hasSkip(this.version)) { + for (const gap of gaps) { + if (hole.skip) { + dropped = removeRun(dropped, gap.start, gap.end); + droppedStale = removeRun(droppedStale, gap.start, gap.end); + continue; + } + insertRun(dropped, gap.start, gap.end); + // The wire names every OLD drop stale, which the next update reopens. One the + // reader gave up as final, such as one relayed from a version whose drops are all + // final, is dropped again by each update, as an expired stream is. + if (!sentStale(hole)) { + droppedStale = removeRun(droppedStale, gap.start, gap.end); + continue; + } + insertRun(droppedStale, gap.start, gap.end); + if (hole.stale === undefined) insertRun(expired, gap.start, gap.end); + } + } + for (const gap of gaps) { insertRun(served, gap.start, gap.end); - const msg = SubscribeDrop.given(gap.start, gap.end - 1n, err, this.version); - if (!(await controls.response(encodeSubscribeResponse(stream, { drop: msg }, this.version)))) - return false; + const msg = holeMessage(gap.start, gap.end - 1n, hole); + if (!(await controls.response(encodeSubscribeResponse(stream, msg, this.version)))) return false; } return true; }; + // Forward a verdict the reader reached on `[start, end)`, as `drop` does. On lite-07 it also + // replaces what it overrides: a skip any drop already sent, and a final (non-OLD) drop a stale one, + // which the next update would otherwise reopen. A reader the earlier drop ended stays ended. + const forward = async (start: bigint, end: bigint, hole: Hole): Promise => { + if (served && hasSkip(this.version)) { + const replaces = hole.skip ? dropped : sentStale(hole) ? [] : droppedStale; + for (const run of [...replaces]) { + const from = max(run.start, start); + const to = min(run.end, end); + if (from >= to) continue; + replaceStale(from, to); + served.splice(0, served.length, ...removeRun(served, from, to)); + } + } + return drop(start, end, hole); + }; + + // Forget that `[start, end)` went out as a drop: a verdict or group replaces it. A stream of + // this subscription that expired there is no longer owed again on an update, since what + // replaced it settles it. + const replaceStale = (start: bigint, end: bigint) => { + dropped = removeRun(dropped, start, end); + droppedStale = removeRun(droppedStale, start, end); + expired.splice(0, expired.length, ...removeRun(expired, start, end)); + }; + // The track's exclusive final boundary. A Rust subscriber feeds SUBSCRIBE_FINISH // straight into finish_at, so it must name the track's boundary (which counts // datagram sequences too), not the delivered range: a subscription cap can hold @@ -1461,7 +1544,10 @@ export class Publisher { // Every group this subscription started serving, until its stream finishes or resets. const groups = new Set>(); const drops = new Signal(0); - const pendingDrops: SubscribeDrop[] = []; + // Groups whose stream failed, with why, for the control stream to name, as runs. + const pendingDrops: { start: bigint; end: bigint; error: Error }[] = []; + // The expired groups an update re-offers, still cached and in range, served again in turn. + const reoffered: group.Consumer[] = []; // Cancels groups still queued for a stream slot. Only the subscriber leaving counts: // a track that ran out of groups still has to flush the ones already queued, and the @@ -1478,15 +1564,23 @@ export class Publisher { // What was given up below the start is never named, so a wider budget may serve it. if (range.start < startSequence) hooks.unreport(track, range.start, min(range.end, startSequence)); - if (!(await drop(max(range.start, startSequence), range.end, range.error))) return false; + if (!(await forward(max(range.start, startSequence), range.end, range))) return false; } skipped.length = 0; } if (pendingDrops.length > 0) { - for (const dropped of pendingDrops.splice(0)) { - if ( - !(await controls.response(encodeSubscribeResponse(stream, { drop: dropped }, this.version))) - ) + for (const { start, end, error } of pendingDrops.splice(0)) { + // A stream expired under this subscription's budget is a stale verdict, which + // a later verdict or group replaces as any stale drop sent. + if (hasSkip(this.version) && toStreamCode(error) === StreamCode.Old) { + insertRun(expired, start, end); + insertRun(dropped, start, end); + insertRun(droppedStale, start, end); + } + // A re-offer an update owed again is given up now, so the end sweep never names it. + if (served) insertRun(served, start, end); + const msg = SubscribeDrop.given(start, end - 1n, error, this.version, updateNumber); + if (!(await controls.response(encodeSubscribeResponse(stream, { drop: msg }, this.version)))) return false; } } @@ -1510,6 +1604,7 @@ export class Publisher { // An update may reach the finished track's history, so it is served again. finished = false; const update = control.update; + updateNumber = update.update; if (hasRanges(this.version)) { // A range retired before its start resolved is owed no SUBSCRIBE_STARTED. const live = new Set(control.ids); @@ -1546,6 +1641,45 @@ export class Publisher { bounds.endGroup = update.endGroup; bounds.endFrame = update.endFrame; moveCursor(); + // Lite-07 reopens every stale drop on any update, judged under its new terms + // (applied as the update was decoded): what was reported stale is offered + // again, and dropped again under this update's number if still stale. + if (hasSkip(this.version)) { + const owe = (start: bigint, end: bigint) => { + if (served) served.splice(0, served.length, ...removeRun(served, start, end)); + dropped = removeRun(dropped, start, end); + droppedStale = removeRun(droppedStale, start, end); + }; + for (const run of hooks.reofferStale(track)) owe(run.start, run.end); + // A stale drop not sent yet, such as one an earlier update left to drop + // again, is judged again under these terms too. + for (const pending of pendingDrops.filter( + (d) => toStreamCode(d.error) === StreamCode.Old, + )) + insertRun(expired, pending.start, pending.end); + pendingDrops.splice( + 0, + pendingDrops.length, + ...pendingDrops.filter((d) => toStreamCode(d.error) !== StreamCode.Old), + ); + // A group re-offered that has not opened yet is judged again too, since these + // terms may no longer admit it. + for (const group of reoffered.splice(0)) { + insertRun(expired, group.sequence, group.sequence + 1n); + group.close(); + } + // A group whose stream already reset as stale: one still cached, in range, and + // fresh under the new budget is served again; the rest is dropped again, by run. + for (const run of expired.splice(0)) { + const { groups, givenUp } = hooks.reofferGroups(track, run.start, run.end); + for (const group of groups) { + owe(group.sequence, group.sequence + 1n); + reoffered.push(group); + } + for (const gone of givenUp) + pendingDrops.push({ ...gone, error: new StreamError(StreamCode.Old) }); + } + } // A microtask is too short: decoding one framed update crosses several // awaits, each of which can requeue behind the serving continuation. A // task boundary usually lets the decoder finish whatever the transport @@ -1567,16 +1701,20 @@ export class Publisher { !startSent && hasLargest(this.version) ? newer(first?.largest, newer(hooks.bufferedLargest(track), hooks.peekLargest(track).largest)) : undefined; + // A group an update re-offers goes ahead of new ones. + const again = first ? undefined : reoffered.shift(); const recv = first ? ({ kind: "group", group: first.group } as const) - : hooks.tryRecvGroup(track, emitRange); + : again + ? ({ kind: "group", group: again } as const) + : hooks.tryRecvGroup(track, emitRange); first = undefined; switch (recv.kind) { case "error": throw recv.error; case "drop": - if (!startSent) skipped.push({ start: recv.start, end: recv.end, error: recv.error }); - else if (!(await drop(recv.start, recv.end, recv.error))) return false; + if (!startSent) skipped.push(recv); + else if (!(await forward(recv.start, recv.end, recv))) return false; continue; case "idle": // A start past everything the track has (a subscriber resuming just after @@ -1678,7 +1816,11 @@ export class Publisher { // Name every sequence from the start to the end this subscription never // got. A cap below the end bounds what the subscriber is owed, as on its side. if (startSequence !== undefined) { - const unserved = new StreamError(StreamCode.NotFound, { message: "group not served" }); + // A sequence the publisher declared skipped went out as a skip already; one + // it never declared might have existed, so it is not found. + const unserved: Hole = { + error: new StreamError(StreamCode.NotFound, { message: "group not served" }), + }; // Only the groups inside the ranges are owed. for (const run of hooks.servingRuns(track, startSequence, boundary())) { if (!(await drop(run.start, run.end, unserved))) return false; @@ -1704,6 +1846,18 @@ export class Publisher { } const group = recv.group; + // Never a second stream for a sequence this subscription already accounted for + // (served, sent as a datagram, or dropped): before lite-07 a subscriber settles a + // drop for good. A stale drop a lite-07 update reopened is owed again (see the update above). + // A lite-07 stale drop is no final verdict, and a Group Stream wins over it, so a group + // handed out after one is served, as one a route the front switched to delivers: the + // update that reopens the drop would never re-offer it. + if (runGaps(droppedStale, group.sequence, group.sequence + 1n).length === 0) { + replaceStale(group.sequence, group.sequence + 1n); + } else if (served && runGaps(served, group.sequence, group.sequence + 1n).length === 0) { + group.close(); + continue; + } const range = frameRange(bounds, group.sequence); if (emitRange && !startSent) { @@ -1760,7 +1914,7 @@ export class Publisher { const task = opened.then(async (writer) => { const failed = writer instanceof Error ? writer : await this.#serveGroup(writer, options); if (emitRange && failed instanceof Error) { - pendingDrops.push(SubscribeDrop.given(group.sequence, group.sequence, failed, this.version)); + pendingDrops.push({ start: group.sequence, end: group.sequence + 1n, error: failed }); drops.update((revision) => revision + 1); } }); @@ -1772,6 +1926,8 @@ export class Publisher { } } finally { if (!finished) unsubscribe(); + // An update's re-offers are reads of their own, which closing the track leaves open. + for (const group of reoffered.splice(0)) group.close(); // A lite-07 subscriber FINs once it read the finished track's tail, or once it wants // nothing more, even while a reply is still being written. Either way the groups still // waiting for a slot are cancelled, but an opened stream may be a final group whose FIN diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index a50d7874673..7b5e9b01bce 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -14,6 +14,7 @@ import { SubscribeFinish, SubscribeOk, type SubscribeResponse, + SubscribeSkip, SubscribeStart, SubscribeStarted, SubscribeUpdate, @@ -199,10 +200,10 @@ test("lite-07 carries the span as its only range, exclusive at the end", async ( test("a lite-07 list decodes as the span covering it", async () => { // Priority, order, max delay, and an unstated min delay as SubscribeUpdate writes them, - // then no Range ID to retire, the ranges it adds, and an absent wanted group. + // then no Range ID to retire, the ranges it adds, the update number, and an absent wanted group. const head = (await encodeMessage(Version.DRAFT_07, new SubscribeUpdate({ priority: 0, ranges: [] }))).slice(1, 5); const update = async (ranges: number[]) => { - const body = [...head, 0, ...ranges, 0]; + const body = [...head, 0, ...ranges, 0, 0]; const got = await SubscribeUpdate.decode( new Reader(undefined, new Uint8Array([body.length, ...body]), Version.DRAFT_07), Version.DRAFT_07, @@ -335,12 +336,37 @@ test("SubscribeFinish carries its error on draft-07", async () => { test("SubscribeDrop is restored on draft-07", async () => { const drop: SubscribeResponse = { drop: new SubscribeDrop({ start: 1n, end: 3n, error: 7 }) }; const wire = await encode(Version.DRAFT_07, drop); - expect(wire).toEqual(new Uint8Array([2, 3, 1, 3, 7])); + // The drop names the SUBSCRIBE's update, 0. + expect(wire).toEqual(new Uint8Array([2, 4, 1, 3, 7, 0])); const got = await responseRoundtrip(Version.DRAFT_07, drop); if (!("drop" in got)) throw new Error("expected drop"); expect([got.drop.start, got.drop.end, got.drop.error]).toEqual([1n, 3n, 7]); }); +test("a drop names its update on draft-07 alone", async () => { + const drop = new SubscribeDrop({ start: 4n, end: 4n, error: 0x34, update: 2n }); + expect(await encode(Version.DRAFT_07, { drop })).toEqual(new Uint8Array([2, 4, 4, 4, 0x34, 2])); + const got = await responseRoundtrip(Version.DRAFT_07, { drop }); + if (!("drop" in got)) throw new Error("expected drop"); + expect(got.drop.update).toBe(2n); + const old = new SubscribeDrop({ start: 4n, end: 4n, error: 0, update: 2n }); + expect(await encode(Version.DRAFT_06, { drop: old })).toEqual(new Uint8Array([2, 3, 4, 4, 0])); + const back = await responseRoundtrip(Version.DRAFT_06, { drop: old }); + if (!("drop" in back)) throw new Error("expected drop"); + expect(back.drop.update).toBe(0n); +}); + +test("SubscribeSkip is type 3 on draft-07 alone", async () => { + const skip: SubscribeResponse = { skip: new SubscribeSkip({ start: 1n, end: 3n }) }; + expect(await encode(Version.DRAFT_07, skip)).toEqual(new Uint8Array([3, 2, 1, 3])); + const got = await responseRoundtrip(Version.DRAFT_07, skip); + if (!("skip" in got)) throw new Error("expected skip"); + expect([got.skip.start, got.skip.end]).toEqual([1n, 3n]); + await expect(encode(Version.DRAFT_06, skip)).rejects.toThrow(); + const malformed = new Reader(undefined, new Uint8Array([2, 3, 1]), Version.DRAFT_07); + await expect(SubscribeSkip.decode(malformed)).rejects.toThrow("invalid skipped range"); +}); + test("SubscribeDrop is type 0x2 on draft-05 and 0x1 on draft-04", async () => { const drop: SubscribeResponse = { drop: new SubscribeDrop({ start: 1n, end: 3n, error: 0 }) }; @@ -479,7 +505,7 @@ test("wanted groups refuse malformed input before writing bytes", async () => { for (const version of [Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07]) { test(`SUBSCRIBE_DROP refuses reversed ranges while retaining empty ranges on ${version}`, async () => { const malformed = new Reader(undefined, new Uint8Array([3, 3, 1, 0]), version); - await expect(SubscribeDrop.decode(malformed)).rejects.toThrow("invalid dropped range"); + await expect(SubscribeDrop.decode(malformed, version)).rejects.toThrow("invalid dropped range"); await expect(encode(version, { drop: new SubscribeDrop({ start: 3n, end: 1n, error: 0 }) })).rejects.toThrow(); for (const [start, end] of [ [2n, 1n], diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 53e47deb676..22a6e59bcf6 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -25,10 +25,21 @@ import { hasOrder, hasRanges, hasRefresh, + hasSkip, resolvesStart, Version, } from "./version.ts"; +/** Decode the `Update` number after the ranges. A wire without the field decodes as 0. */ +async function decodeUpdate(r: Reader, version: Version): Promise { + return hasSkip(version) ? await r.u62() : 0n; +} + +/** Encode the `Update` number, dropping it on a wire without the field. */ +async function encodeUpdate(w: Writer, version: Version, update: bigint) { + if (hasSkip(version)) await w.u62(update); +} + /** * Encode the `Group Start` field shared by SUBSCRIBE and SUBSCRIBE_UPDATE. * @@ -646,6 +657,8 @@ export class SubscribeUpdate { * {@link remove}; {@link LiveRanges.apply} resolves a received update to every live range. */ ranges?: WireRange[]; + /** See {@link Subscribe.update}: larger than the subscription's previous one. */ + update: bigint; /** Lite-07: the Range IDs of the live ranges to retire before adding {@link ranges}. */ remove?: bigint[]; /** Lite-07: the ranges a received update adds, with the positions they name; see {@link RangeEntry}. */ @@ -662,6 +675,7 @@ export class SubscribeUpdate { startFrame?: number; endFrame?: number; ranges?: WireRange[]; + update?: bigint; remove?: bigint[]; entries?: RangeEntry[]; }) { @@ -677,6 +691,7 @@ export class SubscribeUpdate { this.startFrame = props.startFrame ?? 0; this.endFrame = props.endFrame; this.ranges = props.ranges; + this.update = props.update ?? 0n; this.remove = props.remove; this.entries = props.entries; } @@ -712,6 +727,7 @@ export class SubscribeUpdate { } else { await encodeSpan(w, version, this, this.ranges); } + await encodeUpdate(w, version, this.update); if (hasRefresh(version)) await w.u62(this.wantedGroup === undefined ? 0n : this.wantedGroup + 1n); break; } @@ -737,6 +753,7 @@ export class SubscribeUpdate { } else { span = await decodeSpan(r, version); } + const update = await decodeUpdate(r, version); const wanted = hasRefresh(version) ? await r.u62() : 0n; return new SubscribeUpdate({ wantedGroup: wanted === 0n ? undefined : wanted - 1n, @@ -745,6 +762,7 @@ export class SubscribeUpdate { maxDelay, minDelay, ...span, + update, }); } } @@ -809,6 +827,13 @@ export class Subscribe { /** Lite-07: the decoded ranges with the positions they name; see {@link RangeEntry}. */ entries?: RangeEntry[]; + /** + * The number of the terms in effect: the SUBSCRIBE is 0 and every SUBSCRIBE_UPDATE carries a + * larger one, which a {@link SubscribeDrop} echoes. Lite-07+; earlier wires decode as 0 and + * drop it on encode. + */ + update: bigint; + constructor(props: { id: bigint; broadcast: Path.Valid; @@ -823,6 +848,7 @@ export class Subscribe { startFrame?: number; endFrame?: number; ranges?: WireRange[]; + update?: bigint; entries?: RangeEntry[]; }) { this.id = props.id; @@ -838,6 +864,7 @@ export class Subscribe { this.startFrame = props.startFrame ?? 0; this.endFrame = props.endFrame; this.ranges = props.ranges; + this.update = props.update ?? 0n; this.entries = props.entries; } @@ -859,6 +886,7 @@ export class Subscribe { await encodeSpan(w, version, this, this.ranges); break; } + await encodeUpdate(w, version, this.update); } static async #decode(r: Reader, version: Version): Promise { @@ -877,6 +905,7 @@ export class Subscribe { const maxDelay = await r.u53(); const minDelay = await decodeMinDelay(r, version); const span = await decodeSpan(r, version); + const update = await decodeUpdate(r, version); return new Subscribe({ id, broadcast, @@ -887,6 +916,7 @@ export class Subscribe { maxDelay, minDelay, ...span, + update, }); } } @@ -1124,24 +1154,40 @@ export class SubscribeStarted { } } -/// Indicates that one or more groups have been dropped. -/// -/// Draft-03+; publishers send it on Draft-05 and later. +/** + * Names groups that existed but will not be delivered: aborted or reset, evicted or expired with + * no copy left, or stale. + * + * Draft-03+; publishers send it on Draft-05 and later. Before lite-07 it also names sequences + * declared skipped, which lite-07 names in {@link SubscribeSkip}. + */ export class SubscribeDrop { start: bigint; end: bigint; error: number; + /** + * The number of the SUBSCRIBE or SUBSCRIBE_UPDATE in effect when the groups were given up + * ({@link Subscribe.update}): a stale (OLD) drop under a superseded one is no verdict. + * Lite-07+; earlier wires decode as 0. + */ + update: bigint; - constructor(props: { start: bigint; end: bigint; error: number }) { + constructor(props: { start: bigint; end: bigint; error: number; update?: bigint }) { this.start = props.start; this.end = props.end; this.error = props.error; + this.update = props.update ?? 0n; } - /** Name `[start, end]` given up with `err`, as `version` carries it. */ - static given(start: bigint, end: bigint, err: unknown, version: Version): SubscribeDrop { - // Before lite-07 the code is the application's, so it says nothing of why. - return new SubscribeDrop({ start, end, error: hasDropCode(version) ? toStreamCode(err) : 0 }); + /** Name `[start, end]` given up with `err` under the terms numbered `update`, as `version` carries it. */ + static given(start: bigint, end: bigint, err: unknown, version: Version, update = 0n): SubscribeDrop { + return new SubscribeDrop({ + start, + end, + // Before lite-07 the code is the application's, so it says nothing of why. + error: hasDropCode(version) ? toStreamCode(err) : 0, + update: hasSkip(version) ? update : 0n, + }); } /** @@ -1157,25 +1203,64 @@ export class SubscribeDrop { return fromTransport({ source: "stream", streamErrorCode: this.error }); } - async #encode(w: Writer) { + async #encode(w: Writer, version: Version) { if (this.start - this.end > 1n) throw new RangeError("invalid dropped range"); await w.u62(this.start); await w.u62(this.end); await w.u53(this.error); + await encodeUpdate(w, version, this.update); + } + + static async #decode(r: Reader, version: Version): Promise { + const start = await r.u62(); + const end = await r.u62(); + if (start - end > 1n) throw new ProtocolViolation("invalid dropped range"); + const error = await r.u53(); + const update = await decodeUpdate(r, version); + return new SubscribeDrop({ start, end, error, update }); + } + + async encode(w: Writer, version: Version): Promise { + return Message.encode(w, (w) => this.#encode(w, version)); + } + + static async decode(r: Reader, version: Version): Promise { + return Message.decode(r, (r) => SubscribeDrop.#decode(r, version)); + } +} + +/** + * Names sequences the publisher declared it will never produce (see `Producer.skipGroups`): + * final, with no error, ending no reader. Lite-07+; earlier versions name them in a + * {@link SubscribeDrop}. + */ +export class SubscribeSkip { + start: bigint; + end: bigint; + + constructor(props: { start: bigint; end: bigint }) { + this.start = props.start; + this.end = props.end; + } + + async #encode(w: Writer) { + if (this.start - this.end > 1n) throw new RangeError("invalid skipped range"); + await w.u62(this.start); + await w.u62(this.end); } - static async #decode(r: Reader): Promise { - const drop = new SubscribeDrop({ start: await r.u62(), end: await r.u62(), error: await r.u53() }); - if (drop.start - drop.end > 1n) throw new ProtocolViolation("invalid dropped range"); - return drop; + static async #decode(r: Reader): Promise { + const skip = new SubscribeSkip({ start: await r.u62(), end: await r.u62() }); + if (skip.start - skip.end > 1n) throw new ProtocolViolation("invalid skipped range"); + return skip; } async encode(w: Writer): Promise { return Message.encode(w, this.#encode.bind(this)); } - static async decode(r: Reader): Promise { - return Message.decode(r, SubscribeDrop.#decode); + static async decode(r: Reader): Promise { + return Message.decode(r, SubscribeSkip.#decode); } } @@ -1188,13 +1273,14 @@ export class SubscribeDrop { * - Draft-05+: `0x0` SUBSCRIBE_START (SUBSCRIBE_OK on lite-07), `0x1` SUBSCRIBE_FINISH * (SUBSCRIBE_END before lite-07), `0x2` SUBSCRIBE_DROP (SUBSCRIBE_OK was removed; * acceptance is implicit). - * - Lite-07+: also `0x4` SUBSCRIBE_STARTED. + * - Lite-07+: also `0x3` SUBSCRIBE_SKIP and `0x4` SUBSCRIBE_STARTED. */ export type SubscribeResponse = | { ok: SubscribeOk } | { start: SubscribeStart } | { finish: SubscribeFinish } | { drop: SubscribeDrop } + | { skip: SubscribeSkip } | { started: SubscribeStarted }; export async function encodeSubscribeResponse(w: Writer, resp: SubscribeResponse, version: Version): Promise { @@ -1214,7 +1300,7 @@ export async function encodeSubscribeResponse(w: Writer, resp: SubscribeResponse await resp.ok.encode(w, version); } else if ("drop" in resp) { await w.u53(0x1); - await resp.drop.encode(w); + await resp.drop.encode(w, version); } else { throw new Error("SUBSCRIBE_START/FINISH not supported for this version"); } @@ -1229,7 +1315,10 @@ export async function encodeSubscribeResponse(w: Writer, resp: SubscribeResponse await resp.finish.encode(w, version); } else if ("drop" in resp) { await w.u53(0x2); - await resp.drop.encode(w); + await resp.drop.encode(w, version); + } else if ("skip" in resp && hasSkip(version)) { + await w.u53(0x3); + await resp.skip.encode(w); } else if ("started" in resp && hasRanges(version)) { await w.u53(0x4); await resp.started.encode(w); @@ -1252,7 +1341,7 @@ export async function decodeSubscribeResponse(r: Reader, version: Version): Prom case 0x0: return { ok: await SubscribeOk.decode(r, version) }; case 0x1: - return { drop: await SubscribeDrop.decode(r) }; + return { drop: await SubscribeDrop.decode(r, version) }; default: throw new ProtocolViolation(`unknown subscribe response type: ${typ}`); } @@ -1265,7 +1354,10 @@ export async function decodeSubscribeResponse(r: Reader, version: Version): Prom case 0x1: return { finish: await SubscribeFinish.decode(r, version) }; case 0x2: - return { drop: await SubscribeDrop.decode(r) }; + return { drop: await SubscribeDrop.decode(r, version) }; + case 0x3: + if (!hasSkip(version)) throw new ProtocolViolation(`unknown subscribe response type: ${typ}`); + return { skip: await SubscribeSkip.decode(r) }; case 0x4: if (!hasRanges(version)) throw new ProtocolViolation(`unknown subscribe response type: ${typ}`); return { started: await SubscribeStarted.decode(r) }; diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 505a591c083..ffe91c753f1 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -12,6 +12,7 @@ import { reason, SessionCode, SessionError, + Skipped, StreamCode, StreamError, TimestampMismatch, @@ -20,7 +21,7 @@ import type { Consumer as GroupConsumer } from "../group.ts"; import { HopSchema, isAnonymous, MAX_HOPS, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; import { hooks } from "../internal.ts"; -import { GRACE } from "../ledger.ts"; +import { CACHE_EXPIRY, GRACE } from "../ledger.ts"; import * as Path from "../path.ts"; import { Reader, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; @@ -36,9 +37,11 @@ import { StreamId } from "./stream.ts"; import { encodeSubscribeResponse, Subscribe, + SubscribeDrop, SubscribeFinish, SubscribeOk, type SubscribeResponse, + SubscribeSkip, SubscribeStart, SubscribeStarted, SubscribeUpdate, @@ -1450,6 +1453,79 @@ test("a group that beats its track info waits for it", async () => { } }); +// A lite-07 re-offer can overtake the reset of the stream it replaces: it waits, unread, until that +// copy ends. It is refused as a duplicate once the copy finishes, and takes the group once it fails. +test.each([false, true])("a duplicate of an arriving copy waits for it to settle (first fails: %p)", async (fails) => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + try { + await opened(0); + await (await opened(1)).writing; + await answerTrackInfo(streams[0], version, 1000); + + // The original copy, still arriving: one frame in, no end yet. + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + await writer.u62(0n); + await writer.u53(1); + await writer.write(new Uint8Array([7])); + let first!: ReadableStreamDefaultController; + const arriving = new ReadableStream({ + start: (controller) => { + first = controller; + }, + }); + const original = subscriber.runGroup( + new Group({ subscribe: 0n, sequence: 0n }), + new Reader(arriving, undefined, version), + ); + for (const chunk of chunks) first.enqueue(chunk); + const group = await track.recvGroup(); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([7])); + + const duplicate = await groupStream(0, version); + const stop = spyOn(duplicate, "stop"); + let held = true; + const receiving = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), duplicate).finally(() => { + held = false; + }); + await drainReactions(); + expect(held).toBe(true); + expect(stop).not.toHaveBeenCalled(); + + if (!fails) { + first.close(); + await original; + await receiving; + expect(stop.mock.calls[0]?.[0]).toBeInstanceOf(Error); + expect(await group?.readFrame()).toBeUndefined(); + return; + } + + // The reset lands: the held stream takes the group. + first.error(new Error("peer reset")); + await original; + await receiving; + expect(await group?.closed).toBeInstanceOf(Error); + const later = subscriber.consume(Path.from("room")).track("video").subscribe(); + try { + const again = await later.recvGroup(); + expect(again?.sequence).toBe(0n); + expect((await again?.readFrame())?.payload).toEqual(new Uint8Array([0])); + } finally { + later.close(); + } + } finally { + track.close(); + subscriber.close(); + } +}); + // A TRACK reset while its SUBSCRIBE is live fails the subscription, so the origin can try another // route, and resets the SUBSCRIBE. test("a reset track rejects the subscription and resets its subscribe", async () => { @@ -2617,6 +2693,310 @@ test("a widened range's SUBSCRIBE_STARTED resolves the feed's start", async () = } }); +// A group that arrives over a lite-07 stale (OLD) drop wins over it, whichever lands first: the +// publisher serves a group after its stale drop, as a front that switched routes does, and the +// group's stream may overtake the drop. The next update owes it no more, and a group still +// arriving is never aborted by the drop. Mirrors the Rust `a_group_wins_over_its_stale_drop`. +test("a group wins over its stale drop on lite-07", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + type Entry = { + track: Producer; + tail: { covers(start: bigint, end: bigint): boolean; open(sequence: bigint): () => void }; + }; + let entry: Entry | undefined; + const internal = subscriber as unknown as { + runResponses: (stream: unknown, entry: Entry) => Promise; + }; + const runResponses = internal.runResponses.bind(internal); + const responses = spyOn(internal, "runResponses").mockImplementation((stream, captured) => { + entry = captured; + return runResponses(stream, captured); + }); + const broadcast = subscriber.consume(Path.from("room")); + const track = broadcast.track("video").subscribe({ delay: { max: Milli(500) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version); + const stream = await opened(1); + await stream.reading; + const send = async (...resps: SubscribeResponse[]) => { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + for (const resp of resps) await encodeSubscribeResponse(writer, resp, version); + for (const chunk of chunks) stream.inbound.enqueue(chunk); + await drainReactions(); + }; + const old = (sequence: bigint): SubscribeResponse => ({ + drop: new SubscribeDrop({ start: sequence, end: sequence, error: StreamCode.Old }), + }); + // A group stream's header: the tail accounts for it, and the track holds the group. + const arrive = (sequence: bigint) => { + if (!entry) throw new Error("no subscription"); + const read = entry.tail.open(sequence); + return { group: hooks.receiveGroup(entry.track, { sequence, start: 0 }), read }; + }; + + await send({ start: new SubscribeStart([0n]) }); + // OLD, then the group, which finishes. + await send(old(1n)); + const one = arrive(1n); + one.group.finish(); + one.read(); + // The group first, still arriving when its OLD lands. + const two = arrive(2n); + await send(old(2n)); + // OLD, then the group, still arriving at the update. + await send(old(3n)); + const three = arrive(3n); + // OLD, and no group. + await send(old(4n)); + if (!entry) throw new Error("no subscription"); + + track.update({ priority: 1, delay: { max: Milli(500) } }); + await drainReactions(); + // Every group that arrived stays accounted for; the one that never arrived is owed again. + expect(entry.tail.covers(1n, 4n)).toBe(true); + expect(entry.tail.covers(4n, 5n)).toBe(false); + // Neither the drop nor the update aborts a group still arriving. + expect(two.group.closed.peek()).toBeUndefined(); + expect(three.group.closed.peek()).toBeUndefined(); + two.read(); + three.read(); + } finally { + responses.mockRestore(); + track.close(); + broadcast.close(); + subscriber.close(); + } +}); + +// A group whose stream finished wins over a lite-07 stale drop that lands after the cache evicted +// it: the publisher served it over that drop, so the next update owes it no more. A stream that +// finished under an earlier update says nothing of a drop under the new one. Mirrors the Rust +// `a_finished_group_wins_over_a_late_stale_drop_once_evicted`. +test("a finished group wins over a late stale drop once evicted on lite-07", async () => { + const clock = new Manual(); + const installed = clock.install(); + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + type Entry = { + track: Producer; + tail: { + covers(start: bigint, end: bigint): boolean; + }; + }; + let entry: Entry | undefined; + const internal = subscriber as unknown as { + runResponses: (stream: unknown, entry: Entry) => Promise; + }; + const runResponses = internal.runResponses.bind(internal); + const responses = spyOn(internal, "runResponses").mockImplementation((stream, captured) => { + entry = captured; + return runResponses(stream, captured); + }); + const broadcast = subscriber.consume(Path.from("room")); + const track = broadcast.track("video").subscribe({ delay: { max: Milli(500) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version); + const stream = await opened(1); + await stream.reading; + const send = async (...resps: SubscribeResponse[]) => { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + for (const resp of resps) await encodeSubscribeResponse(writer, resp, version); + for (const chunk of chunks) stream.inbound.enqueue(chunk); + await drainReactions(); + }; + const old = (sequence: bigint, update: bigint): SubscribeResponse => ({ + drop: new SubscribeDrop({ start: sequence, end: sequence, error: StreamCode.Old, update }), + }); + // A group stream that arrives whole, through the path that records it. + const arrive = (sequence: bigint) => + subscriber.runGroup( + new Group({ subscribe: 0n, sequence }), + new Reader(new ReadableStream({ start: (controller) => controller.close() }), undefined, version), + ); + // Everything below the live edge ages out of the cache. + const evict = () => clock.advance(Nano.fromMilli(Milli(2 * CACHE_EXPIRY))); + let priority = 0; + const update = async () => { + track.update({ priority: ++priority, delay: { max: Milli(500) } }); + await drainReactions(); + }; + + await send({ start: new SubscribeStart([0n]) }); + await arrive(1n); + await arrive(10n); + await evict(); + await send(old(1n, 0n)); + await update(); + if (!entry) throw new Error("no subscription"); + expect(entry.tail.covers(1n, 2n)).toBe(true); + + // Finished under the first update, then judged stale under the second. + await arrive(2n); + await update(); + await evict(); + await send(old(2n, 2n)); + await update(); + expect(entry.tail.covers(2n, 3n)).toBe(false); + } finally { + responses.mockRestore(); + track.close(); + broadcast.close(); + subscriber.close(); + installed.close(); + } +}); + +// A SKIP, or any final drop, that follows an OLD drop of the same groups replaces it: an update +// reopens only stale verdicts, so the groups stay accounted for, and the skip stays final rather +// than being superseded with the OLD drop. An empty range names nothing. Mirrors Rust. +test("a skip or final drop replaces an earlier stale one on lite-07", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + type Entry = { track: Producer; tail: { covers(start: bigint, end: bigint): boolean } }; + let entry: Entry | undefined; + const internal = subscriber as unknown as { + runResponses: (stream: unknown, entry: Entry) => Promise; + }; + const runResponses = internal.runResponses.bind(internal); + const responses = spyOn(internal, "runResponses").mockImplementation((stream, captured) => { + entry = captured; + return runResponses(stream, captured); + }); + const broadcast = subscriber.consume(Path.from("room")); + const track = broadcast.track("video").subscribe({ delay: { max: Milli(500) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version); + const stream = await opened(1); + await stream.reading; + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + const old = (start: bigint, end: bigint): SubscribeResponse => ({ + drop: new SubscribeDrop({ start, end, error: StreamCode.Old }), + }); + for (const resp of [ + { start: new SubscribeStart([0n]) }, + old(1n, 2n), + old(4n, 4n), + { skip: new SubscribeSkip({ start: 1n, end: 2n }) }, + { drop: new SubscribeDrop({ start: 4n, end: 4n, error: StreamCode.NotFound }) }, + // Empty ranges. + old(6n, 5n), + { skip: new SubscribeSkip({ start: 6n, end: 5n }) }, + ] satisfies SubscribeResponse[]) { + await encodeSubscribeResponse(writer, resp, version); + } + for (const chunk of chunks) stream.inbound.enqueue(chunk); + await drainReactions(); + if (!entry) throw new Error("no subscription"); + + track.update({ priority: 1, delay: { max: Milli(500) } }); + await drainReactions(); + expect(entry.tail.covers(1n, 3n)).toBe(true); + expect(entry.tail.covers(4n, 5n)).toBe(true); + // Still skipped after the update. + expect(() => entry?.track.createGroup(1n)).toThrow(Skipped); + } finally { + responses.mockRestore(); + track.close(); + broadcast.close(); + subscriber.close(); + } +}); + +// A SKIP replaces an earlier final DROP of the same groups too: "never produced" is the most +// authoritative verdict. A later DROP leaves the skip in place, and a SKIP naming a group that +// arrived keeps the group, logging a protocol error without ending the session. Mirrors Rust. +test("a skip replaces a final drop but not a cached group on lite-07", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + type Entry = { track: Producer; tail: { covers(start: bigint, end: bigint): boolean } }; + let entry: Entry | undefined; + const internal = subscriber as unknown as { + runResponses: (stream: unknown, entry: Entry) => Promise; + }; + const runResponses = internal.runResponses.bind(internal); + const responses = spyOn(internal, "runResponses").mockImplementation((stream, captured) => { + entry = captured; + return runResponses(stream, captured); + }); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const broadcast = subscriber.consume(Path.from("room")); + const track = broadcast.track("video").subscribe({ delay: { max: Milli(500) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version); + const stream = await opened(1); + await stream.reading; + const send = async (resps: SubscribeResponse[]) => { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + for (const resp of resps) await encodeSubscribeResponse(writer, resp, version); + for (const chunk of chunks) stream.inbound.enqueue(chunk); + await drainReactions(); + }; + await send([ + { start: new SubscribeStart([0n]) }, + { drop: new SubscribeDrop({ start: 1n, end: 2n, error: StreamCode.NotFound }) }, + ]); + if (!entry) throw new Error("no subscription"); + entry.track.createGroup(5n).finish(); + + await send([ + { skip: new SubscribeSkip({ start: 1n, end: 2n }) }, + // A later drop leaves the skip in place. + { drop: new SubscribeDrop({ start: 2n, end: 2n, error: StreamCode.NotFound }) }, + // Group 5 arrived, so the skip names only 4 and 6. + { skip: new SubscribeSkip({ start: 4n, end: 6n }) }, + ]); + expect(() => entry?.track.createGroup(1n)).toThrow(Skipped); + expect(() => entry?.track.createGroup(2n)).toThrow(Skipped); + expect(entry.tail.covers(4n, 7n)).toBe(true); + expect(warn).toHaveBeenCalledTimes(1); + + const reader = entry.track.subscribe({ delay: { max: null } }); + const got: unknown[] = []; + for ( + let next = hooks.tryRecvGroup(reader, true); + next.kind !== "idle"; + next = hooks.tryRecvGroup(reader, true) + ) { + if (next.kind === "group") got.push(next.group.sequence); + else if (next.kind === "drop") got.push([next.start, next.end, next.skip === true]); + else break; + } + expect(got).toEqual([[1n, 3n, true], [4n, 5n, true], [6n, 7n, true], 5n]); + reader.close(); + } finally { + warn.mockRestore(); + responses.mockRestore(); + track.close(); + broadcast.close(); + subscriber.close(); + } +}); + // Before lite-05 the publisher FINs the subscribe stream only once the track is over, so that FIN // finishes the track once the tail settles, rather than releasing it for another route. test.each( @@ -2781,6 +3161,41 @@ test("an update whose bound the wire can't carry fails the track", async () => { } }); +// A SUBSCRIBE_DROP can only echo an update the subscriber sent: one naming a later update is a +// protocol violation. Mirrors the Rust `a_drop_naming_an_unsent_update_is_a_protocol_violation`. +test("a lite-07 drop naming an unsent update closes the session", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened, closes } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const broadcast = subscriber.consume(Path.from("room")); + const track = broadcast.track("video").subscribe(); + try { + await opened(0); + await answerTrackInfo(streams[0], version); + const stream = await opened(1); + await stream.reading; + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }), + version, + ); + await encodeSubscribeResponse(writer, { start: new SubscribeStart([0n]) }, version); + await encodeSubscribeResponse( + writer, + { drop: new SubscribeDrop({ start: 1n, end: 1n, error: StreamCode.Old, update: 1n }) }, + version, + ); + for (const chunk of chunks) stream.inbound.enqueue(chunk); + expect(await track.closed).toBeInstanceOf(ProtocolViolation); + await drainReactions(); + expect(closes.map((info) => info?.closeCode)).toEqual([SessionCode.ProtocolViolation]); + } finally { + track.close(); + broadcast.close(); + subscriber.close(); + } +}); + test("an unknown subscribe response type closes the session", async () => { const { quic, streams, opened, closes } = fakeSession(); const subscriber = new Subscriber(quic, Version.DRAFT_07, HopSchema.parse(1n)); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index c5efe95780d..df7eaa6aada 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -18,12 +18,27 @@ import { StreamCode, StreamError, sessionCause, + toStreamCode, unauthorized, } from "../error.ts"; import * as netGroup from "../group.ts"; import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, type WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; -import { hiddenBelow, hooks, maxDelayOf, minDelayOf, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; +import { + type DropRun, + hiddenBelow, + hooks, + insertDrop, + insertRun, + maxDelayOf, + minDelayOf, + type Run, + removeRun, + scopeCaptures, + scopeHead, + scopeOverlaps, + staleHole, +} from "../internal.ts"; import type { Lister } from "../listing.ts"; import * as Path from "../path.ts"; import { sendOrder } from "../priority.ts"; @@ -36,7 +51,7 @@ import { closeUnwanted } from "../util/announce.ts"; import { Deadline } from "../util/deadline.ts"; import { isUnrequested, unrequested, unwanted } from "../util/demand.ts"; import { LIMITS, type Limits, SlotMap, Slots } from "../util/limits.ts"; -import { min, UNBOUNDED } from "../util/sequence.ts"; +import { max, min, UNBOUNDED } from "../util/sequence.ts"; import { Budget } from "../util/yield.ts"; import { hasTime, rangesStart, sameRanges, Window, wireRanges, wireSpan } from "../window.ts"; import { overrideBroadcastWire, wireOf } from "../wire.ts"; @@ -82,6 +97,7 @@ import { hasRanges, hasRefresh, hasSetupStream, + hasSkip, hasSynced, hasTrackStream, hasTracks, @@ -175,6 +191,16 @@ interface SubscribeEntry { declared?: { start?: bigint | null; requested?: bigint }; // Whether a SUBSCRIBE_UPDATE went out, which the publisher may have read before answering. updated?: boolean; + // The number of the latest SUBSCRIBE or SUBSCRIBE_UPDATE sent: a stale drop given up under an + // earlier one is no verdict on the current terms. Undefined is the SUBSCRIBE's 0. + update?: bigint; + // The max delay the latest SUBSCRIBE or SUBSCRIBE_UPDATE asked for, in milliseconds: the budget + // a stale drop under it was judged by. + maxDelay?: number; + // The runs `tail` accounts for by a stale drop under `update`: the next update reopens them, + // since the publisher re-offers or drops them again under it. + // Coalesced, so a peer repeating a drop holds one run, not one per message. + stale?: DropRun[]; // Whether SUBSCRIBE_START arrived, which `start` alone cannot say when no range starts. started?: boolean; // Whether our side is FINned, which lite-07 does once the tail settles past SUBSCRIBE_FINISH. @@ -232,6 +258,21 @@ export class Minted { } } +/** + * Every group `ranges` name, which a stale drop under them was judged against. A bound only the + * publisher resolves is taken at its widest: a start from group 0, an end past the live edge + * (`UNBOUNDED`). Mirrors the Rust `judged`. + */ +function judged(ranges: readonly track.Range[]): Run[] { + const runs: Run[] = []; + for (const range of ranges) { + const start = range.start?.absolute ?? 0n; + const end = range.end?.absolute ?? UNBOUNDED; + if (start < end) insertRun(runs, start, end); + } + return runs; +} + /** Subscribe to the peer and own its receive-side protocol tasks. * @internal */ @@ -1534,12 +1575,15 @@ export class Subscriber { resolvesStart: resolvesStart(this.version), fillsFromStart: fillsFromStart(this.version), }); + // The SUBSCRIBE's terms: the groups a stale drop under them was judged against. + hooks.supersedeStale(producer, judged(ranges)); if (!hasRanges(this.version)) hooks.dropAnchors(producer); // Lite-06+ resolves where the feed starts only with SUBSCRIBE_START, so a relay serving // this copy holds its first group until the answer arrives. if (resolvesStart(this.version)) hooks.startTrack(producer, { requested: msg.startGroup }); const entry: SubscribeEntry = { track: producer, + maxDelay: msg.maxDelay, ...framing, // The effective max delay is the stopgap grace: the wrong clock (it bounds // presentation-time drift), but it is how long the subscriber was willing to wait @@ -2135,9 +2179,48 @@ export class Subscriber { } } else if ("drop" in resp) { // The publisher will never send these groups, so they are accounted for without a - // stream, and a reader of one still arriving ends with the drop's error now. - entry.tail.account(resp.drop.start, resp.drop.end + 1n); - hooks.dropTrackGroups(entry.track, resp.drop.start, resp.drop.end + 1n, resp.drop.cause(this.version)); + // stream, and a reader of one still arriving ends with the drop's error now. A lite-07 + // stale drop leaves a group whose stream arrived, since the stream wins over it. + const { start, end, update } = resp.drop; + // It can only echo an update this subscriber sent. + if (update > (entry.update ?? 0n)) + throw new ProtocolViolation(`SUBSCRIBE_DROP names update ${update}, which was never sent`); + const error = resp.drop.cause(this.version); + // On lite-07 an OLD drop is a stale verdict, which the next update reopens; every other + // drop is final, whatever update it names. + const stale = hasSkip(this.version) && toStreamCode(error) === StreamCode.Old; + // One given up under an update since replaced says nothing of the current terms: the + // publisher re-offers it or drops it again under the new number, so it is still owed, + // and a reader that wants it fills it meanwhile. + if (stale && update < (entry.update ?? 0n)) continue; + entry.tail.account(start, end + 1n); + // Judged under the budget this subscriber sent with that update. + const hole = stale ? staleHole(entry.maxDelay ?? 0) : { error }; + // A group whose stream finished under this update was served over a stale drop, so it + // won, even once the cache evicted it. + const named = stale ? entry.tail.unfinished(start, end + 1n) : [{ start, end: end + 1n }]; + const recorded = named.flatMap((run) => hooks.dropTrackGroups(entry.track, run.start, run.end, hole)); + // A final drop is never reopened, even where it replaces a stale one. A sequence + // already skipped stays skipped, and one whose group arrived keeps it, so no update + // reopens either. + entry.stale = stale + ? recorded.reduce((runs, run) => insertDrop(runs, run.start, run.end, hole), entry.stale ?? []) + : removeRun(entry.stale ?? [], start, end + 1n); + } else if ("skip" in resp) { + // Never produced, so nothing is waited for and no reader ends. A skip is final and + // replaces any drop of the same groups, stale or final, so no update reopens them; a + // reader a drop already ended stays ended. + const { start, end } = resp.skip; + entry.tail.account(start, end + 1n); + entry.stale = removeRun(entry.stale ?? [], start, end + 1n); + // A group that arrived was produced after all, so the skip contradicts it: it keeps + // the group, and the session goes on, since the draft requires no more. + const [cached] = hooks.skipTrackGroups(entry.track, start, end + 1n); + if (cached) { + console.warn( + `protocol error: SUBSCRIBE_SKIP [${start}, ${end}] names group ${cached.start}, which arrived; keeping it`, + ); + } } } } @@ -2295,7 +2378,27 @@ export class Subscriber { endGroup: inclusiveGroupEnd(span.end), ranges: edit?.add, remove: edit?.remove, + update: (entry.update ?? 0n) + 1n, }); + // A stale drop given up under the previous update may still be in flight, and this one + // may move the range or budget it was judged against, so the runs held as stale become + // unknown again: a reader fills them rather than trust an old verdict. + entry.update = update.update; + entry.maxDelay = update.maxDelay; + entry.tail.updated(); + const held = hooks.supersedeStale(entry.track, judged(ranges)); + // A lite-07 publisher re-offers every group it reported stale on any update, and drops + // again under the new number what is still stale, so the tail owes every stale run again. + // A group that arrived since its stale drop won over it: the publisher served it, so it + // is neither re-offered nor owed again. + for (const run of entry.stale ?? []) { + for (const reopened of held) { + const start = max(run.start, reopened.start); + const end = min(run.end, reopened.end); + if (start < end) entry.tail.unaccount(start, end); + } + } + entry.stale = undefined; // Encoded apart from the write, so a bound the wire can't carry still fails the track. const encoded = await Writer.collect(this.version, (w) => update.encode(w, this.version)); // A failed write means the publisher stopped reading, as it does once it finishes the @@ -2398,6 +2501,15 @@ export class Subscriber { track.abort(violation); throw violation; } + // A duplicate of a copy still arriving, such as a lite-07 re-offer that overtook the reset + // of the stream it replaces, waits unread, which flow control bounds. Created once that copy + // ends, it is refused as a duplicate if the copy finished and takes the group if it failed. + // Its own reset ends the wait; a clean end leaves it on the copy alone. + for (let arriving = hooks.arrivingGroup(track, group.sequence); arriving !== undefined; ) { + const settled = arriving; + await race([settled, stream.closed.then(() => settled)]); + arriving = hooks.arrivingGroup(track, group.sequence); + } producer = hooks.receiveGroup(track, { sequence: group.sequence, start: group.frameStart }); await readFrames(stream, producer, { @@ -2406,6 +2518,9 @@ export class Subscriber { }); producer.finish(); + // Only a group that arrived whole, not one a drop or the track ended meanwhile, and only + // where a lite-07 stale drop asks for it. + if (hasSkip(this.version) && producer.closed.peek() === null) tail.finish(group.sequence); stream.stop(StreamCode.Cancel); } catch (err: unknown) { // A violation ends the session, which the connection does once it escapes; anything diff --git a/js/net/src/lite/tail.test.ts b/js/net/src/lite/tail.test.ts index 5689b3c404d..1ea5c1cd475 100644 --- a/js/net/src/lite/tail.test.ts +++ b/js/net/src/lite/tail.test.ts @@ -116,6 +116,9 @@ async function settlesWithin(promise: Promise, ms: number): Promise { test("the source start waits for SUBSCRIBE_START where it resolves the start", async () => { const { reader, respond } = await subscribed(version, GRACE, { start: { absolute: 2n } }); + // The accepted copy registers once the SUBSCRIBE write resolves, which can trail the publisher + // reading it, so let that run before asking where the source starts. + await settle(); expect(hooks.sourceStart(reader)).toBe(resolvesStart(version) ? "pending" : undefined); await respond({ start: new SubscribeStart([3n]) }); await settle(); diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index 228e7266344..d5a9e089fe3 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -603,6 +603,27 @@ export function hasDropCode(version: Version): boolean { } } +/** + * Whether holes split into SUBSCRIBE_SKIP, for sequences declared never produced, and + * SUBSCRIBE_DROP, which names the `Update` in effect when it was given up; SUBSCRIBE and + * SUBSCRIBE_UPDATE carry that `Update` number. Added in lite-07; earlier versions name every + * hole in SUBSCRIBE_DROP. + */ +export function hasSkip(version: Version): boolean { + // Explicitly list older versions so future versions keep the lite-07+ behavior. + switch (version) { + case Version.DRAFT_01: + case Version.DRAFT_02: + case Version.DRAFT_03: + case Version.DRAFT_04: + case Version.DRAFT_05: + case Version.DRAFT_06: + return false; + default: + return true; + } +} + /** * Whether SUBSCRIBE_FINISH carries the error the track ended with, so an errored end keeps its * history across a hop. Added in lite-07; earlier versions can only reset. diff --git a/js/net/src/lite/widened_max_age.test.ts b/js/net/src/lite/widened_max_age.test.ts index 9ef760cfb96..8b6f87d2c1a 100644 --- a/js/net/src/lite/widened_max_age.test.ts +++ b/js/net/src/lite/widened_max_age.test.ts @@ -2,7 +2,7 @@ // as stale, including those below where SUBSCRIBE_START resolved, and never one twice. import { expect, test } from "bun:test"; -import { Milli } from "@moq/clock"; +import { Milli, now } from "@moq/clock"; import { accept, connect } from "../connection/index.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; @@ -10,7 +10,7 @@ import * as Path from "../path.ts"; import { Timescale, Timestamp } from "../time.ts"; import type { Subscriber as TrackSubscriber } from "../track.ts"; import { wireOf } from "../wire.ts"; -import { ALPN_07_WIP, Version } from "./version.ts"; +import { ALPN_06, ALPN_07_WIP, Version } from "./version.ts"; const url = new URL("https://localhost:4443/test"); @@ -61,3 +61,151 @@ test("a widened maxDelay re-offers what the publisher skipped below the start", client.abort(); server.abort(); }); + +for (const [version, alpn] of [ + [Version.DRAFT_06, ALPN_06], + [Version.DRAFT_07, ALPN_07_WIP], +] as const) { + const reoffers = version === Version.DRAFT_07; + test(`a widened maxDelay ${reoffers ? "re-offers" : "keeps dropped"} the groups reported stale on ${alpn}`, async () => { + const pair = createMockTransportPair(alpn); + const origin = new OriginProducer(); + const publication = origin.route(Path.from("room")); + const broadcast = publication.broadcast; + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const write = (sequence: number, millis: number) => { + const group = producer.createGroup(BigInt(sequence)); + group.writeFrame({ payload: new Uint8Array([sequence]), timestamp: Timestamp.fromMillis(millis) }); + group.finish(); + }; + write(0, 0); + + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client }), + accept({ transport: pair.server, url, version, publish: origin.consume() }), + ]); + const remote = wireOf(client).consume(Path.from("room")); + const reader = remote.track("video").subscribe({ delay: { max: Milli(1500) } }); + expect(await received(reader, 1)).toEqual([0n]); + // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge, so they are reported + // stale after SUBSCRIBE_START. + for (const [sequence, millis] of [ + [1, 1000], + [2, 2000], + [3, 3000], + [4, 6000], + ]) { + write(sequence, millis); + } + expect(await received(reader, 2)).toEqual([3n, 4n]); + + // Lite-07 re-offers only the groups reported stale; earlier subscribers settled them for good. + reader.update({ delay: { max: Milli(10_000) } }); + if (reoffers) expect(await received(reader, 2)).toEqual([1n, 2n]); + producer.finish(); + producer.close(); + expect(await reader.recvGroup()).toBeUndefined(); + + reader.close(); + remote.close(); + broadcast.close(); + origin.close(); + client.abort(); + server.abort(); + }); +} + +test("a widening keeps a bounded lite-07 subscription open for the stale groups it re-offers", async () => { + const pair = createMockTransportPair(ALPN_07_WIP); + const origin = new OriginProducer(); + const publication = origin.route(Path.from("room")); + const broadcast = publication.broadcast; + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const write = (sequence: number, millis: number) => { + const group = producer.createGroup(BigInt(sequence)); + group.writeFrame({ payload: new Uint8Array([sequence]), timestamp: Timestamp.fromMillis(millis) }); + group.finish(); + }; + write(0, 0); + + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client }), + accept({ transport: pair.server, url, version: Version.DRAFT_07, publish: origin.consume() }), + ]); + const remote = wireOf(client).consume(Path.from("room")); + const ranges = [{ start: { absolute: 0n }, end: { absolute: 5n } }]; + const reader = remote.track("video").subscribe({ delay: { max: Milli(1500) }, ranges }); + expect(await received(reader, 1)).toEqual([0n]); + // Groups 1 and 2 are reported stale after SUBSCRIBE_START. + for (const [sequence, millis] of [ + [1, 1000], + [2, 2000], + [3, 3000], + [4, 6000], + ]) { + write(sequence, millis); + } + expect(await received(reader, 2)).toEqual([3n, 4n]); + + // The widening re-offers them, and the track's end does not settle the tail before they arrive. + reader.update({ delay: { max: Milli(10_000) }, ranges }); + producer.finish(); + expect(await received(reader, 2)).toEqual([1n, 2n]); + + reader.close(); + remote.close(); + broadcast.close(); + origin.close(); + client.abort(); + server.abort(); +}); + +// Every lite-07 update reopens the stale drops, widening or not: the publisher drops again, under +// the new update's number, what is still stale, so the subscriber settles at once rather than +// waiting out its grace for a re-offer that never comes. +test("an update keeping the budget drops the stale groups again on lite-07", async () => { + const pair = createMockTransportPair(ALPN_07_WIP); + const origin = new OriginProducer(); + const publication = origin.route(Path.from("room")); + const broadcast = publication.broadcast; + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const write = (sequence: number, millis: number) => { + const group = producer.createGroup(BigInt(sequence)); + group.writeFrame({ payload: new Uint8Array([sequence]), timestamp: Timestamp.fromMillis(millis) }); + group.finish(); + }; + write(0, 0); + + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client }), + accept({ transport: pair.server, url, version: Version.DRAFT_07, publish: origin.consume() }), + ]); + const remote = wireOf(client).consume(Path.from("room")); + const reader = remote.track("video").subscribe({ delay: { max: Milli(1500) } }); + expect(await received(reader, 1)).toEqual([0n]); + for (const [sequence, millis] of [ + [1, 1000], + [2, 2000], + [3, 3000], + [4, 6000], + ]) { + write(sequence, millis); + } + expect(await received(reader, 2)).toEqual([3n, 4n]); + + // Only the priority changes: groups 1 and 2 are still stale, so they are dropped again. + reader.update({ priority: 5 }); + const before = now(); + producer.finish(); + producer.close(); + expect(await reader.recvGroup()).toBeUndefined(); + // Well inside the 1.5s grace a missing re-drop would wait out, in nanoseconds. + expect(now() - before).toBeLessThan(500_000_000); + + reader.close(); + remote.close(); + broadcast.close(); + origin.close(); + client.abort(); + server.abort(); +}); diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index fa9ee178bfe..39a0dd25174 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -391,7 +391,7 @@ class Model { entry.writer.abort(giveUp()); entry.writer = undefined; } else if (!entry) { - hooks.dropTrackGroups(copy.track, BigInt(op.group), BigInt(op.group + 1), giveUp()); + hooks.dropTrackGroups(copy.track, BigInt(op.group), BigInt(op.group + 1), { error: giveUp() }); } instance.givenUp.add(op.group); break; diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 7f30a51a18f..f3b2b5e1b9b 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -8,7 +8,7 @@ import type { Session as IetfSession } from "./ietf/session.ts"; import { Subscriber as IetfSubscriber } from "./ietf/subscriber.ts"; import { Version as IetfVersion } from "./ietf/version.ts"; import * as Instance from "./instance.ts"; -import { hooks } from "./internal.ts"; +import { hooks, staleHole } from "./internal.ts"; import type { Lister } from "./listing.ts"; import { Producer } from "./origin.ts"; import * as Path from "./path.ts"; @@ -1427,7 +1427,8 @@ for (const resolution of ["arrival", "drop", "other copy", "budget"] as const) { tb.write(5n); tb.write(6n); } - if (resolution === "drop") hooks.dropTrackGroups(ta.producer, 5n, 6n, new StreamError(StreamCode.Old)); + if (resolution === "drop") + hooks.dropTrackGroups(ta.producer, 5n, 6n, { error: new StreamError(StreamCode.Old) }); if (resolution === "budget") await clock.advance(Nano.fromMilli(Milli(1000))); const expected = resolution === "arrival" || resolution === "other copy" ? [5n, 6n, 7n] : [6n, 7n]; expect((await held)?.sequence).toBe(expected[0]); @@ -1530,7 +1531,7 @@ for (const fails of [true, false]) { expect(tb.producer.demand().wanted.peek()).not.toBeUndefined(); const cut = new StreamError(StreamCode(70), { message: "cut" }); - if (terminal === "drop") hooks.dropTrackGroups(tb.producer, 0n, 1n, cut); + if (terminal === "drop") hooks.dropTrackGroups(tb.producer, 0n, 1n, { error: cut }); else tb.producer.abort(cut); expect(await isPending(read)).toBe(false); await expect(read).rejects.toBe(cut); @@ -1564,9 +1565,9 @@ test("a resumed reader reports a drop once and still serves a late group", async ta.write(0n); expect(recvDisposition(sub)).toBe(0n); // A route failing is no verdict: another route may still deliver the group. - hooks.dropTrackGroups(ta.producer, 1n, 2n, new StreamError(StreamCode.SessionClosed)); + hooks.dropTrackGroups(ta.producer, 1n, 2n, { error: new StreamError(StreamCode.SessionClosed) }); expect(recvDisposition(sub)).toBe("idle"); - hooks.dropTrackGroups(ta.producer, 2n, 4n, new StreamError(StreamCode(70), { message: "cut" })); + hooks.dropTrackGroups(ta.producer, 2n, 4n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); expect(recvDisposition(sub)).toEqual([2n, 4n, 70]); expect(recvDisposition(sub)).toBe("idle"); // Re-created under a run already reported dropped. @@ -1581,6 +1582,39 @@ test("a resumed reader reports a drop once and still serves a late group", async a.close(); }); +// A skip of a run the serving copy already dropped replaces the drop, so the resumed reader +// reports it again, as a skip, and only once. Mirrors Rust. +test("a resumed reader reports a skip over a reported drop", () => { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + const recv = () => { + const next = hooks.tryRecvGroup(sub, true); + if (next.kind === "group") { + next.group.close(); + return next.group.sequence; + } + if (next.kind === "drop") return [next.start, next.end, next.skip === true]; + return next.kind; + }; + ta.write(0n); + expect(recv()).toBe(0n); + hooks.dropTrackGroups(ta.producer, 1n, 3n, { error: new StreamError(StreamCode.NotFound) }); + expect(recv()).toEqual([1n, 3n, false]); + expect(recv()).toBe("idle"); + ta.producer.skipGroups(1n, 3n); + expect(recv()).toEqual([1n, 3n, true]); + expect(recv()).toBe("idle"); + ta.write(3n); + expect(recv()).toBe(3n); + + sub.close(); + front.consumer.close(); + a.close(); +}); + test("a separately polled drop does not deliver its run", () => { const front = new Front(); const source = new BroadcastProducer(); @@ -1590,7 +1624,7 @@ test("a separately polled drop does not deliver its run", () => { track.write(0n); expect(drain(sub)).toEqual([0n]); const error = new StreamError(StreamCode.NotFound); - hooks.dropTrackGroups(track.producer, 2n, 4n, error); + hooks.dropTrackGroups(track.producer, 2n, 4n, { error: error }); expect(hooks.tryDrop(sub)).toEqual({ kind: "drop", start: 2n, end: 4n, error }); expect(hooks.tryDrop(sub)).toBeUndefined(); track.write(3n); @@ -1643,7 +1677,7 @@ test("a drop waits while a copy the serving one replaced may still deliver it", front.serve(b.consume()); await settle(); // Only the new route gave groups 1 and 2 up; the old one is not yet bounded. - hooks.dropTrackGroups(tb.producer, 1n, 3n, new StreamError(StreamCode(70), { message: "cut" })); + hooks.dropTrackGroups(tb.producer, 1n, 3n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); expect(recvDisposition(sub)).toBe("idle"); ta.write(1n); expect(recvDisposition(sub)).toBe(1n); @@ -1666,6 +1700,76 @@ test("a drop waits while a copy the serving one replaced may still deliver it", } }); +// A received stale run the reader held back is unknown again once new terms supersede it on the +// serving copy: its upstream re-offers or drops it again, so it is not reported meanwhile. +test("a held-back stale run new terms supersede is not reported", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(1_000) } }); + ta.write(0n); + expect(recvDisposition(sub)).toBe(0n); + front.serve(b.consume()); + await settle(); + // The new route's upstream judged groups 1 and 2 stale; the old route may still deliver them. + hooks.dropTrackGroups(tb.producer, 1n, 3n, staleHole(1_000)); + expect(recvDisposition(sub)).toBe("idle"); + tb.write(3n); + expect(recvDisposition(sub)).toBe(3n); + // The route's subscription updates before the old copy is past the budget. + hooks.supersedeStale(tb.producer, undefined); + await clock.advance(Nano(1_000_000_000)); + expect(recvDisposition(sub)).toBe("idle"); + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + } finally { + installed.close(); + } +}); + +// A group the new route delivers over a stale run the old route reported retires that report, and +// a final drop the new route gives replaces it: only what is still stale is taken back, so an +// update reopens no group already served. Mirrors Rust. +test("a resumed reader retires a stale report its group or final drop replaces", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + hooks.dropTrackGroups(ta.producer, 1n, 4n, staleHole(60_000)); + expect(hooks.tryDrop(sub)).toMatchObject({ kind: "drop", start: 1n, end: 4n, stale: 60_000 }); + + front.serve(b.consume()); + await settle(); + tb.write(1n); + expect(drain(sub)).toEqual([1n]); + // Reported once the replaced route no longer feeds. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + await settle(); + const cut = new StreamError(StreamCode(70), { message: "cut" }); + hooks.dropTrackGroups(tb.producer, 2n, 3n, { error: cut }); + expect(hooks.tryDrop(sub)).toMatchObject({ kind: "drop", start: 2n, end: 3n, stale: undefined }); + // Only group 3 is still stale. + expect(hooks.reofferStale(sub)).toEqual([{ start: 3n, end: 4n }]); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); +}); + test("a drop a replaced route held back is no verdict on the route serving next", async () => { const front = new Front(); const a = new BroadcastProducer(); @@ -1681,7 +1785,7 @@ test("a drop a replaced route held back is no verdict on the route serving next" front.serve(b.consume()); await settle(); // Held back: the copy b replaced may still deliver group 1. - hooks.dropTrackGroups(tb.producer, 1n, 2n, new StreamError(StreamCode(70), { message: "cut" })); + hooks.dropTrackGroups(tb.producer, 1n, 2n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); expect(recvDisposition(sub)).toBe("idle"); front.serve(c.consume()); await settle(); @@ -1709,7 +1813,7 @@ test("a front serving another front reports the inner serving route's drop", asy const sub = outer.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); ta.write(0n); expect(recvDisposition(sub)).toBe(0n); - hooks.dropTrackGroups(ta.producer, 1n, 3n, new StreamError(StreamCode(70), { message: "cut" })); + hooks.dropTrackGroups(ta.producer, 1n, 3n, { error: new StreamError(StreamCode(70), { message: "cut" }) }); expect(recvDisposition(sub)).toEqual([1n, 3n, 70]); expect(recvDisposition(sub)).toBe("idle"); // Re-created under a run already reported dropped. @@ -2486,7 +2590,7 @@ test.each(["lands", "dropped", "expired"])( expect(drain(sub)).toEqual([]); expect(ta.producer.demand().wanted.peek()).toBeDefined(); - if (outcome === "dropped") hooks.dropTrackGroups(ta.producer, 1n, 2n, new NotFound("group")); + if (outcome === "dropped") hooks.dropTrackGroups(ta.producer, 1n, 2n, { error: new NotFound("group") }); if (outcome === "expired") await clock.advance(Nano.fromMilli(SECOND)); if (outcome !== "lands") { // The groups drained above are released once their close settles. @@ -3468,6 +3572,75 @@ test("a route whose IETF session closed before subscribing moves the track to th b.close(); }); +// A front's reader re-offers what its serving route's copy still caches and judges fresh, and gives +// up the rest, so a relay's publisher re-serves an expired stream through it. Mirrors Rust. +test("a resumed reader re-offers a group its serving copy still caches", () => { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + const { groups, givenUp } = hooks.reofferGroups(sub, 0n, 3n); + expect(groups.map((group) => group.sequence)).toEqual([0n]); + expect(givenUp).toEqual([{ start: 1n, end: 3n }]); + for (const group of groups) group.close(); + sub.close(); + front.consumer.close(); + a.close(); +}); + +// A front's reader takes back the stale runs its serving route's copy reported, so a relay's +// publisher re-offers them under a new update rather than keep them reported. Mirrors Rust. +test("a resumed reader takes back its stale reports", () => { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + hooks.dropTrackGroups(ta.producer, 1n, 3n, staleHole(60_000)); + expect(hooks.tryDrop(sub)).toMatchObject({ kind: "drop", start: 1n, end: 3n, stale: 60_000 }); + expect(hooks.reofferStale(sub)).toEqual([{ start: 1n, end: 3n }]); + expect(hooks.reofferStale(sub)).toEqual([]); + sub.close(); + front.consumer.close(); + a.close(); +}); + +// A stale run one route reported is still taken back once another serves: the update reopened it +// downstream, and the new route's copy knows nothing of it, so the groups it delivers there +// afterwards are owed rather than kept reported. Mirrors Rust. +test("a resumed reader takes back a replaced route's stale reports", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + hooks.dropTrackGroups(ta.producer, 1n, 3n, staleHole(60_000)); + expect(hooks.tryDrop(sub)).toMatchObject({ kind: "drop", start: 1n, end: 3n, stale: 60_000 }); + + front.serve(b.consume()); + await settle(); + expect(hooks.tryDrop(sub)).toBeUndefined(); + // Reopened, though the new route never reported it. + expect(hooks.reofferStale(sub)).toEqual([{ start: 1n, end: 3n }]); + expect(hooks.reofferStale(sub)).toEqual([]); + tb.write(1n); + tb.write(2n); + expect(drain(sub)).toEqual([1n, 2n]); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); +}); + test("routed metadata receives a refusal without polling a group cursor", async () => { const clock = new Manual(); const installed = clock.install(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index b9aa9f6824a..54dc9d9289d 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -402,6 +402,11 @@ class Reader implements ResumeSource { #delivered = new Delivered(); /** Runs the serving copy gave up and the reader has yet to report; see {@link #tryDrop}. */ #drops: DropRun[] = []; + /** + * The stale runs reported since the last {@link reofferStale}, whichever copy gave them up: an + * update reopens each, and the copy that reported it may no longer serve. + */ + #reportedStale: Run[] = []; /** A copy that declared the track's end ran out cleanly at it, even if since let go. */ #settled = false; /** Whether groups were ever asked for; until then the reader keeps no replaced copy. */ @@ -713,6 +718,36 @@ class Reader implements ResumeSource { return holding; } + /** + * Fresh reads of the groups in `[start, end)` the serving copy still caches and judges fresh, + * handed out as any group of that copy is, and the runs still given up: all of them while no + * copy serves. Mirrors the Rust `Reader::reoffer_groups`. + */ + reofferGroups(start: bigint, end: bigint): { groups: GroupConsumer[]; givenUp: Run[] } { + const serving = this.#serving(); + if (!serving) return { groups: [], givenUp: [{ start, end }] }; + const { groups, givenUp } = hooks.reofferGroups(serving.sub, start, end); + return { groups: groups.map((group) => this.#handOut(serving, group)), givenUp }; + } + + /** + * Take back every stale run: those queued here, those this reader reported, and those the + * serving copy reported, which its cursor owes again. What this reader reported counts + * whichever copy gave it up, since a route change since leaves the serving copy knowing + * nothing of it, though the update reopened it. Only the serving copy is judged again: a + * replaced copy may no longer deliver, so what it gave up is the new route's to deliver or + * give up. Mirrors the Rust `Reader::reoffer_stale`. + */ + reofferStale(): Run[] { + const taken = this.#reportedStale; + this.#reportedStale = []; + for (const run of this.#drops) if (run.stale !== undefined) insertRun(taken, run.start, run.end); + this.#drops = this.#drops.filter((run) => run.stale === undefined); + const serving = this.#serving(); + if (serving) for (const run of hooks.reofferStale(serving.sub)) insertRun(taken, run.start, run.end); + return taken; + } + cachedGroup(sequence: bigint, from = 0): GroupConsumer | undefined { for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; @@ -853,11 +888,22 @@ class Reader implements ResumeSource { if (this.#closed) return undefined; const serving = this.#serving(); if (serving) { + // A received stale run queued here that new terms superseded since is unknown again: the + // copy no longer holds it, and its upstream re-offers or drops it again. + for (const run of this.#drops) { + if (!run.received) continue; + const held = hooks.staleHeld(serving.sub, run.start, run.end); + for (const gap of runGaps(held, run.start, run.end)) + this.#drops = removeRun(this.#drops, gap.start, gap.end); + } for (let dropped = hooks.tryDrop(serving.sub); dropped; dropped = hooks.tryDrop(serving.sub)) { // A route failing is no verdict: another route may still deliver the group, and // one it failed while buffered is handed out to continue through its recovery. if (routeFailed(dropped.error)) continue; - this.#drops = insertDrop(this.#drops, dropped.start, dropped.end, dropped.error); + // A stale run the copy holds came from its upstream, which new terms may supersede. + const received = + dropped.stale !== undefined && hooks.staleHeld(serving.sub, dropped.start, dropped.end).length > 0; + this.#drops = insertDrop(this.#drops, dropped.start, dropped.end, { ...dropped, received }); } } this.#drops = this.#drops.filter((run) => !delivered.covers(run.start, run.end)); @@ -875,7 +921,10 @@ class Reader implements ResumeSource { if (open) { // Report once without marking it delivered: a late fill may still land. this.#drops = removeRun(this.#drops, open.start, open.end); - return { kind: "drop", ...open, error: run.error }; + if (run.stale !== undefined) insertRun(this.#reportedStale, open.start, open.end); + // A skip or final drop is final, so no update reopens a stale report it replaced. + else this.#reportedStale = removeRun(this.#reportedStale, open.start, open.end); + return { kind: "drop", ...open, error: run.error, skip: run.skip, stale: run.stale }; } } } @@ -1178,6 +1227,11 @@ class Reader implements ResumeSource { #handOut(copy: Copy, group: GroupConsumer): GroupConsumer { this.#newest = max(this.#newest ?? 0n, group.sequence); + // The group won over any stale report on it, whichever copy made it, so no update reopens + // it. + if (this.#reportedStale.length > 0) { + this.#reportedStale = removeRun(this.#reportedStale, group.sequence, group.sequence + 1n); + } // Another copy holding the group, now or once it lands there, is where a failure continues // it, so it stays while the group is read; see #backs. Recorded even with one copy, since a // later route's copy may come to hold it. diff --git a/js/net/src/socket.test.ts b/js/net/src/socket.test.ts index 968b839aff5..cc4f296d578 100644 --- a/js/net/src/socket.test.ts +++ b/js/net/src/socket.test.ts @@ -164,7 +164,7 @@ test("a track session skips dropped groups and keeps the next message", async () const { local, remote } = await track(); const reader = remote.consumer.ordered(); const reading = reader.readFrame(); - hooks.dropTrackGroups(local.producer, 0n, 1n, new NotFound("message unavailable")); + hooks.dropTrackGroups(local.producer, 0n, 1n, { error: new NotFound("message unavailable") }); const group = local.producer.createGroup(1n); group.writeFrame(bytes(7)); group.finish(); @@ -482,7 +482,7 @@ for (const dropped of [false, true]) { const producer = answer.accept(); const stream = await acceptance; const reading = stream.consumer.readFrame(); - if (dropped) hooks.dropTrackGroups(producer, 0n, 1n, new NotFound("TCP group unavailable")); + if (dropped) hooks.dropTrackGroups(producer, 0n, 1n, { error: new NotFound("TCP group unavailable") }); producer.finish(); await expect(reading).rejects.toBeInstanceOf(NotFound); expect(await stream.producer.closed).toBeInstanceOf(NotFound); diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 5c6bb52be7d..1b87fb6e44d 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -1,5 +1,6 @@ import { type Milli, Nano, now, sleep } from "@moq/clock"; import { type GetPromise, Signal } from "@moq/signals"; +import { insertRun, type Run, runGaps } from "./internal.ts"; import { GRACE, Ledger } from "./ledger.ts"; /** How a {@link Tail} measures its grace. */ @@ -29,6 +30,10 @@ export class Tail { #active = 0; #changed = new Signal(0); #grace: () => Milli; + // Groups whose stream finished since the last update was sent. A lite-07 stale drop under that + // update names one only if it was sent before the publisher served the group, so the group won + // over it, even once the cache evicted it. + #finished: Run[] = []; constructor(options: TailOptions = {}) { this.#grace = options.grace ?? (() => GRACE); @@ -89,6 +94,24 @@ export class Tail { }; } + /** Record that the group stream for `sequence` arrived whole. */ + finish(sequence: bigint): void { + insertRun(this.#finished, sequence, sequence + 1n); + } + + /** The parts of `[start, end)` whose stream did not finish since the last update was sent. */ + unfinished(start: bigint, end: bigint): Run[] { + return runGaps(this.#finished, start, end); + } + + /** + * Forget which streams finished, for an update being sent: a stale drop under an earlier one is + * no verdict, so none needs them. + */ + updated(): void { + this.#finished = []; + } + /** Re-check what {@link settle} waits on, after something outside the ledger changed. */ touch(): void { this.#bump(); @@ -99,6 +122,11 @@ export class Tail { this.#account(start, end); } + /** Owe `[start, end)` again: a stale drop that accounted for it was reversed by a widening. */ + unaccount(start: bigint, end: bigint): void { + this.#ledger.unaccount(start, end, now()); + } + /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ demand(start: bigint, end: bigint): void { this.#ledger.demand(start, end, now()); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 9f118d4974f..7944ed61e46 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2,10 +2,10 @@ import { expect, spyOn, test } from "bun:test"; import { Manual, Milli, Nano, sleep, Wall, yieldNow } from "@moq/clock"; import { hostTime } from "@moq/clock/test"; import { race } from "@moq/signals"; -import { ProtocolViolation, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; +import { ProtocolViolation, Skipped, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; import type { Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { Version as IetfVersion } from "./ietf/version.ts"; -import { hooks } from "./internal.ts"; +import { hooks, skipHole, staleHole } from "./internal.ts"; import { textFrame } from "./mock.ts"; import { Timescale, Timestamp } from "./time.ts"; import { authoredInfo, infoDefaults, rescale, Subscription, Producer as TrackProducer } from "./track.ts"; @@ -686,7 +686,7 @@ test("a bounded read ends once its window is read, unless a cap or a wire publis // Groups 3 and 4 may still arrive. expect(hooks.tryRecvGroup(reader, false)).toEqual({ kind: "idle" }); - hooks.dropTrackGroups(producer, 3n, 4n, DROPPED); + hooks.dropTrackGroups(producer, 3n, 4n, { error: DROPPED }); producer.createGroup(4n).finish(); expect(read(reader)).toEqual([4n]); expect(hooks.tryRecvGroup(reader, false)).toEqual({ kind: "done" }); @@ -865,7 +865,7 @@ test("a drop a time end settles over is reported", () => { }); expect(hooks.tryRecvGroup(reader, true)).toMatchObject({ kind: "group", group: { sequence: 0n } }); // Past the cap while group 0 has yet to start. - hooks.dropTrackGroups(producer, 1n, 2n, DROPPED); + hooks.dropTrackGroups(producer, 1n, 2n, { error: DROPPED }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "idle" }); zero.writeFrame(textFrame("0", Timestamp.fromMillis(0))); producer.createGroup(2n).writeFrame(textFrame("2", Timestamp.fromMillis(2000))); @@ -3313,7 +3313,7 @@ const DROPPED = new StreamError(StreamCode.NotFound, { message: "dropped" }); test("explicit and producer-cut drops survive outside the group cache", async () => { const producer = new TrackProducer("video").accept({}); const reader = producer.subscribe({ delay: { max: Milli(60_000) } }); - hooks.dropTrackGroups(producer, 3n, 4n, DROPPED); + hooks.dropTrackGroups(producer, 3n, 4n, { error: DROPPED }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "drop", start: 3n, end: 4n, error: DROPPED }); const group = producer.createGroup(4n); @@ -3362,7 +3362,7 @@ test("an upstream drop ends a group still arriving with its error", async () => expect(group?.sequence).toBe(3n); expect(await group?.readString()).toBe("3"); const cut = new StreamError(StreamCode(70), { message: "cut" }); - hooks.dropTrackGroups(producer, 3n, 5n, cut); + hooks.dropTrackGroups(producer, 3n, 5n, { error: cut }); // Settled by the drop itself, not a later reset; the finished group stays whole. await expect(group?.readString()).rejects.toBe(cut); const whole = await reader.recvGroup(); @@ -3377,7 +3377,7 @@ test("an upstream drop ends a group still arriving with its error", async () => test("a drop below the window is reported once the window lowers to it", () => { const producer = new TrackProducer("video").accept({}); const reader = producer.subscribe({ delay: { max: Milli(60_000) } }).withRange({ start: { included: 5n } }); - hooks.dropTrackGroups(producer, 2n, 3n, DROPPED); + hooks.dropTrackGroups(producer, 2n, 3n, { error: DROPPED }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "idle" }); reader.withRange({ start: { included: 0n } }); @@ -3408,7 +3408,7 @@ test("a re-created cut group is served, not dropped", async () => { test("a reported drop still allows a late group, handed out once", () => { const producer = new TrackProducer("video").accept({}); const reader = producer.subscribe({ delay: { max: Milli(60_000) } }); - hooks.dropTrackGroups(producer, 2n, 4n, DROPPED); + hooks.dropTrackGroups(producer, 2n, 4n, { error: DROPPED }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "drop", start: 2n, end: 4n, error: DROPPED }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "idle" }); producer.createGroup(3n).finish(); @@ -3421,15 +3421,155 @@ test("a reported drop still allows a late group, handed out once", () => { producer.close(); }); +test("a received stale drop settles only the readers it covers", () => { + const producer = new TrackProducer("video").accept({ maxAge: null }); + const reader = (max: number, start: bigint, end?: bigint) => + producer.subscribe({ + delay: { max: Milli(max) }, + ranges: [{ start: { absolute: start }, end: end === undefined ? undefined : { absolute: end } }], + }); + const covering = reader(500, 0n); + const narrower = reader(100, 0n); + const wider = reader(5000, 0n); + const capped = reader(500, 0n, 4n); + const above = reader(500, 3n); + const stale = staleHole(500); + hooks.dropTrackGroups(producer, 1n, 3n, stale); + const skip = skipHole(); + hooks.dropTrackGroups(producer, 3n, 4n, skip); + const dropped = (subscriber: typeof covering) => { + const runs: [bigint, bigint, boolean][] = []; + for (let run = hooks.tryDrop(subscriber); run; run = hooks.tryDrop(subscriber)) + runs.push([run.start, run.end, run.skip === true]); + return runs; + }; + const settled: [bigint, bigint, boolean][] = [ + [1n, 3n, false], + [3n, 4n, true], + ]; + expect(dropped(covering)).toEqual(settled); + // A narrower budget is settled too. + expect(dropped(narrower)).toEqual(settled); + // A wider budget waits, a capped range may judge otherwise, and a range above the run lacks it. + for (const subscriber of [wider, capped, above]) expect(dropped(subscriber)).toEqual([[3n, 4n, true]]); + for (const subscriber of [covering, narrower, wider, capped, above]) subscriber.close(); + producer.finish(); + producer.close(); +}); + +test("a superseded stale run is unknown again, and its group is handed out", () => { + const producer = new TrackProducer("video").accept({}); + const reader = producer.subscribe({ delay: { max: Milli(500) }, ranges: [{ start: { absolute: 0n } }] }); + hooks.dropTrackGroups(producer, 1n, 2n, staleHole(500)); + hooks.supersedeStale(producer, undefined); + expect(hooks.tryDrop(reader)).toBeUndefined(); + producer.createGroup(1n).finish(); + const recv = hooks.tryRecvGroup(reader, true); + expect(recv.kind === "group" ? recv.group.sequence : recv).toBe(1n); + if (recv.kind === "group") recv.group.close(); + reader.close(); + producer.finish(); + producer.close(); +}); + +test("superseding withdraws a partly reported stale run", () => { + const producer = new TrackProducer("video").accept({}); + const reader = producer.subscribe({ delay: { max: Milli(500) }, ranges: [{ start: { absolute: 0n } }] }); + hooks.dropTrackGroups(producer, 4n, 5n, { error: DROPPED }); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 4n, end: 5n }); + hooks.dropTrackGroups(producer, 3n, 7n, staleHole(500)); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 3n, end: 4n, stale: 500 }); + hooks.supersedeStale(producer, undefined); + // 5 and 6 are unknown again. + expect(hooks.tryDrop(reader)).toBeUndefined(); + reader.close(); + producer.finish(); + producer.close(); +}); + +// A received stale run a reader reported and the relay's own update then superseded is unknown +// again: the reader's update owes it until the upstream serves or drops it again, rather than +// dropping it again as if the reader had judged it stale itself. Mirrors Rust. +test("a superseded received stale report stays owed after an update", () => { + const producer = new TrackProducer("video").accept({}); + const reader = producer.subscribe({ delay: { max: Milli(500) }, ranges: [{ start: { absolute: 0n } }] }); + hooks.dropTrackGroups(producer, 1n, 3n, staleHole(500)); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 1n, end: 3n, stale: 500 }); + hooks.supersedeStale(producer, undefined); + expect(hooks.reofferStale(reader)).toEqual([{ start: 1n, end: 3n }]); + expect(hooks.tryDrop(reader)).toBeUndefined(); + // The upstream drops it again under the new terms: reported again. + hooks.dropTrackGroups(producer, 1n, 3n, staleHole(500)); + expect(hooks.tryDrop(reader)).toMatchObject({ start: 1n, end: 3n, stale: 500 }); + reader.close(); + producer.finish(); + producer.close(); +}); + +test("a received stale run settles a bounded reader only through the judged ranges", () => { + const producer = new TrackProducer("video").accept({ maxAge: null }); + hooks.receiveTrack(producer); + const bounded = (max: number) => + producer.subscribe({ + delay: { max: Milli(max) }, + ranges: [{ start: { absolute: 0n }, end: { absolute: 2n } }], + }); + const wider = bounded(5000); + const matching = bounded(500); + // The upstream judged its own range of 0..2 under 500ms. + hooks.supersedeStale(producer, [{ start: 0n, end: 2n }]); + hooks.dropTrackGroups(producer, 0n, 2n, staleHole(500)); + // A wider budget still wants the groups, however the feed accounted for them. + expect(hooks.tryRecvGroup(wider, false)).toEqual({ kind: "idle" }); + expect(hooks.tryDrop(matching)).toMatchObject({ start: 0n, end: 2n, stale: 500 }); + expect(hooks.tryRecvGroup(matching, false)).toEqual({ kind: "done" }); + // Judged against an open range, a capped reader may see an older edge. + hooks.supersedeStale(producer, undefined); + hooks.dropTrackGroups(producer, 0n, 2n, staleHole(500)); + const capped = bounded(500); + expect(hooks.tryDrop(capped)).toBeUndefined(); + producer.createGroup(0n).finish(); + const recv = hooks.tryRecvGroup(wider, false); + expect(recv.kind === "group" ? recv.group.sequence : recv).toBe(0n); + if (recv.kind === "group") recv.group.close(); + for (const reader of [wider, matching, capped]) reader.close(); + producer.finish(); + producer.close(); +}); + +test("a superseded stale run leaves a bounded reader waiting, and a later reader judges it too", () => { + const producer = new TrackProducer("video").accept({ maxAge: null }); + hooks.receiveTrack(producer); + const bounded = (max: number) => + producer.subscribe({ + delay: { max: Milli(max) }, + ranges: [{ start: { absolute: 0n }, end: { absolute: 2n } }], + }); + const wider = bounded(5000); + hooks.supersedeStale(producer, [{ start: 0n, end: 2n }]); + producer.createGroup(2n).finish(); + hooks.dropTrackGroups(producer, 0n, 2n, staleHole(500)); + // A reader added after the verdict judges it against the same judged ranges. + const later = bounded(500); + expect(hooks.tryDrop(later)).toMatchObject({ start: 0n, end: 2n, stale: 500 }); + expect(hooks.tryRecvGroup(wider, false)).toEqual({ kind: "idle" }); + // New terms make the groups unknown again: still wanted, never proven delivered. + hooks.supersedeStale(producer, [{ start: 0n, end: 2n }]); + expect(hooks.tryRecvGroup(wider, false)).toEqual({ kind: "idle" }); + for (const reader of [wider, later]) reader.close(); + producer.finish(); + producer.close(); +}); + test("a wide drop is held and reported as one run", () => { const producer = new TrackProducer("video").accept({}); const reader = producer.subscribe({ delay: { max: Milli(60_000) } }); - hooks.dropTrackGroups(producer, 2n, 2n ** 40n, DROPPED); + hooks.dropTrackGroups(producer, 2n, 2n ** 40n, { error: DROPPED }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "drop", start: 2n, end: 2n ** 40n, error: DROPPED }); // A wider drop reports only what this cursor has not reported yet, with its own error. const wider = new StreamError(StreamCode.Old, { message: "old" }); - hooks.dropTrackGroups(producer, 1n, 2n ** 40n + 2n, wider); + hooks.dropTrackGroups(producer, 1n, 2n ** 40n + 2n, { error: wider }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "drop", start: 1n, end: 2n, error: wider }); expect(hooks.tryRecvGroup(reader, true)).toEqual({ kind: "drop", @@ -3449,12 +3589,9 @@ test("adjacent drops with the same code merge into one run", () => { const reader = producer.subscribe({ delay: { max: Milli(60_000) } }); // Each DROP decodes to its own error, so runs must merge by code, not identity. for (let sequence = 2n; sequence < 5n; sequence++) { - hooks.dropTrackGroups( - producer, - sequence, - sequence + 1n, - new StreamError(StreamCode.NotFound, { message: "dropped" }), - ); + hooks.dropTrackGroups(producer, sequence, sequence + 1n, { + error: new StreamError(StreamCode.NotFound, { message: "dropped" }), + }); } const recv = hooks.tryRecvGroup(reader, true); expect(recv.kind === "drop" ? [recv.start, recv.end, recv.error] : recv).toEqual([2n, 5n, DROPPED]); @@ -3800,6 +3937,8 @@ test("replay and backlogs deliver sparse out-of-order arrivals ascending without // pending, so lower groups may still arrive behind a higher one. function pendingTail(end: bigint, maxDelay = Milli(60_000)) { const producer = new TrackProducer("test"); + // A received copy, whose groups arrive in any order. + hooks.receiveTrack(producer); const reader = producer.subscribe({ delay: { max: maxDelay } }); hooks.startTrack(producer, { start: 0n }); hooks.endTrack(producer, { group: end, frame: 0 }); @@ -4041,7 +4180,7 @@ for (const abort of [false, true]) { const next = reader.recvGroup(); expect(await settled(next)).toBe(false); - hooks.dropTrackGroups(producer, 0n, 2n, DROPPED); + hooks.dropTrackGroups(producer, 0n, 2n, { error: DROPPED }); if (abort) producer.abort(new Error("dropped")); expect(await next).toBeUndefined(); if (abort) expect(await reader.closed).toBeNull(); @@ -4288,3 +4427,240 @@ test("an errored end keeps its history", async () => { expect(polledOrdered.tryNextGroup()?.sequence).toBe(1n); expect(() => polledOrdered.tryNextGroup()).toThrow(failed); }); + +// Creating a group says nothing about the gap it leaves: groups may be created out of order. An +// append says the producer will not go back, so it skips every hole below it that was never +// produced, replacing a drop already given and reported. A received copy's groups arrive in any +// order, so its gap is not skipped. Mirrors Rust. +test("an append skips the holes created groups left below it", () => { + const producer = new TrackProducer("test"); + producer.accept({ maxAge: null }); + const reader = producer.subscribe({ delay: { max: null } }); + const drain = () => { + const got: unknown[] = []; + for ( + let next = hooks.tryRecvGroup(reader, true); + next.kind !== "idle"; + next = hooks.tryRecvGroup(reader, true) + ) { + if (next.kind === "group") got.push(next.group.sequence); + else if (next.kind === "drop") got.push([next.start, next.end, next.skip === true]); + else throw new Error(`unexpected ${next.kind}`); + } + return got; + }; + producer.createGroup(0n).finish(); + hooks.dropTrackGroups(producer, 2n, 3n, { error: DROPPED }); + producer.createGroup(4n).finish(); + expect(drain()).toEqual([[2n, 3n, false], 0n, 4n]); + + producer.appendGroup().finish(); + expect(drain()).toEqual([[1n, 4n, true], 5n]); + + // The next append only names what was left since this one. + producer.createGroup(7n).finish(); + producer.appendGroup().finish(); + expect(drain()).toEqual([[6n, 7n, true], 7n, 8n]); + + const copy = new TrackProducer("test"); + hooks.receiveTrack(copy); + copy.accept({ maxAge: null }); + const late = copy.subscribe({ delay: { max: null } }); + copy.createGroup(0n).finish(); + copy.createGroup(2n).finish(); + copy.appendGroup().finish(); + expect(hooks.tryRecvGroup(late, true)).toMatchObject({ kind: "group" }); + expect(hooks.tryRecvGroup(late, true)).toMatchObject({ kind: "group" }); + expect(hooks.tryRecvGroup(late, true)).toMatchObject({ kind: "group" }); + expect(hooks.tryRecvGroup(late, true)).toEqual({ kind: "idle" }); +}); + +// A late group created out of order reaches a reader, since creating a group skips nothing. +test("a late local group reaches a reader, since creating out of order skips nothing", () => { + const producer = new TrackProducer("test"); + producer.accept({ maxAge: null }); + producer.createGroup(0n).finish(); + producer.createGroup(2n).finish(); + const reader = producer.subscribe({ delay: { max: null }, order: "ascending" }); + producer.createGroup(1n).finish(); + const got: bigint[] = []; + for (let group = reader.tryRecvGroup(); group; group = reader.tryRecvGroup()) got.push(group.sequence); + expect(got).toEqual([0n, 1n, 2n]); +}); + +// Declared skips up to the end settle it, as written groups would: an open-range reader ends after +// the groups below, and the track's outcome resolves. Mirrors the Rust `TrackState::reaches`. +test("declared skips up to the end settle the track", async () => { + const producer = new TrackProducer("test"); + producer.accept({}); + const reader = producer.subscribe({ delay: { max: null }, order: "ascending" }); + producer.createGroup(0n).finish(); + // Group 0's finish settles first, so only the skip can reach the end. + await yieldNow(); + producer.finishAt({ group: 3n, frame: 0 }); + expect(producer.closed.peek()).toBeUndefined(); + producer.skipGroups(1n, 3n); + expect(producer.closed.peek()).toBeNull(); + expect((await reader.recvGroup())?.sequence).toBe(0n); + expect(await reader.recvGroup()).toBeUndefined(); + reader.close(); +}); + +// A skip is final: creating at a sequence an append or skipGroups skipped throws, and a later drop +// leaves it skipped, while a dropped sequence may be created again. skipGroups leaves cached +// sequences alone and replaces every drop, final or stale. Mirrors Rust. +test("creating a skipped sequence throws Skipped", () => { + const producer = new TrackProducer("test"); + producer.accept({}); + producer.createGroup(0n).finish(); + producer.createGroup(3n).finish(); + producer.appendGroup().finish(); + expect(() => producer.createGroup(1n)).toThrow(Skipped); + + producer.skipGroups(5n, 7n); + expect(() => producer.createGroup(6n)).toThrow(Skipped); + // No skip below group 0. + expect(() => producer.skipGroups(-1n, 1n)).toThrow(RangeError); + // The next sequence, 5, was skipped too. + expect(() => producer.appendGroup()).toThrow(Skipped); + // A later drop leaves a skipped sequence skipped. + expect(hooks.dropTrackGroups(producer, 1n, 2n, { error: DROPPED })).toEqual([]); + expect(() => producer.createGroup(1n)).toThrow(Skipped); + // A dropped sequence may be created again. + hooks.dropTrackGroups(producer, 9n, 10n, { error: DROPPED }); + producer.createGroup(9n).finish(); + + const other = new TrackProducer("test"); + other.accept({}); + const reader = other.subscribe({ delay: { max: null } }); + other.createGroup(1n).finish(); + hooks.dropTrackGroups(other, 3n, 4n, { error: DROPPED }); + hooks.dropTrackGroups(other, 4n, 5n, staleHole(1000)); + other.skipGroups(0n, 5n); + hooks.supersedeStale(other, undefined); + const drops: unknown[] = []; + for (let next = hooks.tryRecvGroup(reader, true); next.kind !== "idle"; next = hooks.tryRecvGroup(reader, true)) { + if (next.kind === "drop") drops.push([next.start, next.end, next.skip === true]); + } + expect(drops).toEqual([ + [0n, 1n, true], + [2n, 5n, true], + ]); +}); + +// A skip after a drop already reported goes out again, replacing it, while a reader that moved past +// the drop is handed nothing again; a skip already reported is not reported twice, and a later drop +// leaves it skipped. Mirrors Rust. +test("a skip is reported over a reported drop", () => { + const producer = new TrackProducer("test"); + producer.accept({ maxAge: null }); + const reader = producer.subscribe({ delay: { max: null } }); + const drain = () => { + const got: unknown[] = []; + for ( + let next = hooks.tryRecvGroup(reader, true); + next.kind !== "idle"; + next = hooks.tryRecvGroup(reader, true) + ) { + if (next.kind === "group") got.push(next.group.sequence); + else if (next.kind === "drop") got.push([next.start, next.end, next.skip === true]); + else throw new Error(`unexpected ${next.kind}`); + } + return got; + }; + producer.createGroup(0n).finish(); + hooks.dropTrackGroups(producer, 1n, 3n, { error: DROPPED }); + hooks.dropTrackGroups(producer, 4n, 5n, { error: DROPPED }); + producer.createGroup(5n).finish(); + expect(drain()).toEqual([[1n, 3n, false], [4n, 5n, false], 0n, 5n]); + + // Group 0 arrived, so the skip leaves it; both drops and the gap between become one skip. + expect(hooks.skipTrackGroups(producer, 0n, 5n)).toEqual([{ start: 0n, end: 1n }]); + expect(drain()).toEqual([[1n, 5n, true]]); + + expect(hooks.dropTrackGroups(producer, 2n, 3n, { error: DROPPED })).toEqual([]); + expect(drain()).toEqual([]); + expect(() => producer.createGroup(2n)).toThrow(Skipped); + reader.close(); + producer.close(); +}); + +// A received stale run is judged from where it meets the reader's span, as Rust clips it: a reader +// asking only its suffix, through every group the upstream's range named, is settled by that suffix. +test("a received stale run settles the suffix a narrower range asks for", () => { + const producer = new TrackProducer("video").accept({}); + hooks.receiveTrack(producer); + hooks.supersedeStale(producer, [{ start: 0n, end: 10n }]); + const reader = producer.subscribe({ + delay: { max: Milli(500) }, + ranges: [{ start: { absolute: 5n }, end: { absolute: 10n } }], + }); + hooks.dropTrackGroups(producer, 0n, 10n, staleHole(500)); + expect(hooks.tryDrop(reader)).toMatchObject({ kind: "drop", start: 5n, end: 10n, stale: 500 }); + expect(hooks.tryDrop(reader)).toBeUndefined(); + reader.close(); + producer.close(); +}); + +// Skips are kept and reported as runs: a huge skipGroups or append gap costs one run and one +// report, whatever its length. Mirrors Rust. +test("a huge skip costs one run", () => { + const FAR = 1n << 61n; + for (const append of [false, true]) { + const producer = new TrackProducer("test"); + producer.accept({ maxAge: null }); + const reader = producer.subscribe({ delay: { max: null } }); + producer.createGroup(0n).finish(); + if (append) { + producer.createGroup(FAR).finish(); + producer.appendGroup().finish(); + } else { + producer.skipGroups(1n, FAR); + producer.createGroup(FAR).finish(); + } + const got: unknown[] = []; + for ( + let next = hooks.tryRecvGroup(reader, true); + next.kind !== "idle"; + next = hooks.tryRecvGroup(reader, true) + ) { + if (next.kind === "group") got.push(next.group.sequence); + else if (next.kind === "drop") got.push([next.start, next.end, next.skip === true]); + } + const groups = append ? [0n, FAR, FAR + 1n] : [0n, FAR]; + expect(got.filter((entry) => typeof entry === "bigint")).toEqual(groups); + expect(got.filter((entry) => typeof entry !== "bigint")).toEqual([[1n, FAR, true]]); + } +}); + +// A group stale at the track's max age bound is final under any budget, yet an update that reopens +// its stale drop must report it again rather than leave it owed: no skip offers it again. Mirrors +// Rust. +test("reopening stale drops reports again what is stale at the max age bound", () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: Milli(1500) }); + const reader = producer.subscribe({ delay: { max: Milli(1500) } }); + const drain = () => { + const got: unknown[] = []; + for ( + let next = hooks.tryRecvGroup(reader, true); + next.kind !== "idle"; + next = hooks.tryRecvGroup(reader, true) + ) { + if (next.kind === "group") got.push(next.group.sequence); + else if (next.kind === "drop") got.push([next.start, next.end, next.stale]); + } + return got; + }; + putGroup(producer, 0n, 0); + expect(drain()).toEqual([0n]); + for (const [sequence, millis] of [ + [1n, 1000], + [2n, 2000], + [3n, 3000], + [4n, 6000], + ] as const) + putGroup(producer, sequence, millis); + expect(drain()).toEqual([[1n, 2n, 1500], [2n, 3n, 1500], 3n, 4n]); + expect(hooks.reofferStale(reader)).toEqual([{ start: 1n, end: 3n }]); + expect(drain()).toEqual([[1n, 3n, 1500]]); +}); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index deeeaed5b70..1d6f689f9d4 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -10,6 +10,7 @@ import { Delivered } from "./delivered.ts"; import { GroupTooLarge, ProtocolViolation, + Skipped, StreamCode, StreamError, TimestampMismatch, @@ -25,6 +26,7 @@ import { type FillStarts, finalSequence, groupBounds, + type Hole, hooks, insertDrop, insertRun, @@ -37,6 +39,8 @@ import { removeRun, runGaps, type StagedTrack, + skipHole, + staleHole, type TrackRequestOptions, } from "./internal.ts"; import { CACHE_EXPIRY, grace, Ledger } from "./ledger.ts"; @@ -695,12 +699,15 @@ class TrackState { */ reads?: (sequence: bigint) => boolean; groups = new Signal([]); - // Sequences the upstream publisher gave up, with the error each was given up with: an - // incoming SUBSCRIBE_DROP or a producer cut. Kept apart from the cache so reclaim cannot + // Sequences the upstream publisher gave up, with why: an incoming SUBSCRIBE_SKIP or + // SUBSCRIBE_DROP, or a producer cut. Kept apart from the cache so reclaim cannot // erase a disposition a downstream publisher still has to forward, and as runs so a wide // DROP costs one entry. Replaced rather than mutated, so a cursor can tell it changed by // identity. drops = new Signal([]); + // Every group the ranges of a received copy's current upstream terms name, which its stale runs + // were judged against; undefined when unknown, which counts as every group. + judged?: readonly Run[]; // Every group still in the producer's replay cache, including groups this // subscriber already consumed, sorted by sequence. Drift anchors have the same // lifetime as content. Sorted so the latency guard, evaluated per group and per @@ -805,6 +812,9 @@ type Start = { sequence: bigint; start: Timestamp }; let cachedMirror: (producer: Producer, sequence: bigint, from: number) => GroupConsumer | undefined; // Whether the producer still caches a group that has not failed; assigned in its static block. let cachedHolds: (producer: Producer, sequence: bigint) => boolean; +// The sequences in `[start, end)` the producer caches, unsorted, costing the smaller of the span +// and the cache; assigned in its static block. +let cachedIn: (producer: Producer, start: bigint, end: bigint) => bigint[]; // Mirror the cached groups in `[start, end)` that `offer` accepts into a sink again, cut ones // included as a fresh sink gets them, for a reader whose floor moved below them; assigned in // the producer's static block. @@ -834,6 +844,53 @@ let startsOf: (producer: Producer) => readonly Start[]; // The exclusive end of the run of sequences from `sequence` the producer's feed can no longer // deliver, undefined while it may; assigned in its static block. let goneAt: (producer: Producer, sequence: bigint) => bigint | undefined; + +/** The groups a reader reads, and the exclusive end it stops at; undefined before its ranges resolve. */ +type Coverage = { runs: readonly Run[]; cap?: bigint } | undefined; + +/** Whether two coverages hold the same groups and cap. */ +function sameCoverage(a: Coverage, b: Coverage): boolean { + if (a === undefined || b === undefined) return a === b; + return ( + a.cap === b.cap && + a.runs.length === b.runs.length && + a.runs.every((run, i) => run.start === b.runs[i].start && run.end === b.runs[i].end) + ); +} + +/** + * Whether the live edge `received` reached `final`: no sequence below it is left that was neither + * written nor given up, as a declared skip or a drop. Mirrors the Rust `TrackState::reaches`. + */ +function reaches(received: bigint, drops: readonly Run[], final: bigint): boolean { + return received >= final || runGaps(drops, received, final).length === 0; +} + +/** + * Whether `run` settles a reader judging under `budget` with `coverage`, on a track whose upstream + * judged stale runs against ranges naming the groups in `judged` (undefined for every group). Any + * run but a received stale one does. A stale verdict holds for a reader no wider than the budget it + * was judged under that reads every group the upstream's ranges name above the run's start: its + * own successor and edge are then no older than those the verdict was judged against, since a + * group's verdict looks only above it. Mirrors the Rust `settles`. + */ +function settles(run: DropRun, budget: number, coverage: Coverage, judged: readonly Run[] | undefined): boolean { + if (run.stale === undefined) return true; + if (budget > run.stale || coverage === undefined) return false; + const above = run.start + 1n; + // Whether the reader reads every group from `start` up to the exclusive `end`, `UNBOUNDED` + // past the live edge. + const reads = (start: bigint, end: bigint): boolean => { + if (coverage.cap !== undefined && (end === UNBOUNDED || end > coverage.cap)) return false; + const held = coverage.runs.find((run) => run.start <= start && start < run.end); + return held !== undefined && held.end >= end; + }; + for (const range of judged ?? [{ start: 0n, end: UNBOUNDED }]) { + if (range.end <= above) continue; + if (!reads(max(range.start, above), range.end)) return false; + } + return true; +} // Actual object metadata outlives cached payloads, unlike empty group heads in model largest. let objectLargest: (producer: Producer) => Position | undefined; // The fills serving the producer's misses; assigned in its static block. @@ -996,6 +1053,13 @@ export class Producer { // later arrival opened it, as the session's tail does, so this holds only the gaps still // worth waiting for. #arrivals = new Ledger(); + // What this track's own publisher produced since its last append, which skips every + // sequence below it that never arrived and was not given up: the producer moved on. The + // arrivals cannot say so, since a gap folds into them once nobody waits for it. Only + // sequences at or above `#appended` are kept. Mirrors the Rust `produced`. + #produced: Run[] = []; + // One past the last append: every sequence below it arrived, was given up, or was skipped. + #appended?: bigint; constructor(name: string) { this.name = name; @@ -1009,7 +1073,7 @@ export class Producer { this.accept({ ...info, maxAge: info.maxAge ?? null }); return this.#createGroup(sequence, 0); }, - drop: (start, end, error) => this.#drop(start, end, error), + drop: (start, end, error) => this.#drop(start, end, { error }), holds: (sequence) => cachedHolds(this, sequence), closed: () => this.#state.closed.peek() !== undefined, released: this.#state.closed, @@ -1045,6 +1109,10 @@ export class Producer { producer.#starts = starts; }; hooks.receiveGroup = (producer, { sequence, start }) => producer.#createGroup(sequence, start); + hooks.arrivingGroup = (producer, sequence) => { + const closed = producer.#cached.get(sequence)?.group.closed; + return closed?.peek() === undefined ? closed : undefined; + }; sourceStart = (producer) => producer.#served && producer.#feedStart(); fillFloor = (producer, named) => producer.#fillFloor(named); startsOf = (producer) => producer.#bounds; @@ -1062,7 +1130,18 @@ export class Producer { hooks.releaseTrack = (producer) => producer.#close(undefined); hooks.trackReleased = (producer) => producer.#state.closed; hooks.trackFinished = (producer) => producer.#state.end.peek() !== undefined; - hooks.dropTrackGroups = (producer, start, end, error) => producer.#drop(start, end, error, true); + hooks.dropTrackGroups = (producer, start, end, hole) => producer.#drop(start, end, hole, true); + hooks.skipTrackGroups = (producer, start, end) => producer.#skipReceived(start, end); + hooks.supersedeStale = (producer, judged) => { + const held = producer.#state.drops.peek().filter((run) => run.stale !== undefined); + for (const state of [producer.#state, ...producer.#sinks]) { + state.judged = judged; + const drops = state.drops.peek(); + if (drops.some((run) => run.stale !== undefined)) + state.drops.set(drops.filter((run) => run.stale === undefined)); + } + return held.map(({ start, end }) => ({ start, end })); + }; hooks.dropAnchors = (producer) => { for (const state of [producer.#state, ...producer.#sinks]) state.dropsAnchors.set(true); }; @@ -1081,6 +1160,15 @@ export class Producer { const entry = producer.#cached.get(sequence); return entry !== undefined && !(entry.group.closed.peek() instanceof Error); }; + cachedIn = (producer, start, end) => { + if (end - start <= BigInt(producer.#cached.size)) { + const found: bigint[] = []; + for (let sequence = start; sequence < end; sequence++) + if (producer.#cached.has(sequence)) found.push(sequence); + return found; + } + return [...producer.#cached.keys()].filter((sequence) => sequence >= start && sequence < end); + }; cachedMirror = (producer, sequence, from) => { const entry = producer.#cached.get(sequence); if (!entry || entry.group.closed.peek() instanceof Error) return undefined; @@ -1157,25 +1245,46 @@ export class Producer { } } - // Record `[start, end)` given up with `error`. An upstream drop (`open`) also ends the - // groups still arriving in it with that error, so their readers settle now rather than - // once the stream's reset lands; one that finished here stays whole. - #drop(start: bigint, end: bigint, error: Error, open = false): void { - if (start >= end) return; - // A fill no longer owes what the track gave up. - this.#fills.dropped(start, end); + // Record `[start, end)` given up as `hole` says, returning the runs recorded. A skip is final + // and the most authoritative verdict, so a drop never replaces one. An upstream drop (`open`) + // also ends the groups still arriving in it with its error, so their readers settle now + // rather than once the stream's reset lands; one that finished here stays whole. A skip names + // nothing produced. + #drop(start: bigint, end: bigint, hole: Hole, open = false): Run[] { + if (start >= end) return []; + const skipped = hole.skip ? [] : this.#state.drops.peek().filter((run) => run.skip); + let runs = runGaps(skipped, start, end); + // A received stale verdict leaves a group that arrived and was not aborted: a lite-07 + // publisher may serve a group after its stale drop, and the group's stream may land first. + // The group wins, still arriving or not. + if (open && hole.stale !== undefined) { + for (const [sequence, entry] of this.#cached) { + if (sequence >= start && sequence < end && !(entry.group.closed.peek() instanceof Error)) { + runs = removeRun(runs, sequence, sequence + 1n); + } + } + } + // A fill no longer owes what the track gave up for good. A stale verdict settles only + // the readers it covers, so a fill asked by any other is still owed. + if (hole.stale === undefined) this.#fills.dropped(start, end); for (const state of [this.#state, ...this.#sinks]) { - state.drops.update((drops) => insertDrop(drops, start, end, error)); + state.drops.update((drops) => + runs.reduce((drops, run) => insertDrop(drops, run.start, run.end, hole), drops), + ); } - if (open) { + if (open && !hole.skip && hole.stale === undefined) { for (const [sequence, entry] of this.#cached) { if (sequence >= start && sequence < end && entry.group.closed.peek() === undefined) { - entry.group.abort(error); + entry.group.abort(hole.error); } } } - // A pending tail owes nothing the publisher gave up. - this.#arrive(start, end); + // A pending tail owes nothing the publisher gave up. A stale verdict is not receipt: new + // terms may supersede it, leaving those groups unknown again, as in Rust. + if (hole.stale === undefined) this.#arrive(start, end); + // Giving up the last sequences below the end reaches it, as writing them would. + this.#endLargest(); + return runs; } // A publisher may re-create a group it cut; the sequence is no longer given up. @@ -1362,6 +1471,7 @@ export class Producer { this.#prune(); for (const entry of this.#cache) this.#mirror(entry, sink); sink.drops.set(this.#state.drops.peek()); + sink.judged = this.#state.judged; sink.received.set(this.#received); sink.endError = this.#state.endError; @@ -1621,7 +1731,7 @@ export class Producer { // A re-created group already replaced this one. const current = this.#cached.get(group.sequence); if (current && current.group !== group) return; - this.#drop(group.sequence, group.sequence + 1n, closed); + this.#drop(group.sequence, group.sequence + 1n, { error: closed }); }); const sequence = group.sequence; if (this.#unchecked()) { @@ -1637,6 +1747,7 @@ export class Producer { this.#fills.arrived(group.sequence); this.#advance(group.sequence, true); this.#arrive(group.sequence); + this.#produce(group.sequence); this.#state.latest = max(this.#state.latest ?? 0n, group.sequence); for (const sink of this.#sinks) this.#mirror(entry, sink); // Follow the group's frames as they land, past the head `#advance` already named. @@ -1708,6 +1819,12 @@ export class Producer { /** * Append a new group with the next sequence number: one past this producer's own writes. * + * Appending says the producer will not go back, so it skips every sequence below the new + * group that it never produced, from just past its previous append (before the first, from + * the lowest group it created), as {@link skipGroups} would, replacing any drop there. Those + * sequences are then final: creating one throws {@link Skipped}. Appending alone never leaves + * such a gap; one left by out-of-order {@link createGroup} calls is what it names. + * * The broadcast remembers nothing of an earlier producer of the name, so a replacement * starts at 0. Since a name always means the same content, a replacement continues past * its predecessor's groups with {@link createGroup}. @@ -1717,17 +1834,96 @@ export class Producer { * So does a first frame that starts before the group below it or after the one above it. */ appendGroup(): GroupProducer { - const group = new GroupProducer(this.#nextSequence()); + const sequence = this.#nextSequence(); + this.#refuseSkipped(sequence); + const group = new GroupProducer(sequence); this.#bind(group); this.#cap(group); this.#publish(group); + this.#settleAppend(sequence); return group; } + /** + * Declare that this producer will never produce `[start, end)`: none will arrive, so a reader + * waiting on one moves past it, the declared end does not wait for them, and a relay serving + * this track forwards the skip (lite-07 SUBSCRIBE_SKIP). Unlike a drop, it ends no reader. + * + * This and {@link appendGroup} are the only sources of a skip: a gap between created groups + * says nothing, since groups may be created out of order, so only the producer knows a + * sequence will never exist. A skip is final: creating a group at a skipped sequence throws + * {@link Skipped}, and a later drop of it is ignored. + * + * A skip replaces any drop of the same sequences, whatever its error: a reader the drop + * already ended stays ended, and every later reader and relay sees the skip. A group already + * cached keeps what it is, since it was produced after all. Mirrors the Rust `skip_groups`. + */ + skipGroups(start: bigint, end: bigint): void { + checkSequence(start); + checkSequence(end); + this.#skipReceived(start, end); + } + + // `skipGroups`, returning the cached sequences it left alone: a skip naming a group that + // arrived contradicts it, which a subscriber reports. + #skipReceived(start: bigint, end: bigint): Run[] { + if (this.#state.closed.peek() !== undefined) return []; + const cached: Run[] = []; + for (const sequence of this.#cached.keys()) { + if (sequence >= start && sequence < end) insertRun(cached, sequence, sequence + 1n); + } + this.#skipGaps(cached, start, end); + return cached; + } + + // Skip what of `[start, end)` neither `kept` nor an earlier skip names, replacing any drop. + #skipGaps(kept: readonly Run[], start: bigint, end: bigint): void { + const skipped = this.#state.drops.peek().filter((run) => run.skip); + for (const gap of runGaps(kept, start, end)) { + for (const open of runGaps(skipped, gap.start, gap.end)) { + this.#drop(open.start, open.end, skipHole()); + } + } + } + + // Record a sequence this track's own publisher produced, for the next append to skip around. + #produce(sequence: bigint): void { + if (this.#subscribed || (this.#appended !== undefined && sequence < this.#appended)) return; + insertRun(this.#produced, sequence, sequence + 1n); + } + + // Settle an append at `sequence`: skip every sequence below it this track's own publisher + // never produced, from just past its previous append (before the first, from the lowest it + // produced), since appending says it will not go back. The skip replaces a drop given there, + // as `skipGroups` does. + #settleAppend(sequence: bigint): void { + const from = this.#appended ?? this.#produced[0]?.start; + if (from !== undefined) { + const kept = [...this.#produced]; + for (const cached of this.#cached.keys()) { + if (cached >= from && cached < sequence) insertRun(kept, cached, cached + 1n); + } + this.#skipGaps(kept, from, sequence); + } + this.#produced = []; + this.#appended = sequence + 1n; + } + + // Refuse a sequence the publisher declared skipped: a skip is final. + #refuseSkipped(sequence: bigint): void { + const run = this.#state.drops.peek().find((run) => run.start <= sequence && sequence < run.end); + if (run?.skip) throw new Skipped(sequence); + } + /** * Create a new group with the given sequence number. * + * It says nothing about the sequences around it: groups may be created out of order, and an + * aborted or evicted sequence created again. A gap it leaves is skipped only once the + * producer says it will not fill it, with {@link skipGroups} or {@link appendGroup}. A + * sequence already skipped throws {@link Skipped}. + * * Throws on a sequence that is still cached: a live duplicate would fan out to every * subscriber twice. An aborted incarnation is evicted so a fresh group can serve the * sequence again. Best effort (mirrors Rust): nothing remembers a sequence whose cache @@ -1740,6 +1936,9 @@ export class Producer { #createGroup(sequence: bigint, start: number): GroupProducer { this.#writable(sequence); + // A skip says the sequence will never be produced, so it is final. A drop is not: a + // publisher may re-create a group it cut or aborted. + this.#refuseSkipped(sequence); const existing = this.#cached.get(sequence); if (existing) { if (!(existing.group.closed.peek() instanceof Error)) { @@ -1927,7 +2126,7 @@ export class Producer { this.#settle(); if (this.#state.largestEnded.peek() || this.#largestOpen) return; const end = this.#state.end.peek(); - if (end === undefined || this.#received < finalSequence(end)) return; + if (end === undefined || !reaches(this.#received, this.#state.drops.peek(), finalSequence(end))) return; for (const state of [this.#state, ...this.#sinks]) state.largestEnded.set(true); } @@ -2007,7 +2206,7 @@ export class Producer { const end = this.#state.end.peek(); if (end === undefined) return false; const final = finalSequence(end); - if (this.#received < final || this.#owes(final)) return false; + if (!reaches(this.#received, this.#state.drops.peek(), final) || this.#owes(final)) return false; return this.#cache.every(({ group }) => group.sequence >= final || group.closed.peek() === null); } @@ -2099,8 +2298,25 @@ export class Subscriber { // the track's drops last folded in, and every sequence already reported. Drops are // persistent track state, so they are tracked here rather than removed for everyone. #pending: DropRun[] = []; - #dropsSeen?: { drops: readonly DropRun[]; start: bigint; end?: bigint; revision: number }; + #dropsSeen?: { + drops: readonly DropRun[]; + start: bigint; + end?: bigint; + revision: number; + budget: number; + coverage?: Coverage; + judged?: readonly Run[]; + }; #reported: Run[] = []; + // The reported runs judged stale, for a wire whose update re-offers them; see + // `hooks.reofferStale`. + #reportedStale: Run[] = []; + // The part of #reportedStale an upstream's received stale run decided, which is the upstream's + // to judge again once an update reopens it. + #reportedReceived: Run[] = []; + // The reported runs skipped: a skip replaces a drop, so a run reported dropped is reported + // again once it is skipped, and only a skip already reported stops it. + #reportedSkip: Run[] = []; // The group the frame-level helpers are currently draining, acquired through the // sequence cursor so frame reads and {@link Ordered.nextGroup} share one floor. #frameGroup?: GroupConsumer; @@ -2351,6 +2567,12 @@ export class Subscriber { if (ordered) this.#delivered.forgetBelow(sequence); this.#delivered.add(sequence); this.#skipped.delete(sequence); + // The group won over any stale report on it, so no update reopens it. A resumed reader + // retires its own. + if (this.#reportedStale.length > 0) { + this.#reportedStale = removeRun(this.#reportedStale, sequence, sequence + 1n); + this.#reportedReceived = removeRun(this.#reportedReceived, sequence, sequence + 1n); + } // Another copy of it buffered here is never handed out. this.#judge(sequence, sequence + 1n); if (this.#resume) this.#resume.forgetPastBudget(this.#delivered); @@ -2460,7 +2682,7 @@ export class Subscriber { // Judged again, so a stale drop not reported yet is withdrawn. if (dropAt(this.#pending, sequence) instanceof Error) { this.#pending = this.#pending.flatMap((run) => - run.start <= sequence && sequence < run.end && toStreamCode(run.error) === StreamCode.Old + run.start <= sequence && sequence < run.end && run.stale !== undefined ? removeRun([run], sequence, sequence + 1n) : [run], ); @@ -2471,6 +2693,72 @@ export class Subscriber { if (producer && !this.#closed) reofferSequences(producer, this.#state, sequences); } + // Owe again every run this cursor gave up as stale, reported or not: an update reopens the + // verdict. A group still cached is offered again as a widened skip is, and whatever is still + // stale under the new terms is given up and reported again. Returns the runs taken back. A + // resumed reader takes back what it queued and what its serving route's copy reported. + #reofferStale(): Run[] { + if (this.#resume) return this.#resume.reofferStale(); + const taken: Run[] = []; + const received = this.#reportedReceived; + for (const run of this.#reportedStale) { + this.#reported = removeRun(this.#reported, run.start, run.end); + insertRun(taken, run.start, run.end); + } + this.#reportedStale = []; + this.#reportedReceived = []; + for (const run of this.#pending) { + if (run.stale === undefined) continue; + insertRun(taken, run.start, run.end); + if (run.received) insertRun(received, run.start, run.end); + } + this.#pending = this.#pending.filter((run) => run.stale === undefined); + // A received stale run is folded in again by the next scan, if the track still holds it. + this.#dropsSeen = undefined; + const sequences = [...this.#skipped.keys()].filter((sequence) => + taken.some((run) => run.start <= sequence && sequence < run.end), + ); + // What no skip offers again, such as a group written off at the track's max age bound, + // which no budget reverses, is still stale under the new terms: reported again. What an + // upstream's stale run decided is the upstream's to judge again: the scan above folds it + // in again if the track still holds it, and it stays owed if new terms superseded it. + // Mirrors Rust. + let kept: Run[] = taken.map((run) => ({ ...run })); + for (const sequence of sequences) kept = removeRun(kept, sequence, sequence + 1n); + for (const decided of received) kept = removeRun(kept, decided.start, decided.end); + if (kept.length > 0) { + const hole = staleHole(this.#drift().budget); + for (const run of kept) this.#pending = insertDrop(this.#pending, run.start, run.end, hole); + } + const producer = this.#state.producer; + if (sequences.length > 0 && producer && !this.#closed) reofferSequences(producer, this.#state, sequences); + return taken; + } + + // Fresh reads of the cached groups in `[start, end)` this cursor admits and no longer judges + // stale, each expiring against the budget, and the runs still given up. A resumed reader asks + // its serving route's copy. Mirrors the Rust `reoffer_groups`. + #reofferGroups(start: bigint, end: bigint): { groups: GroupConsumer[]; givenUp: Run[] } { + if (this.#resume) return this.#resume.reofferGroups(start, end); + const groups: GroupConsumer[] = []; + let givenUp: Run[] = [{ start, end }]; + const producer = this.#state.producer; + if (!producer) return { groups, givenUp }; + const drift = this.#drift(); + const cap = this.#cursor.peek().end; + // Only what the cache holds, so a long run of groups long gone costs nothing. + const cached = cachedIn(producer, start, end).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + for (const sequence of cached) { + const admitted = (cap === undefined || sequence < cap) && this.#window.contains(sequence); + if (!admitted || this.#staleAt(sequence, drift)) continue; + const group = cachedMirror(producer, sequence, 0); + if (!group) continue; + groups.push(this.#guard(group)); + givenUp = removeRun(givenUp, sequence, sequence + 1n); + } + return { groups, givenUp }; + } + private constructor(name: string, state: TrackState) { this.name = name; this.#state = state; @@ -2618,7 +2906,8 @@ export class Subscriber { priority: () => this.#state.update.peek()?.priority ?? 0, maxDelay: () => this.#drift().budget, order: () => this.#state.update.peek()?.order ?? "descending", - refuse: (start, end, error) => this.#state.drops.update((drops) => insertDrop(drops, start, end, error)), + refuse: (start, end, error) => + this.#state.drops.update((drops) => insertDrop(drops, start, end, { error })), }; return this.#state.asker; } @@ -2638,18 +2927,37 @@ export class Subscriber { const asker = this.#askerOf(); this.#clipAsked(); const timeline = this.#state.timeline; - const gone = [...this.#state.drops.peek()]; + const accounted: Run[] = []; // Reordered streams still owed by the live feed are not cache misses. for (const run of owedRuns(producer, start, end)) { - insertRun(gone, run.start, run.end); + insertRun(accounted, run.start, run.end); if (run.folds !== undefined) this.#wakeAt(run.folds); } // Below where the feed starts, only the history the copy's start rule leaves fillable. const floor = fillFloor(producer, this.#window.start()); - for (const run of this.#wants( - floor === undefined ? start : max(start, floor), - min(end, max(this.#state.received.peek(), (fillEdge(producer) ?? -1n) + 1n)), - )) { + // A received stale run that does not settle this reader is unknown to it, so it is + // still asked for, as the Rust `Misses::unclaimed` judges it. + const budget = drift.budget; + const coverage = this.#coverage(); + const drops = this.#state.drops.peek(); + const low = floor === undefined ? start : max(start, floor); + const edge = min(end, max(this.#state.received.peek(), (fillEdge(producer) ?? -1n) + 1n)); + const spans: Run[] = [{ start: low, end: edge }]; + // The upstream accounted for a stale run, so the live feed brings none of it unless an + // update supersedes it: asked even above the newest group received. + for (const dropped of drops) { + const span = { ...dropped, start: max(dropped.start, max(low, edge)), end: min(dropped.end, end) }; + if (span.start < span.end && !settles(span, budget, coverage, this.#state.judged)) spans.push(span); + } + for (const run of spans.flatMap((span) => this.#wants(span.start, span.end))) { + const gone = [...accounted]; + for (const dropped of drops) { + if (dropped.end <= run.start || run.end <= dropped.start) continue; + // Judged from where it meets the asked run, as the reader would read it. + const clipped = { ...dropped, start: max(dropped.start, run.start) }; + if (settles(clipped, budget, coverage, this.#state.judged)) + insertRun(gone, clipped.start, dropped.end); + } for (const open of runGaps(gone, run.start, run.end)) { let next = open.start; for (let i = timelineIndex(timeline, open.start); i < timeline.length; i++) { @@ -2781,12 +3089,23 @@ export class Subscriber { const held = this.#state.groups.peek(); const producer = this.#state.producer; const received = max(this.#state.received.peek(), ((producer && fillEdge(producer)) ?? -1n) + 1n); - const drops = this.#state.drops.peek(); + // A received stale run that does not settle this reader is unknown to it (see `settles`): + // the reader still wants those groups, however the feed accounted for them. + const budget = this.#drift().budget; + const coverage = this.#coverage(); + const all = this.#state.drops.peek(); for (const run of this.#window.runs(this.#window.floor() ?? 0n, end)) { let gaps = [run]; for (const delivered of taken) gaps = gaps.flatMap((gap) => delivered.gaps(gap.start, gap.end)); for (const gap of gaps) { + // Each run judged from where it meets the gap, as the reader would read it. + const overlapping = all + .filter((run) => run.start < gap.end && gap.start < run.end) + .map((run) => ({ ...run, start: max(run.start, gap.start), end: min(run.end, gap.end) })); + const drops = overlapping.filter((run) => settles(run, budget, coverage, this.#state.judged)); + const unknown = overlapping.filter((run) => !settles(run, budget, coverage, this.#state.judged)); for (const open of runGaps(drops, gap.start, gap.end)) { + if (unknown.some((run) => run.start < open.end && open.start < run.end)) return false; // Not received yet, so it may still arrive. if (open.end > received) return false; // Held, so a read still hands it out, skips it, or waits for the cap to rise. @@ -2875,6 +3194,8 @@ export class Subscriber { hooks.unreport = (subscriber, start, end) => { subscriber.#reported = removeRun(subscriber.#reported, start, end); }; + hooks.reofferStale = (subscriber) => subscriber.#reofferStale(); + hooks.reofferGroups = (subscriber, start, end) => subscriber.#reofferGroups(start, end); hooks.deliveredRuns = (subscriber) => subscriber.#delivered.runs; hooks.handedOut = (subscriber, sequence) => subscriber.#delivered.has(sequence); hooks.hasUnreadGroup = (subscriber, start, end, delivered) => @@ -2907,6 +3228,14 @@ export class Subscriber { const producer = subscriber.#state.producer; return producer !== undefined && cachedHolds(producer, sequence); }; + hooks.staleHeld = (subscriber, start, end) => { + const held: Run[] = []; + for (const run of subscriber.#state.drops.peek()) { + if (run.stale === undefined || run.end <= start || run.start >= end) continue; + insertRun(held, max(run.start, start), min(run.end, end)); + } + return held; + }; hooks.givenUp = (subscriber, sequence) => { const drops = subscriber.#state.drops; return { error: dropAt(drops.peek(), sequence), changed: drops }; @@ -3319,7 +3648,6 @@ export class Subscriber { this.#fillWait = false; const groups = this.#state.groups.peek(); const { start, end } = this.#cursor.peek(); - const old = () => new StreamError(StreamCode.Old, { message: "group is past the budget" }); if (dispositions) { const dropped = this.#tryDrop(); @@ -3345,7 +3673,12 @@ export class Subscriber { groups.splice(index, 1); const failed = group.closed.peek(); if (dispositions && failed instanceof Error) { - const dropped = this.#report(group.sequence, group.sequence + 1n, failed); + // A group expired under this subscription's budget is a stale verdict. + const hole = + toStreamCode(failed) === StreamCode.Old + ? { ...staleHole(drift.budget), error: failed } + : { error: failed }; + const dropped = this.#report(group.sequence, group.sequence + 1n, hole); if (dropped) return dropped; continue; } @@ -3359,11 +3692,16 @@ export class Subscriber { if (drift.bounded) this.#skipped.delete(group.sequence); else this.#skip(group.sequence, drift.budget); if (dispositions) { - const dropped = this.#report(group.sequence, group.sequence + 1n, old()); + const dropped = this.#report(group.sequence, group.sequence + 1n, staleHole(drift.budget)); if (dropped) return dropped; } else if (this.#dropsSeen) { // A cursor #tryDrop reads reports it there. - this.#pending = insertDrop(this.#pending, group.sequence, group.sequence + 1n, old()); + this.#pending = insertDrop( + this.#pending, + group.sequence, + group.sequence + 1n, + staleHole(drift.budget), + ); } continue; } @@ -3384,14 +3722,29 @@ export class Subscriber { return this.#parked(groups) ? { kind: "boundary" } : { kind: "done" }; } - // The first part of `[from, to)` this cursor has not reported yet, now reported, with why. - #report(from: bigint, to: bigint, error: Error): (Recv & { kind: "drop" }) | undefined { + // The first part of `[from, to)` this cursor has not reported yet, now reported, with why. A + // skip goes out over a drop already reported, and a final drop over a stale one, since each + // replaces what it goes out over. + #report(from: bigint, to: bigint, hole: Hole & { received?: boolean }): (Recv & { kind: "drop" }) | undefined { const { start, end } = this.#cursor.peek(); + const final = !hole.skip && hole.stale === undefined; for (const run of this.#window.runs(max(from, start), min(to, end ?? to))) { - const [gap] = runGaps(this.#reported, run.start, run.end); + let blocked = hole.skip ? this.#reportedSkip : this.#reported; + if (final) for (const stale of this.#reportedStale) blocked = removeRun(blocked, stale.start, stale.end); + const [gap] = runGaps(blocked, run.start, run.end); if (!gap) continue; insertRun(this.#reported, gap.start, gap.end); - return { kind: "drop", ...gap, error }; + if (hole.skip) insertRun(this.#reportedSkip, gap.start, gap.end); + if (hole.stale !== undefined) { + insertRun(this.#reportedStale, gap.start, gap.end); + if (hole.received) insertRun(this.#reportedReceived, gap.start, gap.end); + else this.#reportedReceived = removeRun(this.#reportedReceived, gap.start, gap.end); + } else { + // A skip or final drop is final, so no update reopens a stale report it replaced. + this.#reportedStale = removeRun(this.#reportedStale, gap.start, gap.end); + this.#reportedReceived = removeRun(this.#reportedReceived, gap.start, gap.end); + } + return { kind: "drop", ...gap, error: hole.error, skip: hole.skip, stale: hole.stale }; } return undefined; } @@ -3409,20 +3762,48 @@ export class Subscriber { // Out-of-window runs were skipped, so a moved window rescans too. const drops = this.#state.drops.peek(); const revision = this.#revision; + const { budget } = this.#drift(); + const coverage = this.#coverage(); + const judged = this.#state.judged; const seen = this.#dropsSeen; - if (!seen || seen.drops !== drops || seen.start !== start || seen.end !== end || seen.revision !== revision) { - this.#dropsSeen = { drops, start, end, revision }; - for (const run of drops) this.#pending = insertDrop(this.#pending, run.start, run.end, run.error); + if ( + !seen || + seen.drops !== drops || + seen.start !== start || + seen.end !== end || + seen.revision !== revision || + seen.budget !== budget || + !sameCoverage(seen.coverage, coverage) || + seen.judged !== judged + ) { + this.#dropsSeen = { drops, start, end, revision, budget, coverage, judged }; + // The track's runs are folded in afresh, so one it forgot, such as a stale run new terms + // superseded, or one that stopped settling this reader, leaves nothing behind. + this.#pending = this.#pending.filter((run) => !run.received); + for (const dropped of drops) { + // Judged from where it meets this cursor's span, as Rust clips it. + const run = { ...dropped, start: max(dropped.start, start), end: min(dropped.end, end ?? dropped.end) }; + if (run.start >= run.end) continue; + // A received stale run that does not settle this reader is unknown to it. + if (!settles(run, budget, coverage, judged)) continue; + this.#pending = insertDrop(this.#pending, run.start, run.end, { ...run, received: true }); + } } for (let run = this.#pending.shift(); run; run = this.#pending.shift()) { - const dropped = this.#report(run.start, run.end, run.error); + const dropped = this.#report(run.start, run.end, run); if (!dropped) continue; - if (dropped.end < run.end) this.#pending.unshift({ start: dropped.end, end: run.end, error: run.error }); + if (dropped.end < run.end) this.#pending.unshift({ ...run, start: dropped.end }); return dropped; } return undefined; } + // The groups this reader reads, and its cap; see `Coverage`. + #coverage(): Coverage { + if (this.#window.pending) return undefined; + return { runs: this.#window.runs(), cap: this.#cursor.peek().end }; + } + // A group from the routes' copies; see #resume. #tryResumed(ordered: boolean, dispositions = false): Recv { const resume = this.#resume as ResumeSource; @@ -3440,7 +3821,11 @@ export class Subscriber { // accounted for by the wire layer, and an abort still wins on the next read. #complete(): boolean { const end = this.#state.end.peek(); - return end !== undefined && this.#state.received.peek() >= finalSequence(end) && !this.#state.owed.peek(); + return ( + end !== undefined && + reaches(this.#state.received.peek(), this.#state.drops.peek(), finalSequence(end)) && + !this.#state.owed.peek() + ); } // Package-internal readiness half of recvGroup. Each registration fires at most once, and diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index e3bd133b9d1..d111b16b948 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -180,6 +180,7 @@ client = moq.Client( - `.write_frame(payload, timestamp=None)` writes a payload with optional capture time. - `await .ready(size)` (`GroupProducer`) waits until a frame of `size` bytes fits a window ahead of the fastest reader. Await it before each write for backpressure. A writer that doesn't wait sheds the front of the group instead. - `.create_group(sequence)` creates a sparse or replayed group at an explicit sequence. + - `.skip_groups(start, end)` (`TrackProducer`) declares groups `start` up to the exclusive `end` will never be produced, so readers move past them; creating one there then fails. - `.finish()` ends at the live edge; the handle remains so `.abort(error_code)` can still run. - `.finish_at(final_sequence)` declares the first group that will never be produced while leaving lower groups writable. - `.abort(error_code)` terminates the track or group with an application error. diff --git a/py/moq-rs/moq/publish.py b/py/moq-rs/moq/publish.py index b1498a592da..09c3c1b3dda 100644 --- a/py/moq-rs/moq/publish.py +++ b/py/moq-rs/moq/publish.py @@ -136,6 +136,13 @@ def create_group(self, sequence: int) -> GroupProducer: """Create a group with an explicit sequence number.""" return GroupProducer(self._inner.create_group(sequence)) + def skip_groups(self, start: int, end: int) -> None: + """Declare that groups ``start`` up to the exclusive ``end`` will never be produced. + + Readers move past them, and creating a group there then fails. + """ + self._inner.skip_groups(start, end) + def write_frame(self, payload: bytes, timestamp: Timestamp | None = None) -> None: """Write a single-frame group with its presentation timestamp.""" self._inner.write_frame(Frame(payload, timestamp)._ffi()) diff --git a/quest/m0/test-flakes-2/datagram-end-close.md b/quest/m0/test-flakes-2/datagram-end-close.md index 656e632d966..3cfc55dbf34 100644 --- a/quest/m0/test-flakes-2/datagram-end-close.md +++ b/quest/m0/test-flakes-2/datagram-end-close.md @@ -1,12 +1,12 @@ -# [S] The datagram-end close test survives load +# [S] datagram-tracks' end-before-close case passes under load ## Goal -`js/net/src/lite/datagram-tracks.test.ts`'s -`a session close right after a datagram track's end keeps the end` passes -reliably in loaded runs, fixed at its cause under the -[epic](/quest/m0/test-flakes-2/README.md)'s rules, and its `test.skip` is -removed. +`a session close right after a datagram track's end keeps the end` in +`js/net/src/lite/datagram-tracks.test.ts` passes under load for every draft in +multi-file runs, fixed at its cause under the +[epic](/quest/m0/test-flakes-2/README.md)'s rules. Done when its `test.skip` +is removed with its assertions unchanged. ## Plan @@ -15,38 +15,53 @@ stubbed the publisher write budget. The additional duplicate-SETUP investigation below remains open. Added 2026-11-06 (maintainer's standing rule that flakes join this epic). -[moq-dev/moq#7986](https://github.com/moq-dev/moq/pull/7986)'s Check failed -in this case, whose diff touches only `bench/`, `justfile`, and quest prose. -It reproduces on unchanged `origin/main`, so the flake came from -[moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951)'s new case; -the PR quarantines it with every assertion kept. - -Reproduction 2026-11-06 on this head and on `git archive origin/main`, from -`js/net`, with six CPU loops on four cores: - -- `bun test src/lite/datagram-tracks.test.ts` failed 3 of 4 loaded runs here - and 3 of 3 on `origin/main`; only the lite-06 draft failed, though the - case loops over lite-05, lite-06, and lite-07-wip. -- The same filter `-t "a session close right after a datagram track's end - keeps the end"` passed 10 of 10 idle runs and 8 of 8 loaded runs, so the - whole file (its 37 other cases) is what makes the window miss. +[moq-dev/moq#7986](https://github.com/moq-dev/moq/pull/7986) and +[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) both hit it in +their checks without touching the code it covers, and each quarantined it with +every assertion kept; this file merges their two quests. It came with +[moq-dev/moq#7951](https://github.com/moq-dev/moq/pull/7951)'s new case. + +Reproductions, 2026-11-06, with six CPU loops on four cores: + +- On #7986's head and on `origin/main`, from `js/net`: `bun test + src/lite/datagram-tracks.test.ts` failed 3 of 4 loaded runs there and 3 of + 3 on `main`. The same file filtered to this case passed 10 of 10 idle and 8 + of 8 loaded runs, so the file's other cases are what make the window miss. +- On #7670 at `13a544e41`: 6 of 10 runs, and 4 of 8 on `main` at + `6ca2ed5e3`, always the lite-06 case, with `subscribe error: id=1 + broadcast=live track=audio error=remote error: 0`. After quarantining + lite-06 alone, lite-05 failed 1 of 8 further loaded runs the same way, so the + whole case is quarantined and the fix must cover all three drafts. - The failure rejects with `Session: remote error: 0` at `error.ts`'s - `closeError` from `connection.ts`'s `abort`, i.e. `reader.recv()` - rejected instead of ending. - -Likely cause, unverified: the case finishes the datagram track and then -drains only microtasks (`for (i < 1000) await Promise.resolve()`) before -`client.abort()`, assuming "the mock delivers the end in microtasks, all -within this host task". The FIN handler awaits `#datagramsRouted`, which -starts with a host-task `yieldNow`; `Subscriber.close` finishes only -datagram entries whose `ended` is already set and aborts the rest. Under -load the FIN is not read before the abort, so `ended` is still unset and -the producer aborts. Check the delivery point before choosing between -waiting on the end's arrival and yielding to the host in the drain. + `closeError` from `connection.ts`'s `abort`: `reader.recv()` rejected + instead of ending. + +The test ends the publisher's track, then burns microtasks so the end lands +"all within this host task", then closes the session and expects `recv()` to +report the end rather than the close. That holds only when the end's whole +path runs in microtasks, and the failure says it did not: the close reached +the subscriber while `entry.end` was still unset, so `Subscriber.close` +aborted the datagram track. That is the correct answer for a close that +precedes the end, so the assertion is fine and the assumption about when the +end has landed is the suspect. Lead, unverified: a host task still +outstanding, such as the `yieldNow` (`clock.task`) on the publisher's serve +loop or at the start of the FIN handler's `#datagramsRouted`, is starved by +the microtask chain. Lite-06 fits: its SUBSCRIBE carries no range list, so +fewer writes precede the publisher's read of the end. Confirm which host task +is outstanding before choosing the fix; a later host task landing in the +burst, rather than a leftover one, would need a different answer. + +Fix it by waiting on an event that proves the subscriber read the end before +closing the session, rather than a microtask count: for example capture the +publisher's `SUBSCRIBE_FINISH` write through the mock as `holdNextAnswer` +does, or await a hook the subscriber calls on the response. Keep the close +between that evidence and `recv()`, so the case still covers the close that +lands after the end. Raising the microtask count, adding a wall-clock wait, or +retrying the test are not fixes. Reproduce first: temporarily remove the case's `test.skip`, then run the -whole file beside CPU loops, as above. Keep the failing output before making -other changes to the test or the end path. Restore the skip until the fix. +whole file beside CPU loops, as above. Keep the failing output before +changing the test or the end path, and restore the skip until the fix. ### Additional lite-06 investigation @@ -77,3 +92,14 @@ overridden by session failure before subscription finalization. Reproduce this ordering deterministically and preserve the accepted end during shutdown. Public API: none. Wire: none. + +## Related + +- [moq-dev/moq#8000](https://github.com/moq-dev/moq/pull/8000) edited this + case: it stubs `Budget.prototype.poll` so the publisher's write budget never + yields a host task, and raises the microtask count to 10000. The quarantined + case keeps both. Stubbing the budget removes one suspected outstanding + `yieldNow`, so check whether it is the cause before choosing the fix; the + raised count alone is not one. On 2026-11-06, ten loaded runs (six busy + loops pinned to four of 24 cores) passed with and without the stub, so that + setup no longer reproduces it and cannot tell. diff --git a/quest/m1/README.md b/quest/m1/README.md index 8167de6e377..16f41621062 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -31,7 +31,8 @@ moves what remains of them once their claims merge. ## Required -- [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 +- [JS cache pool](/quest/m1/js-cache-pool.md) - `js/net` bills cached frames and drop-set runs to a per-broadcast byte pool ported from Rust +- [Drop set](/quest/m1/drop-set.md) - the drop set's track and cursor runs are billed to the cache pool, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` - [Media fuzz findings](/quest/m1/fuzz-findings/README.md) - the nightly media fuzz targets run clean: mp4-atom, MKV block, TS step, SPS overflow, and catalog encoder bugs they found are fixed - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate - [Routing guide](/quest/m1/routing-guide.md) - concepts and practical outbound federation setup diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index c250763b2dc..3e4e3e60843 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -2,260 +2,202 @@ ## Goal -Every cursor judges a missing group against its own budget. Today a dropped -group is final for every reader of a track: a narrow reader can wait -indefinitely for an upstream OLD naming a group the relay never cached, -and a wider reader never gets a group an upstream already reported stale. -The track state instead keeps the dropped and expired runs with their -reason and budget, so a narrow reader gets a prompt drop, a wide reader -waits, and a widening re-offers what its new budget covers. On lite-07 this -restores "growing means I want it": a group reported dropped under a -narrower budget is served after a widening, as a cold relay would. +A hostile upstream fragmenting drops costs what the cache's byte pool would +hold, never more: the drop set's per-track runs and each cursor's +disposition record are billed to the pool at a fixed cost per run, in Rust +and `js/net`, and a cursor's CPU per drop tracks the runs it touches, never +their span or the set. Eviction keeps every promise a reader was made: a +final run folds into the cursor's floor, and losing any other ends the +subscription with `TooFarBehind`. ## Plan -Decided 2026-10-27 (maintainer): this runs after the range count cost work, -which finished in [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542), -since both edit `given_up` in `rs/moq-net/src/model/track/miss.rs`, and before -[Rust groups by start](/quest/m2/time-addressing/model.md), which requires it -because runs keyed by group sequence convert mechanically to starts. - -Decided by the maintainer on 2026-10-19, splitting this from the widened -max age re-offer -([moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717)), which keeps -drops final for now. That PR re-offers cached groups a cursor skipped as -stale, in the subscription's order (newest first unless ascending; `js/net` -lowest first, since its buffer is ascending), through the -[moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) owed path. Its -skipped record (`Skipped` in `rs/moq-net/src/model/skipped.rs`, `#skipped` -in `js/net/src/track.ts`) keeps a run only while the cache holds one of its -groups. On 2026-10-25 the maintainer deferred the full pool-billed -disposition record to this quest. - -- **One run-length set per track.** The track state keeps the dropped and - expired group runs within the subscription range. Each run carries its - reason (stale, cut, never produced, or evicted or expired where no copy - remains: received as such from the upstream, or at an origin with no - upstream) and, for a stale run, the - budget it was judged under. A group that arrives removes its sequence; a - drop never removes a group that already arrived. A relay's own eviction - is never a run: the group is unknown again, and any cursor that wants it, - new, widened, or already owed it, fills it from upstream by range, the - existing miss fill. That needs no new state, since the cursor's record - already holds the span as unknown, a run at a time. -- **Each cursor judges for itself, a run at a time.** A cursor walks the - set's runs in its range, never their sequences, and settles each whole run - in O(1): - - A run whose reason does not depend on the budget (a cut, a group never - produced, an upstream or origin eviction or expiry) is dropped for every reader - as one range, as today. - - A received stale run is dropped promptly as one range for a reader - whose budget is no wider than the one it was judged under and whose - range covers this relay's aggregate range for the track, which the - upstream judged under. A verdict depends on the range as well as the - budget, since `drifted` caps the edge to the reader's range - (`Edge::cap`), so a narrower reader treats the run as unknown and fills - it by range, where the upstream judges it under that narrower range. - The cost is an extra range request, only for narrower readers, when a - stale drop arrives. - - A missing run this relay judges itself is dropped for a reader under - whose own budget and range `TrackState::drifted` convicts it. `drifted` - judges a missing sequence by its successor here, and every sequence in - a missing run shares one successor, so one call settles the run. - - A wider reader skips the run and waits for a group, a re-offer, or its - own verdict, each of which splits the run at most once. - - An unknown span (a gap between runs, or an evicted run) is judged the - same way, as one unit bounded by its neighbours: by one `drifted` call - against its shared successor, or by one range miss fill upstream. - - A hostile upstream's huge run costs one Charge and O(1) per cursor, and - splitting it costs per-run Charges the pool bounds, so CPU tracks pool - bytes, never span. +[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) landed the +behavior: lite-07 SUBSCRIBE_SKIP and SUBSCRIBE_DROP with `Update`, a per-run +reason (`Hole` in `rs/moq-net/src/model/track/hole.rs`, +`Hole` in `js/net/src/internal.ts`), relays settling a received stale drop +only for readers it covers (`Drift::settles`, `settles` in +`js/net/src/track.ts`) and forgetting stale runs when they send new terms +(`supersede_stale`), and every lite-07 update re-offering what was reported +stale (`reoffer_stale`). It kept today's storage: `TrackState::drops`, +`DropCursor` with its `reported` and `reported_stale` sets, `Skipped`, +`Delivered`, `TrackRun::served`, the lite subscriber's record of the group +streams that finished since its last update (`Tail::finished`, JS +`Tail#finished`), and the JS `drops` signal, `#pending`, `#reported`, and +`#reportedStale`, none billed to anything. This quest is +what remains: bounding them. - This fixes the gap Codex found on - [moq-dev/moq#5986](https://github.com/moq-dev/moq/pull/5986): today - `TrackRun::poll_forward_drop` emits only recorded drops, cuts, or stale - cached groups. A sequence the relay never cached is not judged until the - track finishes. -- **Each cursor keeps a disposition record,** next to the per-track set: a - group that arrives leaves the shared set, so only the cursor's record - remembers that it reported cached G stale. The record is one run-length - map per cursor, each run marked delivered, skipped as stale, or reported - in SUBSCRIBE_DROP, with its reason and, for a stale run, the budget and - range it was judged under. It replaces the skipped record, `Delivered` - (`rs/moq-net/src/model/delivered.rs`), whose floor folds stale gaps in - with delivered groups, `TrackRun::served` in - `rs/moq-net/src/lite/publisher.rs`, and the JS wire cursor's `#reported`. - Delivery is recorded, not inferred from the cursor position, since an - arrival-order or descending cursor delivers out of sequence order. On - lite-07 a reported stale run is no longer final: like a skipped one, it - is re-offered when a widening covers it. -- **The record is bounded by the pool, never by guessing.** Each run is - billed to the cache's `Pool` through a `Charge` at a fixed per-run cost - and is settled a run at a time, so a cursor's state and CPU track its - runs, never their span. The floor only rises over a final prefix: the - runs from the floor up to the lowest run that is not final. An unknown - hole is never final: a descending cursor can deliver H while a lower G - is still unknown. When the pool evicts a run inside the final prefix, it - folds into the floor; any other run cannot go without guessing, so the - subscription ends with `TooFarBehind` and the subscriber re-subscribes. -- **A widening re-offers the dropped runs now inside its budget,** through - the same re-offer path as skipped runs, in the subscription's order. - Upstream, the relay's own widening asks for them again. -- **Only a widening reverses a stale verdict,** local or received alike: no - run tracks the group that convicted it, and an abort or eviction of that - group leaves the run standing. So warm/cold equivalence after an edge - loss is not promised: when the newer group that made G stale is aborted - or evicted, a cold cursor may serve G while a warm one stays past it. - Watching for it would need both groups a verdict depends on (`drifted` - uses G's successor and a possibly later presentation edge), and a run - skipped at the max age bound would no longer be final. A received run is - also reversed by the upstream re-offering its group after a widening, - since a later group wins over its drop. Cuts and never-produced runs are - unaffected. -- **A stale drop names its budget.** A SUBSCRIBE_UPDATE has no - acknowledgement, so after a widening from B1 to B2, an in-flight B1 drop - and the B2 re-drop look identical, and no guess settles a B2 reader. On - lite-07 SUBSCRIBE_DROP carries `Max Age`: the budget a staleness verdict - was judged under, encoded like `Publisher Max Age` (milliseconds plus one), - with `0` for a drop that does not depend on the budget: a cut, a group - never produced, or a group the sender evicted or expired with no copy it - could fetch (an origin, or a relay forwarding its upstream's eviction - drop), none of which depends on the reader's budget. A relay never sends - an eviction drop for its own eviction; it fills the group by range - instead. A widening never reverses a `0` drop. The relay records each run with that - exact budget. A B1 - drop then settles only readers no wider than B1, and the B2 re-drop - settles a B2 reader. The cost is one varint per drop, a few bytes on an - event that is already per group, which is worth an unambiguous term. - Rejected: an implicit generation counted by both sides per - SUBSCRIBE_UPDATE. It needs a generation-to-budget history at the relay - and agreement on every update counted, while the budget itself is - self-describing, order-free, and unaffected by updates that change - nothing. -- **lite-07 only for re-offering reported groups.** Released lite-05 and - lite-06 peers treat a flushed SUBSCRIBE_DROP as final, so a drop received - over those versions is final for every reader, and a lite-07 hop never - sends a group onto a lite-05 or lite-06 hop after reporting it. On - lite-07, a widening obliges the publisher to serve the newly covered groups - it holds and never delivered on this subscription, including ones it - reported dropped, and to drop again, under the new terms, the ones it - cannot serve. SUBSCRIBE_UPDATE stays incremental and never resets, since a - reset races in-flight, out-of-order data. A shrink sends no drops. -- **Draft wording.** `drafts/draft-lcurley-moq-lite.md` states, for lite-07's - restored SUBSCRIBE_DROP, that a drop holds only under the terms in effect - when it was sent, that a widening re-offers held groups and re-drops the - rest, and that a later group, when sent, wins over its drop. This quest - writes it, including the `Max Age` field above, which is free to add - because lite-07 is unpublished. +Decided 2026-11-03 (maintainer, /quest-unblock): `js/net` bounds run memory +by porting Rust's per-broadcast `cache::Pool` to JS. One shared byte account +per broadcast bills frames and drop-set runs (a fixed cost per run) under +Rust's default budget, with eviction and the fold-into-floor or +`TooFarBehind` behavior below, so Rust and JS mirror each other with no new +setting. Reason: the 2026-11-02 audit found the existing 32 MiB limit in +`js/net/src/group.ts` bounds only one group's frame tail, and a mock-time +probe retained 64, 256, and 1,024 disjoint OLD runs with zero cached groups; +a shared pool bounds drop-only tracks the same way in both languages. Not +chosen: a per-track run cap; upstream backpressure. The pool port itself is +[JS cache pool](/quest/m1/js-cache-pool.md), split out on 2026-11-03 since it +lands on its own; this quest bills the runs to it. -Code it replaces, in Rust and `js/net` alike: +Decided 2026-11-03 (maintainer, /quest-unblock), renamed and narrowed by the +`/quest-complete` decisions below: range terms carry an +explicit update id. Each lite-07 SUBSCRIBE and SUBSCRIBE_UPDATE carries an +increasing range-terms id, and a stale drop echoes the id it was judged +under. A relay treats a drop whose id has been superseded as unknown and +refills it by range, keeping no term history. Reason: a drop sent under an +old broad range can arrive after SUBSCRIBE_UPDATE narrows the aggregate at +the same budget (mock-time cursors over groups at 0, 1, and 2 seconds serve +group 2 under the open range and group 0 under a range capped before group +1), and `Max Age` alone cannot tell the two verdicts apart. Not chosen: a +conservative refill with bounded remembered ranges; treating every range +update as invalidating in-flight drops. -- `TrackState::drop_groups`, the `drops` range set and `DropCursor` in - `rs/moq-net/src/model/track.rs`, which record a whole range without its - reason or budget, even over groups already cached. -- In `js/net/src/track.ts`: `#drop`, `#undrop`, the `drops` signal, and - the wire cursor's `#pending` and `#dropsSeen`, whose `#tryRecvGroup` - closes any group already reported. +Decided 2026-11-03 (maintainer, /quest-unblock), with `Max Age` dropped, +`Terms` renamed `Update`, and SKIP narrowed to declared sequences by the +`/quest-complete` decisions below: lite-07 +splits its hole message by whether the group ever existed. `SUBSCRIBE_SKIP {Group Start, +Group End}` names sequences the publisher never produced, such as skipped +sequence numbers or ones past a cut: it has no error code, ends no reader, +and is final. `SUBSCRIBE_DROP {Group Start, Group End, Error Code, Max Age, +Terms}` names groups that existed but will not be delivered: an aborted or +reset group, eviction or expiry with no copy left, and stale (OLD). It ends +any held reader with its code; a stale drop carries the `Max Age` and the +range-terms id it was judged under, so a widening can re-offer it. lite-06 +and older keep today's SUBSCRIBE_DROP on the wire. The split runs through +moq-net's Rust and JS model and API, whose drop/skip naming follows it, and +through the draft. Reason: a hole that never existed has no reader to end +and no budget to qualify, so one message conflated a final, code-free +verdict with a reversible, reader-ending one. Not chosen: stale in SKIP; a +single message with a reason enum; renaming only. -Untrusted input: a hostile upstream can fragment the set by alternating -drops and arrivals, and an open live range is no bound, since its groups are -evicted while their runs would stay. Bill each run to the cache's existing -byte accounting like a group: the Rust `Pool` through a `Charge` at a fixed -per-run cost. Under pressure the pool evicts the oldest runs as it evicts -groups, and a run older than the cache's retention window is forgotten, so -the set never outgrows what the pool would hold, with no new constant. An -evicted run's sequences become unknown again. `js/net` has no byte pool and -gets no new cap: its shared and per-cursor runs expire with the same idle -window as the groups they sit between, and expiry applies the same -fold-or-`TooFarBehind` rule as a Rust eviction. A run sits between cached -sequences, so there are at most about one per cached sequence (alternating -drop and arrival), and JS run state grows no faster than the JS cache -itself. The JS byte bound for both is the 32 MiB group tail that -[moq-dev/moq#7411](https://github.com/moq-dev/moq/pull/7411) landed. +Decided 2026-11-03 (maintainer, /quest-complete), changing the design +[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) first shipped: -Only a final run folds into a cursor's floor: a delivered run, a run -skipped at the track's max age bound (no widening reaches past it), or a -budget-independent drop (a cut, a group never produced, an upstream or -origin eviction or expiry, or any drop reported over lite-05 or lite-06). A stale run reported on lite-07 within reach of the -max age bound is budget-qualified, so it is treated like an unknown hole: -the floor never rises over it, and if the pool must evict it or a run above -it, the cursor fails with `TooFarBehind`. Folding it would forgo the -re-offer this plan promises after a widening. A later group wins over its -drop only while the cursor's record still holds that drop: a drop folded -into the floor under pool pressure is final for that cursor, so its group -arriving late is not delivered there. +- **`Terms` is renamed `Update`** on lite-07 SUBSCRIBE, SUBSCRIBE_UPDATE and + SUBSCRIBE_DROP: the number of the SUBSCRIBE or SUBSCRIBE_UPDATE in effect, + in the Rust and JS code too. +- **SUBSCRIBE_DROP drops `Max Age`:** it is {Group Start, Group End, Error + Code, Update}. The budget a stale (OLD) drop was judged under comes from + the subscriber's own record of what it sent with that Update, and a drop + whose Update is superseded is no verdict. Any SUBSCRIBE_UPDATE reopens every + outstanding OLD drop in the subscriber's tail accounting; the publisher + re-offers what the new terms cover and drops again, under the new Update, + what is still stale. This replaces reopening only when the max delay widens + past the drop's. A non-OLD drop is final, whatever its Update. +- **`track::Producer::drop_groups` is crate-private.** `skip_groups` stays + public (see the next decision). -A cursor judges an unknown missing sequence (never a stale run its record -still holds as reported, which only a widening settles) as it would a -group never received: locally against its own budget (`TrackState::drifted`), -and otherwise as a cache miss, which the relay fills upstream by range -([Subscribe ranges](/quest/m2/subscribe-ranges/README.md)), so no reader -waits indefinitely on forgotten state. Extend `track_subscriber_churn` in -`rs/moq-net/benches/track.rs` with a fragmented set up to pool capacity, -swept over subscriber count, shared runs, runs per cursor record, and run -span, so enumeration cost tracks the touched runs, never the set, the -record, or a run's span. +Decided 2026-11-03 (maintainer, /quest-complete), replacing the "eager SKIP" +decision above and the PR's `## Waiting` item on out-of-order creation: -Test on mock time through a relay: -- On a live track, a reader no wider than the upstream budget gets a prompt - drop for an upstream OLD naming a group the relay never cached. -- On lite-07, a widened reader gets that group re-offered. A drop already - flushed before the widening SUBSCRIBE_UPDATE was received, so the reader - then gets the re-offer and no further final drop; a drop still in flight - does not settle the widened reader. Check what each reader receives, not - only tail accounting. -- On lite-07, after a widening from B1 to B2 for a group the upstream no - longer holds, an in-flight drop tagged B1 settles only the B1 reader, and - the re-drop tagged B2 settles the B2 reader. -- A group arriving after its drop, while the cursor's record still holds - the drop, is served, and a drop arriving after its group leaves it - served. -- Through a lite-06 hop, a reported group stays dropped for every reader, - with no session error. -- A shrink sends no drops. -- Rust and `js/net` each encode and decode an origin's eviction drop and - retention-expiry drop with `Max Age` `0`, across languages; a relay - forwards them unchanged, and a widening re-offers neither. -- A group a relay evicted while the upstream still holds it is never - reported dropped: a new reader, a widened reader, and a reader already - owed it each fill it by range and receive it. -- A reader whose range is narrower than the relay's aggregate range is not - settled by a received stale drop: it fills G by range and gets G or a - drop judged under its own range, while a reader covering the aggregate - range is settled at once. -- On lite-07, a cursor that delivered F, reported its neighbour G stale - while G stayed cached, and delivered H, gets only G after a widening that - covers all three: G is re-offered and F and H are never resent. -- Under pool pressure, a cursor's delivered runs and budget-independent - drops fold into the floor, while evicting a lite-07 stale-reported run - within reach, or a run above it, ends the cursor with `TooFarBehind` - instead of losing the re-offer. Through a lite-06 hop the reported run - folds. -- A flood of fragmented runs stays within the pool in Rust, evicting the - oldest shared runs first, and a sequence whose shared run was evicted is - judged locally or filled by range, never waited on indefinitely. In - `js/net` the same flood holds at most one run more than its cached - sequences, and the runs expire with the idle window. -- A single run spanning billions of sequences is settled by each cursor in - one step, whether dropped, skipped by a wider reader, or evicted and - judged again. -- Per-reader misses are bounded the same way. [Dynamic range - requests](https://github.com/moq-dev/moq/pull/6171) adds a third unbounded - record: each reader's given-up misses, one per gap, which a reader whose - floor never moves cannot forget. Fold them into the cursor's runs, billed - to the pool (from - [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), decided - 2026-10-23). +- ✅ **SKIP comes only from an explicit, public `track::Producer::skip_groups`**, + and from `append_group` (below). Reason: producers legitimately create + groups out of order (`claim_sequence` re-creating an aborted sequence, + moq-mux and moq-c `seek`, `finish()`'s documented "older groups can still + arrive"), so a gap tells moq-net nothing; only the producer knows a + sequence will never exist. moq-archive's replay is the in-tree consumer of + `skip_groups`, for the sequences up to a recorded end it holds nothing + for; a producer that numbers sparsely would call it too. moq-mux's live + encoders use `append_group`. The relay records a received SKIP with + `skip_groups` (below). +- ✅ **Both inferences are removed:** the eager skip of the gap in + `create_group`, and the automatic SKIP at finish for never-arrived + sequences below the final one (`TrackState::missed` and the publisher's + `drop_unserved`). An undeclared gap is judged stale against the reader's + budget, or named with DROP NOT_FOUND, as before this PR. +- ✅ **SKIP is final:** creating a group (or appending a group or datagram) + at a skipped sequence is refused with `Error::Skipped` (JS `Skipped`). + Re-creating a dropped or aborted sequence still works. +- ✅ **SKIP replaces every drop, final or stale** (maintainer, 2026-11-03, + `/quest-complete`, replacing the 2026-11-04 `/quest-iterate` rule that a + SKIP replaced only a stale OLD drop). "Never produced" is the most + authoritative fact about a sequence, so a received SKIP, `skip_groups`, or + `append_group`'s implicit skip overwrites an earlier DROP of any code (OLD, + aborted, evicted, NOT_FOUND), and a later DROP leaves a skip in place. + Readers that drop already ended stay ended, and nothing is re-delivered; the + run's reason becomes Skip, so every later reader, relay and downstream + report sees SKIP, and a relay forwards the SKIP even over a DROP it already + forwarded (`DropCursor::reported_skip`, `TrackRun::dropped`, and their JS + mirrors). A cached group is never overwritten: a received SKIP naming one + leaves the group, logs a protocol error, and keeps the session, since the + draft requires no more. A received SKIP or final DROP also takes its groups + out of the subscriber's stale record, so the next update doesn't owe them + again. Not chosen: SKIP replacing only a stale OLD drop. +- ✅ **A group or final drop replaces a lite-07 stale drop already sent** + (agent decision from review, 2026-11-04, `/quest-iterate`; the maintainer + chose to keep it and retire the stale report at the 2026-11-03 round-4 + checkpoint, `/quest-complete`). A front that switches routes can hand out + the new route's group, or its final verdict, for a sequence the old route + reported stale, before the update that reopens it, so the publisher serves + the group, or sends the final drop, over the OLD it sent. Its own expired + streams' OLD drops count the same. Whatever replaces a stale report + retires it: a group handed out, a SKIP, or a final drop takes the sequence + out of the reader's stale record (`DropCursor::reported_stale`, the + resumed `Reader::reported_stale`, JS `#reportedStale`) and the publisher's + (`TrackRun::dropped_stale`, `expired`), so the next SUBSCRIBE_UPDATE + neither sends OLD for it again nor names it at the end. On the receiving + side a Group Stream wins over a stale drop of its sequence in either order: + an OLD that lands after the stream leaves the group, still arriving or + not, or finished under the same update and since evicted (agent decision + 26, 2026-11-05, for review), and an update owes again only the stale runs + the track still holds. + The end sweep's NOT_FOUND never replaces a stale drop, so an update can + still reach an ended track's history. Not chosen: reverting the + replacement; merging with the stale report left in place. +- ✅ **`append_group` is an implicit skip** (refinement, same session): it + means "the next group", so the producer declares it won't go back, and it + skips every sequence below it that it never produced, from just past the + previous append (before the first append, from the lowest group created), + as if `skip_groups` were called on them: a drop there becomes a skip, and + they are then final. Today + `append_group` leaves no gap of its own, so this only fires on holes earlier + `create_group` calls left below the head; under time addressing it names + the span between consecutive starts. `create_group` stays non-committal, + so out-of-order creation (`claim_sequence` retries, `seek`) keeps working. + There is still no inference at finish. +- ✅ **`skip_groups` stays public** and is documented as the explicit source + of SKIP; `drop_groups` stays crate-private. Rust and JS mirror each other + (`skipGroups`, `appendGroup`). +- ✅ **Draft:** SKIP names only sequences the publisher declared it will never + produce, and it is final. +- ✅ **Run-length tracking** (refinement, same session): skipped and dropped + groups are kept as runs, never one entry per sequence, in Rust and JS: the + track's hole set, the implicit `append_group` skip, `skip_groups`, and what + the publisher reports. A huge `skip_groups` or append gap costs one run and + one SUBSCRIBE_SKIP (`a_huge_skip_costs_one_run`, + `lite07_skips_a_huge_run_at_once`, and their JS mirrors). +- ✅ **`create_group` refuses a skipped sequence and allows a dropped one** + (refinement, same session): a group stream wins over its drop, which fills + answering a miss and `claim_sequence` retries of an aborted sequence need. +- ✅ **No `create_group` gap error, for now** (maintainer, same session): + gaps stay legal, and `append_group` or `skip_groups` declares them. No + principled bound exists in the model: the cache and the budget bound bytes + and time, not sequence counts, a gap costs one run whatever its length, + and under time addressing every group leaves a gap between starts. Not + chosen: a debug log on a gap; a configurable size limit (an arbitrary + knob). +- Not chosen: eager skip on `create_group` with a late group winning over + it (the shipped design, which inferred SKIP from a gap that out-of-order + creation makes meaningless); refusing every `create_group` below the last + group (broke `claim_sequence`, `seek` and 85 moq-net tests); keeping the + finish-time inference for a local producer (a gap at finish is no more a + declaration than one mid-stream). -- A group the publisher cut while a fetched copy still serves its end is - delivered and never also reported dropped. Today `DropCursor::cut` - (`rs/moq-net/src/model/track.rs`) reports any visible slot's - `cut_error()`, so the arrival walk can hand out the fetched copy and report - the group dropped too; `Slot::cut()` already returns nothing while a copy - serves the end. Found by - [moq-dev/moq#7053](https://github.com/moq-dev/moq/pull/7053) (added - 2026-10-28). +Decided 2026-11-05 (maintainer, /quest-complete): a duplicate Group Stream +waits until the first copy settles. On lite-07 the OLD drop for an expired +group, and the stream the next update re-offers for it, can both reach the +subscriber before the original stream's reset, which used to refuse the +re-offer as a duplicate and lose the group once the reset landed. The +subscriber now holds a stream whose sequence another copy still arriving +holds, unread, and discards it if that copy finishes or takes the group if it +fails (Rust `GroupRecv`'s `Hold` wait over `track::Received::Held`, JS +`runGroup` over `receiveGroup`'s `arriving`). The held stream keeps the +lowest receive priority, so it holds at most its stream credit and is the +first a full receive window evicts. A reader already handed the failed copy +sees it fail, since no sequence is handed out twice; the cache, fills, and +readers that had not reached it get the re-offer. Not chosen: letting the +duplicate replace the copy still arriving; accepting the loss. Mirror each case in `js/net`. In `js/net`, a group a sink buffers keeps its source wanted only while @@ -263,58 +205,155 @@ In `js/net`, a group a sink buffers keeps its source wanted only while ([moq-dev/moq#7948](https://github.com/moq-dev/moq/pull/7948)), so a change to how a cursor judges a missing or stale group keeps that verdict in step. -Public API: none. Wire: lite-07 SUBSCRIBE_DROP (unpublished) gains `Max Age` -and the widening and ordering rules above; published versions are -unchanged. -- A local producer's declared end must settle over dropped groups, as the - doc on `TrackState::drop_groups` already claims (added 2026-10-25): let - `reaches(fin)` count a live edge whose gap `edge..fin` is wholly dropped, - then delete moq-archive's fake empty group at `fin - 1` - (`rs/moq-archive/src/consumer/track.rs`). +Decided 2026-11-05 (maintainer, /quest-plan): the agent decisions +[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) listed for +review stand as recorded, numbered as in its body: -### CPU per drop (2026-10-24 pre-cut audit) +- ✅ **14:** a Group Stream or final drop the publisher's reader reaches + replaces a stale drop it already sent (above). +- ✅ **16:** SUBSCRIBE_UPDATE carries `Update` after the Range edit and before + `Wanted Group`; SUBSCRIBE_SKIP keeps 0x3 beside SUBSCRIBE_STARTED at 0x4. +- ✅ **17:** an OLD drop records no stale run on groups a range fill asked + for, which recovery asks for again. Every drop the subscriber accounts, + and every SKIP, answers the fills owing those groups; an OLD drop under a + superseded `Update` answers none. +- ✅ **18:** the JS covered-FIN reopen loop and its stale-run supersede go, + as main deleted the loop. +- ✅ **19:** a final OLD drop, such as one relayed from lite-06, goes out on + lite-07 as the stale drop its code names, recorded and dropped again on + each update. +- ✅ **20:** an update judges again the re-offers still waiting for stream + credit, and a verdict sent for a queued re-offer keeps it from opening. +- ✅ **21:** a SUBSCRIBE_DROP naming an `Update` later than the last one sent + is a PROTOCOL_VIOLATION. +- ✅ **22:** a received stale run is judged again for a reader whose terms + changed, so a widened budget asks a fill for it. +- ✅ **23:** recording a stale verdict answers no outstanding fill (JS, as + Rust), `skipGroups` checks its bounds, and a fill cannot cache a skipped + sequence. +- ✅ **25:** a relay cursor's stale report that an upstream's received stale + run decided stays the upstream's to judge after an update + (`DropCursor::reported_received`, JS `#reportedReceived`). -Besides memory, run fragmentation costs CPU today, new on main: every -SUBSCRIBE_DROP bumps `drops_revision`, and each reader's `DropCursor::next` -rescans the overlapping runs (`rs/moq-net/src/model/track.rs` about lines -5802-5870), so N fragmented runs and R downstream readers cost O(R*N) per -6-byte DROP. JS `insertDrop` (`js/net/src/internal.ts:136`) copies the whole -run array once for the track and once per sink (`track.ts:914`). The -replacement keeps the per-cursor walk incremental in both languages, with a -benchmark swept over runs and readers. +Decided 2026-11-06 (maintainer, /quest-complete), replacing agent decision 27 +of [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), which asked a +fill for every unknown stale run instead: -### Idle START unreport test +- ✅ **27: a stale verdict accepts every range.** A relay keeps every group + its current upstream terms' ranges name (`TrackState::judged`, JS + `judged`), taking a bound only the publisher resolves at its widest. A + received stale run settles a reader no wider than the budget it was judged + under that reads every one of those groups above the run's start, however + many ranges they span, since a group's verdict looks only above it. A + reader with the upstream's own ranges is settled with nothing received. A + reader narrower than the upstream's ranges still treats the run as unknown, + and asks a fill for it even above the newest group received, since its + ranges add nothing to the upstream's terms and no update brings the run + back. Not chosen: asking a fill for every unknown stale run without the + multi-range verdict; reversing the fix and tracking the hang as a + follow-up. +- ✅ **28 (2026-11-06, maintainer, /quest-complete): keep the fill for + readers narrower than the verdict's ranges**, with no fill-handler gate. A + verdict judged on the union of readers' ranges is valid only for readers + that read every group it names above the run's start; a narrower reader + still needs groups the upstream already dropped, so only a fill can + deliver them. Not chosen: re-judging per reader (it would still need the + fill); separate upstream subscriptions per reader shape (they give up + aggregation); deferring the narrow case as a follow-up. -[moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717) made the JS lite -publisher unreport stale skips below an idle SUBSCRIBE_START, so a later -widening can re-offer them as Rust does, but landed it without a test: the -case needs an update to arrive between START and the first group. Cover it -deterministically, through the track hooks rather than wire timing, when the -drop runs are reworked here. +Decided 2026-10-27 (maintainer): this runs after the range count cost work, +which finished in [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542), +and before [Rust groups by start](/quest/m2/time-addressing/model.md), which +requires it because runs keyed by group sequence convert mechanically to +starts. + +- **One disposition record per cursor,** a run-length map whose runs are + delivered, skipped as stale, or reported dropped or skipped, with the + reason and, for a stale run, the budget it was judged under. It replaces + `Skipped` (`rs/moq-net/src/model/skipped.rs`), `Delivered` + (`rs/moq-net/src/model/delivered.rs`, whose floor folds stale gaps in with + delivered groups), `DropCursor`'s `reported` and `reported_stale`, + `TrackRun::served` in `rs/moq-net/src/lite/publisher.rs`, and the JS + wire cursor's `#reported` and `#reportedStale`, `#skipped`, and the + publisher's `served`. Delivery is recorded, not inferred from the cursor + position, since an arrival-order or descending cursor delivers out of + sequence order. +- **Billed to the pool, never guessed.** Each track run and each cursor run + is billed through a `Charge` at a fixed per-run cost (in JS, through the + [JS cache pool](/quest/m1/js-cache-pool.md)). Under pressure the pool + evicts the oldest track runs as it evicts groups, and a run older than the + cache's retention window is forgotten. An evicted track run's sequences are + unknown again: a reader judges them by `drifted` or fills them by range. + A relay's own eviction is never a run. +- **The floor rises only over a final prefix:** the runs from the floor up to + the lowest run that is not final. A final run is a delivered one, one + skipped at the track's max age bound, a skip, a budget-independent drop, + or any drop reported over lite-05 or lite-06. A lite-07 stale report within + reach of the max age bound, or an unknown hole, is not: when the pool must + evict it or a run above it, the cursor fails with `TooFarBehind` rather + than forgo the re-offer. A drop folded into the floor is final for that + cursor, so its group arriving late is not delivered there. +- **Per-reader misses** fold into the cursor's runs too: the given-up misses + `rs/moq-net/src/model/track/miss.rs` keeps per gap (from + [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), decided + 2026-10-23). +- **CPU per drop.** Every SUBSCRIBE_DROP bumps `drops_revision`, and each + reader's `DropCursor::next` rescans the overlapping runs, so N fragmented + runs and R readers cost O(R*N) per 6-byte drop; JS `insertDrop` + (`js/net/src/internal.ts`) copies the whole run array once for the track + and once per sink. Keep the per-cursor walk incremental in both languages. + Extend `track_subscriber_churn` in `rs/moq-net/benches/track.rs` with a + fragmented set up to pool capacity, swept over subscriber count, shared + runs, runs per cursor record, and run span. + +Still to cover, beyond the memory work: -### A covered cut is still reported +- **Idle START unreport test.** [moq-dev/moq#6717](https://github.com/moq-dev/moq/pull/6717) + made the JS lite publisher unreport stale skips below an idle + SUBSCRIBE_START, without a test: the case needs an update between START + and the first group. Cover it deterministically through the track hooks. +- **Cancelled fetches hold the end.** Found 2026-10-28 by + [moq-dev/moq#7149](https://github.com/moq-dev/moq/pull/7149): a cancelled + fetch below a track's end holds the end open until the cache expiry scan + removes it. Settle it with a run reason. +- **Untimed fills below a resolved start.** Noted 2026-10-31 from + [moq-dev/moq#7235](https://github.com/moq-dev/moq/pull/7235): a lite-07 + copy with no media time asks a fill below the start even for a reader whose + budget the publisher already exceeded. If the extra FETCH shows up in relay + traffic, bound it with a run below the start carrying its budget. +- **A fill's stale verdict is final for everyone.** `TrackState::drop_unfed` + (`rs/moq-net/src/model/track/fill.rs`) records what a fill's source gave up + as a budget-independent drop, including OLD judged under that fill's own + terms. Record it as a stale run, so it settles only the readers it covers. -`DropCursor::cut` reports a group as cut even when a fetched copy covers it, -noted in [moq-dev/moq#7053](https://github.com/moq-dev/moq/pull/7053) and again -by [moq-dev/moq#7149](https://github.com/moq-dev/moq/pull/7149), which made a -fetched copy settle such a group. Decided 2026-10-29 (maintainer): settle it -here, where a cut run gets a reason and a covering copy can clear it. +Tests, on mock time through a relay, in Rust and mirrored in `js/net`: -### Cancelled fetches hold the end +- After a widening from B1 to B2 for a group the upstream no longer holds, an + in-flight drop under the B1 update settles no reader once the B2 update is + sent, and the re-drop under the B2 update settles the B2 reader, over the + wire. +- A group a relay evicted while the upstream still holds it is never reported + dropped: a new, a widened, and an owed reader each fill it by range. +- Rust and `js/net` encode and decode an origin's eviction and retention + expiry drop (EVICTED, not OLD) across languages; a relay forwards them + unchanged, and no update re-offers either. +- Through a lite-06 hop, a reported group stays dropped for every reader, + with no session error; a shrink sends no drops. +- Under pool pressure, delivered runs and budget-independent drops fold into + the floor, while evicting a lite-07 stale report within reach, or a run + above it, ends the cursor with `TooFarBehind`. +- A flood of fragmented runs, including a JS drop-only flood with no cached + groups, stays within the pool, evicting the oldest shared runs first, and + a sequence whose run was evicted is judged locally or filled by range. +- A single run spanning billions of sequences is settled by each cursor in + one step, whether dropped, skipped by a wider reader, or evicted and judged + again. -Found 2026-10-28 by [moq-dev/moq#7149](https://github.com/moq-dev/moq/pull/7149): -a cancelled fetch below a track's end still holds the end open until the cache -expiry scan removes it, so how fast a subscription ends depends on expiry -timing. Settle it here, where cancelled and expired runs get a reason. +Public API: none. Wire: none. -### Untimed fills below a resolved start +## Required -Noted 2026-10-31 from [moq-dev/moq#7235](https://github.com/moq-dev/moq/pull/7235), -which made everything below a resolved SUBSCRIBE_START fillable from upstream: -a lite-07 copy with no media time asks a fill below the start even for a reader -whose budget the publisher already exceeded, and that fill ends empty. If the -extra FETCH shows up in relay traffic, bound it here, where a run below the -start can carry the budget it was resolved under. +- [JS cache pool](/quest/m1/js-cache-pool.md) - the byte account the JS runs bill ## Related diff --git a/quest/m1/js-cache-pool.md b/quest/m1/js-cache-pool.md new file mode 100644 index 00000000000..b9bab0d69c1 --- /dev/null +++ b/quest/m1/js-cache-pool.md @@ -0,0 +1,52 @@ +# [L] JS cache pool + +## Goal + +`js/net` bounds what a broadcast caches in bytes, as Rust does: a port of +Rust's per-broadcast `cache::Pool` (`rs/moq-net/src/model/cache.rs`) bills +every cached group's frames and fixed bookkeeping, and the drop set's track +and cursor runs at a fixed cost per run, and evicts under it the way Rust +does. Rust and JS mirror each other under Rust's default budget, with no new +setting. + +## Plan + +Decided 2026-11-03 (maintainer, /quest-unblock, recorded in +[Drop set](/quest/m1/drop-set.md)): port Rust's per-broadcast `cache::Pool` +to JS. Shared byte accounting per broadcast bills frames and drop-set runs (a +fixed cost per run), under Rust's default budget, with eviction and the +fold-into-floor or `TooFarBehind` behavior the drop set plans. Not chosen: a +per-track run cap; upstream backpressure. Split out of the drop set on +2026-11-03, since the pool lands on its own: its first consumer is the group +cache, and the drop set's run billing follows on top of it. + +What JS has today: `js/net/src/group.ts` caps each group's frame tail at +`TAIL_BYTES` (32 MiB) and evicts that group's oldest frames past it, while +the track cache (`#prune` in `js/net/src/track.ts`) is purely time-based on +`CACHE_EXPIRY` (30 s). Nothing bills bytes across groups or tracks, and +`TrackState.drops` and each cursor's runs grow without bound under a peer +that fragments them. + +Port, mirroring Rust's names and shape: + +- A pool per broadcast, defaulting as `broadcast::Info::default` does in + Rust: no byte capacity and `DEFAULT_EXPIRY` (30 s) idle expiry. +- A per-track account and a per-group charge billing payload plus a fixed + per-group overhead, and a fixed per-run cost for drop-set runs. +- Write-time eviction debt: a track over capacity pays by evicting its own + oldest groups with EVICTED, the latest live group protected, and the + access-time mean ordering eviction across tracks. +- The idle-expiry sweep on the pool's clock, replacing `#prune`'s own timer. + +Test on mock time: a capacity bounds cached groups across tracks of one +broadcast, evicting the oldest first and sparing a fresher track; idle groups +expire; a drop-only flood with no cached groups stays within the pool once +[Drop set](/quest/m1/drop-set.md) bills its runs; the per-group frame-tail +limit still applies. + +Public API: none beyond what Rust already exposes, mirrored if a JS consumer +needs it. Wire: none. + +## Related + +- [Drop set](/quest/m1/drop-set.md) - bills its track and cursor runs to this pool diff --git a/quest/m2/README.md b/quest/m2/README.md index ad168b11ccb..881c5ec714d 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -172,7 +172,7 @@ quests that require them. ## Required -- [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 (held in m1 by an open claim) +- [Drop set](/quest/m1/drop-set.md) - the drop set's track and cursor runs are billed to the cache pool, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` (held in m1 by an open claim) - [Subscribe ranges](/quest/m2/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH for multi-group reads (single-group FETCH stays); merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [Address groups by timestamp](/quest/m2/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate (held in m1 by an open claim) diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index e1064d24f5c..ed267863216 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -65,8 +65,12 @@ The cut: treat a longer SETUP as a protocol violation" stays. The lite-07 changelog names that rejection, not only the cap. -Decided 2026-10-20 (maintainer): [Drop set](/quest/m1/drop-set.md) joins -Required, because it adds `Max Age` to lite-07's SUBSCRIBE_DROP. Nothing +Decided 2026-10-20 (maintainer): the drop set joined Required, because it +changes lite-07's SUBSCRIBE_DROP. That wire work, SUBSCRIBE_SKIP and the +`Update` number on SUBSCRIBE, SUBSCRIBE_UPDATE, and SUBSCRIBE_DROP, landed in +[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670); what remains of +the drop set is memory accounting with no wire change, so it no longer gates +the cut. Nothing else from the auth line gates the cut: its lite-07 wire work, the SETUP Token parameter, has landed, and the rest is API work that does not block the rename. @@ -110,7 +114,6 @@ timestamp bound in [#6168](https://github.com/moq-dev/moq/pull/6168). ## Required - [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - lite-07 carries the route layer: ROUTE per origin node and path-less ANNOUNCE, with the hop list gone -- [Drop set](/quest/m1/drop-set.md) - lite-07's SUBSCRIBE_DROP carries `Max Age`, so a widening re-offers groups reported dropped - [Lite-07 FETCH is single-group only](/quest/m2/subscribe-ranges/lite07-fetch.md) - lite-07 FETCH reads one group or a slice of one, and SUBSCRIBE ranges of whole groups - [Lite-07 by start](/quest/m2/time-addressing/lite07.md) - lite-07 carries group starts, previous-start links, GROUP_END, and time bounds instead of sequence numbers - [@moq/net by start](/quest/m2/time-addressing/js.md) - JS speaks the same lite-07 wire diff --git a/quest/m2/subscribe-ranges/README.md b/quest/m2/subscribe-ranges/README.md index 05c06949bc1..2b7d9e6cf75 100644 --- a/quest/m2/subscribe-ranges/README.md +++ b/quest/m2/subscribe-ranges/README.md @@ -6,7 +6,8 @@ A moq-lite-07 SUBSCRIBE replaces FETCH for anything wider than one group. One su of group ranges, past or live, bounded by group or by time, in ascending or descending order, and the publisher serves their union from one cursor without sending a group twice. -Every sequence in range either arrives or is named by a SUBSCRIBE_DROP, so +Every sequence in range either arrives or is named by a SUBSCRIBE_SKIP or +SUBSCRIBE_DROP, so nothing blocks on a cache miss. Relays fill misses upstream with ranges, not one request per group, including over moq-transport where group numbers may be sparse. Lite-07 FETCH is single-group only: one group, or a slice within one @@ -30,9 +31,9 @@ relay bounds a sparse FETCH only if the model can request ranges: not accidentally give it the live SUBSCRIBE zero-delay default. SUBSCRIBE_UPDATE replaces the list. Lite-07 FETCH is single-group only (decided 2026-11-03, replacing its removal); published versions keep it, answered through the same model. -- **Holes.** SUBSCRIBE_DROP (restored in lite-07) names every sequence in the - requested ranges the publisher won't deliver: never existed, expired, or - refused upstream. On lite-01..06 datagram groups are always dropped in a past range (#4982); on lite-07 ranges never cover datagrams, which have their own track kind. The +- **Holes.** Lite-07 names every sequence in the requested ranges the + publisher won't deliver: SUBSCRIBE_SKIP one that never existed, and + SUBSCRIBE_DROP (restored in lite-07) one expired or refused upstream. On lite-01..06 datagram groups are always dropped in a past range (#4982); on lite-07 ranges never cover datagrams, which have their own track kind. The publisher delivers what it has and fills the rest as upstream answers. - **Model.** `Subscription` gains `ranges` beside its `order`, mirroring the wire; `request_groups` as a separate API was rejected. A publisher's diff --git a/quest/m2/time-addressing/ietf.md b/quest/m2/time-addressing/ietf.md index 39037488e57..4f018b97431 100644 --- a/quest/m2/time-addressing/ietf.md +++ b/quest/m2/time-addressing/ietf.md @@ -26,9 +26,9 @@ TIMESCALE (0x08): moq-lite viewer therefore arrives without a frame PTS, which is accepted. Today's TIMESTAMP code (`ietf/group.rs`, `ietf/object.rs`, `ietf/max_age.rs`) is deleted or reduced to that pass-through. -- **Prior Group ID Gap stays** as a best-effort SUBSCRIBE_DROP signal: +- **Prior Group ID Gap stays** as a best-effort SUBSCRIBE_SKIP signal: egress sends the gap to the previous key, and ingest reads it as the - skipped IDs being dropped. A peer that sends none still works. + skipped IDs never produced. A peer that sends none still works. - **GROUP_END maps to END_OF_GROUP.** The supported drafts refuse properties on a status object (`check_status` in `rs/moq-net/src/ietf/object.rs`), so END_OF_GROUP stays bare and the end time is lost on this wire. @@ -39,7 +39,8 @@ Decision prompts: - Timestamp property: ✅ stop, with LOC proxied only between moq-transport peers; keep reading but stop sending; keep the 2026-10-25 plan. - Prior Group ID Gap: drop it; ✅ keep it as a best-effort SUBSCRIBE_DROP - signal. + signal (a SUBSCRIBE_SKIP signal since lite-07 split never-produced holes + out, 2026-11-03). Test both directions over drafts 18, 20, and 22: IDs round-trip exactly through a relay, a gap reaches the subscriber as a drop, and LOC properties diff --git a/quest/m2/time-addressing/lite07.md b/quest/m2/time-addressing/lite07.md index 327d45e86d3..3abd721ea5d 100644 --- a/quest/m2/time-addressing/lite07.md +++ b/quest/m2/time-addressing/lite07.md @@ -20,8 +20,8 @@ Decided 2026-10-25 (maintainer), from the spike: instead of resetting it, which changes the draft's reset rule for those groups (around line 760). - SUBSCRIBE and SUBSCRIBE_UPDATE bounds, SUBSCRIBE_OK's largest and its Range - Starts, SUBSCRIBE_STARTED, SUBSCRIBE_FINISH, and SUBSCRIBE_DROP address by - start. Datagrams keep their own sequence (maintainer, 2026-11-01). Drops + Starts, SUBSCRIBE_STARTED, SUBSCRIBE_FINISH, SUBSCRIBE_SKIP, and SUBSCRIBE_DROP + address by start. Datagrams keep their own sequence (maintainer, 2026-11-01). Drops name time spans with no group to deliver, not sequence ranges. `Relative(n)` stays a group count. - Positions stay (start, frame index), preserving transport frame offsets. @@ -55,7 +55,8 @@ SUBSCRIBE anchors). Public API: none beyond [the model](/quest/m2/time-addressing/model.md). Wire: lite-07-wip GROUP, GROUP_END, SUBSCRIBE, SUBSCRIBE_UPDATE, -SUBSCRIBE_OK, SUBSCRIBE_STARTED, SUBSCRIBE_FINISH, and SUBSCRIBE_DROP change. +SUBSCRIBE_OK, SUBSCRIBE_STARTED, SUBSCRIBE_FINISH, SUBSCRIBE_SKIP, and +SUBSCRIBE_DROP change. ## Required diff --git a/quest/m2/time-addressing/model.md b/quest/m2/time-addressing/model.md index 7461790708b..a1ecaab2f5a 100644 --- a/quest/m2/time-addressing/model.md +++ b/quest/m2/time-addressing/model.md @@ -64,13 +64,26 @@ a producer would need per-track memory of its last number. one FETCH per tick in the hole. Accepted: "This was a lingering problem. The entire point of (multiple) ranges is to avoid this issue." +- **Skips are declared, never inferred** (decided 2026-11-03, maintainer, + /quest-complete, replacing that day's "eager skips"; + [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670)). A gap tells + moq-net nothing, since groups may be created out of order. SKIP comes only + from the producer: an explicit `skip_groups`, or `append_group`, which + means "the next group" and so skips every key below it that never arrived + and was not given up, since the producer's last append. `create_group` + implies no skip. A skip is final: creating a group at a skipped key is + refused. Today `append_group` leaves no gap of its own, so its skip only + names holes earlier `create_group` calls left. `append_group(timestamp)` + keeps the rule, and there it names the span between consecutive starts: + every key between the previous group's start and the new one. + Every current role of the sequence needs a replacement or a reason to drop it: - The cache key, the duplicate check, the hand-out-once record (`delivered.rs`), and slot stamps. - Successor, reach, expiry, the arrival ledger, tail accounting, fills and - misses, and SUBSCRIBE_DROP accounting. Successor and reach already scan the + misses, and SUBSCRIBE_SKIP and SUBSCRIBE_DROP accounting. Successor and reach already scan the cache by range, so they tolerate sparse keys; `delivered.rs` runs, fills, misses, and `subscription/window.rs` treat `sequence + 1` as the next group and must follow previous-start links instead. @@ -79,7 +92,8 @@ it: refresh. - Not datagrams: they are their own track kind with their own sequence. - The legacy codecs: lite GROUP, SUBSCRIBE and SUBSCRIBE_UPDATE bounds, - SUBSCRIBE_START and its largest, SUBSCRIBE_END, SUBSCRIBE_DROP, FETCH, and + SUBSCRIBE_START and its largest, SUBSCRIBE_END, SUBSCRIBE_SKIP, + SUBSCRIBE_DROP, FETCH, and every IETF Location, Filter, FETCH, Largest, and NEW_GROUP_REQUEST. ### Timestamps are required and legacy IDs pass through diff --git a/quest/m3/README.md b/quest/m3/README.md index 80980f02173..72a9106befe 100644 --- a/quest/m3/README.md +++ b/quest/m3/README.md @@ -142,7 +142,7 @@ federation) land. - [moq-hls exposes each rendition's bound source](/quest/m3/hls-rendition-source.md) - playlists and segments report the source path and instance they read, so a cache can key on the source - [The memory bench finishes at 1 MiB frames](/quest/m3/memory-bench-hang.md) - the allocation mode's park at `size=1048576` is root-caused and fixed - [Audio's Latest start reaches back](/quest/m3/audio-latest-reach-back.md) - moq_audio's start covers min plus warmup like other consumers -- [moq-lite tails count drops](/quest/m3/lite-tail-drop-count.md) - SUBSCRIBE_DROP accounting replaces the session-tail grace from lite-05 +- [moq-lite tails count drops](/quest/m3/lite-tail-drop-count.md) - SUBSCRIBE_SKIP and SUBSCRIBE_DROP accounting replaces the session-tail grace from lite-05 - [Audio track changes are gapless](/quest/m3/gapless-audio-switch.md) - no silence beyond a frame on audio format or rendition changes, enforced by media QA - [We own our WebRTC stack](/quest/m3/webrtc/README.md) - one moq-webrtc crate holds the gateway, its ICE module, the data-channel transport, and a vendored, trimmed str0m/dimpl/is/sctp-proto - [PUBLISH_DONE says why](/quest/m3/publish-done-codes.md) - GOING_AWAY, TOO_FAR_BEHIND, and EXPIRED round-trip as stream codes diff --git a/quest/m3/lite-tail-drop-count.md b/quest/m3/lite-tail-drop-count.md index 0b3e7f1f056..11b5a9bc934 100644 --- a/quest/m3/lite-tail-drop-count.md +++ b/quest/m3/lite-tail-drop-count.md @@ -2,7 +2,7 @@ ## Goal -From lite-05 on, a moq-lite session's tail is complete once every group up to the end has arrived or been reported in SUBSCRIBE_DROP. The 1 s `ledger::GRACE` (`rs/moq-net/src/ledger.rs:19`) then no longer decides it, in Rust and JS. +From lite-05 on, a moq-lite session's tail is complete once every group up to the end has arrived or been reported in SUBSCRIBE_SKIP or SUBSCRIBE_DROP. The 1 s `ledger::GRACE` (`rs/moq-net/src/ledger.rs:19`) then no longer decides it, in Rust and JS. ## Plan diff --git a/quest/m5/ietf-hole-marker.md b/quest/m5/ietf-hole-marker.md index 337da3f190f..e12c5426696 100644 --- a/quest/m5/ietf-hole-marker.md +++ b/quest/m5/ietf-hole-marker.md @@ -4,7 +4,8 @@ A negotiated moq-transport extension (a new `drafts/draft-lcurley-moq-*` document) lets a publisher name each group it won't deliver on a SUBSCRIBE, -like moq-lite's SUBSCRIBE_DROP: never produced, or dropped with its reason. +like moq-lite's SUBSCRIBE_SKIP and SUBSCRIBE_DROP: never produced, or dropped +with its reason. A relay's copy ingested over moq-transport then knows its holes ("null") instead of leaving them unknown ("undefined"). Our moq-transport publishers then answer a multi-group FETCH, and a multi-group draft-22 fill, whenever @@ -12,8 +13,9 @@ the cache knows every group in the range: a local, moq-lite, or extension-carrying copy. Groups go out in order. End of Non-Existent Range (`0x8C`) marks only a proven hole: a group never produced, or a budget-independent drop (a cut, an origin's eviction or expiry). A stale -drop judged under a narrower budget or range, as in -[Drop set](/quest/m1/drop-set.md), proves nothing for a wider FETCH. That +drop judged under a narrower budget or range, as lite-07's `Update` and the +subscriber's record of that update's terms qualify it, proves nothing for a +wider FETCH. That group is unknown, so the publisher fills it or the range is refused. Any other range stays refused, as [moq-dev/moq#7204](https://github.com/moq-dev/moq/pull/7204) @@ -59,9 +61,9 @@ Open, for this quest's interview: - The carrier: a message on the SUBSCRIBE's request stream (drafts 18, 20, and 22 give each request its own stream), or a status on a group stream. -- Whether the drop names its budget, as lite-07's SUBSCRIBE_DROP `Max Age` - does in [Drop set](/quest/m1/drop-set.md), so a stale drop doesn't settle - a wider reader. +- Whether the drop names the terms it was judged under, as lite-07's + SUBSCRIBE_DROP `Update` does in [Drop set](/quest/m1/drop-set.md), so a + stale drop doesn't settle a wider reader. - How a publisher answering a range walks the drop set's runs in group order, and where FETCH_OK's End Location falls. @@ -75,10 +77,6 @@ the extension is unchanged. Public API: none. Wire: a new negotiated SETUP option and its drop message. Our publishers answer multi-group FETCH and fills over known caches. -## Required - -- [Drop set](/quest/m1/drop-set.md) - the per-track runs that say which groups are known holes - ## Related - [Ordered reads wait only on owed groups](https://github.com/moq-dev/moq/pull/7403) - the ordered-reader wait a signalled hole ends diff --git a/rs/moq-archive/src/consumer/tests.rs b/rs/moq-archive/src/consumer/tests.rs index 4e36096eedf..0861ab26479 100644 --- a/rs/moq-archive/src/consumer/tests.rs +++ b/rs/moq-archive/src/consumer/tests.rs @@ -566,10 +566,9 @@ async fn a_completed_archive_ends_as_recorded() { ); assert_eq!(fetch(&broadcast, "errored", 0).await.unwrap(), expected(true, 0, 0..1)); - // The sequence below the end replays as an empty group, and those before it as dropped. + // The sequences the archive holds nothing for are skipped, and the end settles over them. let mut reader = subscribe(&broadcast, "tail").await; assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..1))); - assert_eq!(next(&mut reader).await.unwrap(), (3, Vec::new())); assert!(next(&mut reader).await.is_none()); assert_eq!(reader.finished().await.unwrap(), Position::group(4)); } diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index 2b3b6db58dc..d1d818119c7 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -131,17 +131,11 @@ impl Live { }, None, )?; - // An empty group stores no object, yet the end only settles once the live edge - // reaches it. The sequence just below the end replays as the empty group a - // source most often ended with, and any before it as dropped. + // An empty group stores no object, so the replay declares it never produces what + // the archive holds nothing for up to the end, and the end settles over it. let fin = at.group + u64::from(at.frame != 0); let from = self.next.unwrap_or(0); - if from < fin { - self.producer.drop_groups(from..fin - 1, moq_net::StreamError::NotFound); - self.producer - .create_group(group::Info { sequence: fin - 1 })? - .finish()?; - } + self.producer.skip_groups(from..fin); } End::Error(code) => { // After the newest stored sequence, keeping the history an abort would refuse. diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 5dbb5f9adf8..57822e7a74d 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -3220,6 +3220,20 @@ pub extern "C" fn moq_publish_track_group_at(track: u32, sequence: u64) -> i32 { }) } +/// Declare that a raw track will never produce groups `start` up to the exclusive `end`. +/// +/// Readers waiting on them move past without ending. The skip is final: creating a group +/// there with [moq_publish_track_group_at] then fails, though a group already written stays. +/// +/// Returns a zero on success, or a negative code on failure. +#[unsafe(no_mangle)] +pub extern "C" fn moq_publish_track_skip(track: u32, start: u64, end: u64) -> i32 { + ffi::enter(move || { + let track = ffi::parse_id(track)?; + State::lock().publish.track_skip(track, start, end) + }) +} + /// Write a single-frame group to a raw track with a timestamp. /// /// Convenience for the common one-frame-per-group pattern. Equivalent to diff --git a/rs/moq-c/src/publish.rs b/rs/moq-c/src/publish.rs index f4975f26a96..6d5b77b3f25 100644 --- a/rs/moq-c/src/publish.rs +++ b/rs/moq-c/src/publish.rs @@ -1123,6 +1123,13 @@ impl Publish { self.groups.insert(group) } + /// Declare that a raw track will never produce groups `start..end`. + pub fn track_skip(&mut self, track: Id, start: u64, end: u64) -> Result<(), Error> { + let raw = self.tracks.get(track).ok_or(Error::TrackNotFound)?; + raw.inner.skip_groups(start..end); + Ok(()) + } + /// Write a single-frame group to a raw track, `None` when untimed. pub fn track_frame( &mut self, diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index d4f8846dabb..69276b8603f 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -2407,6 +2407,23 @@ fn raw_track_sparse_groups_and_known_end() { assert_eq!(moq_publish_close(broadcast), 0); } +#[test] +fn raw_track_skips_are_final() { + let origin = id(moq_origin_create()); + let broadcast = publish_broadcast(origin, b"raw-track-skips-are-final"); + let name = b"skipped"; + let track = + id(unsafe { moq_publish_track(broadcast, name.as_ptr() as *const c_char, name.len(), std::ptr::null()) }); + + assert_eq!(moq_publish_track_skip(track, 1, 3), 0); + assert!(moq_publish_track_group_at(track, 2) < 0); + let group = id(moq_publish_track_group_at(track, 3)); + assert_eq!(moq_publish_group_finish(group), 0); + assert_eq!(moq_publish_track_finish(track), 0); + assert!(moq_publish_track_skip(track, 4, 5) < 0); + assert_eq!(moq_publish_close(broadcast), 0); +} + #[test] fn raw_track_and_group_abort_consume_their_handles() { let origin = id(moq_origin_create()); diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index acf270ced14..cc8d7597f8a 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -805,6 +805,19 @@ impl MoqTrackProducer { Ok(Arc::new(MoqGroupProducer::from_inner(group))) } + /// Declare that groups `start` up to the exclusive `end` will never be produced. + /// + /// Readers waiting on them move past without ending, and a relay forwards the skip. A + /// skip is final: creating a group there is then refused. A group already written keeps + /// what it is. [`append_group`](Self::append_group) skips the gap below it on its own. + pub fn skip_groups(&self, start: u64, end: u64) -> Result<(), MoqError> { + let _guard = crate::ffi::enter(); + let guard = self.inner.lock().unwrap(); + let raw = guard.as_ref().ok_or(MoqError::Closed)?; + raw.skip_groups(start..end); + Ok(()) + } + /// Write `frame` as a single-frame group. /// /// A timestamp at another scale than the track's is converted to it, rounding down. The frame must diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 12a7a01c4bf..ccd7ff0701d 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -2225,6 +2225,20 @@ fn raw_track_supports_sparse_groups_and_known_end() { track.finish().unwrap(); } +#[test] +fn raw_track_skips_are_final() { + let broadcast = MoqBroadcastProducer::new().unwrap(); + let track = broadcast.publish_track("skipped".into(), None).unwrap(); + + track.create_group(0).unwrap().finish().unwrap(); + track.skip_groups(0, 3).unwrap(); + // The written group keeps what it is; the rest of the range is final. + assert!(track.create_group(2).is_err()); + track.create_group(3).unwrap().finish().unwrap(); + track.abort(1).unwrap(); + assert!(matches!(track.skip_groups(4, 5), Err(MoqError::Closed))); +} + #[tokio::test] async fn abort_after_finish_keeps_the_track_handle() { let broadcast = MoqBroadcastProducer::new().unwrap(); diff --git a/rs/moq-net/src/error.rs b/rs/moq-net/src/error.rs index 1fb7652a95b..e2edb909549 100644 --- a/rs/moq-net/src/error.rs +++ b/rs/moq-net/src/error.rs @@ -384,6 +384,11 @@ pub enum Error { #[error("duplicate")] Duplicate, + /// A group was created at a sequence its track declared skipped: a skip says the + /// sequence will never be produced, so it is final. See `track::Producer::skip_groups`. + #[error("sequence was skipped")] + Skipped, + /// Nobody is reading any more, so the producer stopped; or the session closed cleanly, /// by either side, as `Session::closed` reports it. Not a failure. #[error("cancelled")] @@ -677,6 +682,7 @@ impl From<&Error> for SessionError { | Error::UnexpectedMessage | Error::UnexpectedStream | Error::Duplicate + | Error::Skipped | Error::Decode(_) | Error::Encode(_) | Error::WrongSize @@ -762,6 +768,7 @@ impl From<&Error> for StreamError { | Error::UnknownAlpn(_) | Error::TooManyRequests | Error::Duplicate + | Error::Skipped | Error::GoawayTimeout | Error::ProtocolViolation | Error::UnexpectedMessage => Self::Session(SessionError::from(err)), diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 2e175ed52bb..96ba9629baf 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -704,6 +704,7 @@ impl LiteSample { max_delay: std::time::Duration::from_secs(10), min_delay: None, ranges: vec![lite::Range::OLDEST], + update: 0, } .encode(w, version) .unwrap(), @@ -905,6 +906,7 @@ impl Default for Messages { start: Some(lite::Anchor::Absolute(crate::track::Position::group(1_000))), end: None, }], + update: 0, }, lite_update: lite::SubscribeUpdate { wanted_group: None, @@ -917,6 +919,7 @@ impl Default for Messages { start: Some(lite::Anchor::Absolute(crate::track::Position::group(1_000))), end: Some(lite::Anchor::Absolute(crate::track::Position::group(2_001))), }]), + update: 0, }, lite_start: lite::SubscribeResponse::Start(lite::SubscribeStart { largest: None, @@ -1083,6 +1086,7 @@ pub fn encode_subscribe(broadcast: &Path, track: &str, ietf: bool, out: &mut Vec min_delay: None, ranges: vec![lite::Range::OLDEST], order: track::Order::Descending, + update: 0, } .encode(w, BENCH_LITE) .unwrap(), diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 0a24837bd15..3b8eb9ccf88 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -6289,7 +6289,10 @@ impl OpenGroup { { tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); // Never arriving whole, the group is dropped so readers stop owing it. - track.drop_groups(sequence..sequence + 1, crate::StreamError::NotFound); + track.give_up( + sequence..sequence + 1, + crate::track::Hole::drop(crate::StreamError::NotFound), + ); return Poll::Ready(Err(Error::Unsupported)); } return Poll::Ready(Ok(Opened::Group(create(track)?, start))); @@ -6332,7 +6335,10 @@ impl OpenGroup { (Some(start), Some((producer, _))) => Ok(Opened::Group(producer, start)), (Some(start), None) => { tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); - track.drop_groups(sequence..sequence + 1, crate::StreamError::NotFound); + track.give_up( + sequence..sequence + 1, + crate::track::Hole::drop(crate::StreamError::NotFound), + ); Err(Error::Unsupported) } (None, Some((producer, next))) => Ok(Opened::Group(producer, next)), @@ -17020,7 +17026,7 @@ mod stitch_tests { run.track.broadcast().pool.poll(now + crate::ledger::GRACE, &waiter); assert_eq!( reader.poll_drop(&waiter), - Poll::Ready((1..3, crate::StreamError::NotFound)), + Poll::Ready((1..3, crate::track::Hole::drop(crate::StreamError::NotFound))), "{version}" ); assert!(reader.poll_drop(&waiter).is_pending(), "each drop is reported once"); diff --git a/rs/moq-net/src/ledger.rs b/rs/moq-net/src/ledger.rs index 002968647b1..8d8bbe2da34 100644 --- a/rs/moq-net/src/ledger.rs +++ b/rs/moq-net/src/ledger.rs @@ -155,6 +155,52 @@ impl Ledger { self.changes.insert(first..end); } + /// Take `groups` back out of what is accounted for, owed again from `now`: a verdict that + /// accounted for them was reversed. The gap they leave opens now, so it is waited for a + /// fresh grace, even where it joins an older gap. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub fn unaccount(&mut self, groups: Range, now: Instant) { + if groups.is_empty() { + return; + } + let overlapping: Vec = self.accounted(groups.clone()).map(|run| run.start).collect(); + if overlapping.is_empty() { + return; + } + for clipped in overlapping { + let start = self + .runs + .range(..=clipped) + .next_back() + .map(|(&start, _)| start) + .expect("an accounted run"); + let run = self.remove(start); + if start < groups.start { + self.ages.insert((run.since, start)); + self.runs.insert( + start, + Run { + end: groups.start, + since: run.since, + }, + ); + self.changes.insert(start..groups.start); + } + if groups.end < run.end { + self.ages.insert((now, groups.end)); + self.runs.insert( + groups.end, + Run { + end: run.end, + since: now, + }, + ); + self.changes.insert(groups.end..run.end); + } + } + self.demand(groups, now); + } + /// Account a live arrival, refreshing a cached incarnation even if its sequence was /// accounted before. Its gap ages stay unchanged, but readers must revisit a refusal. pub fn arrive(&mut self, sequence: u64, now: Instant) { @@ -454,6 +500,38 @@ mod tests { ledger.expire(lowered + GRACE); assert!(ledger.covers(1..10)); } + /// Unaccounting reopens what a reversed verdict accounted for, as a gap waited for a fresh + /// grace, and leaves the rest of the run accounted. + #[test] + fn unaccount_reopens_a_gap_for_a_fresh_grace() { + let start = Instant::now(); + let mut ledger = Ledger::new(GRACE); + arrive(&mut ledger, 0..10, start); + ledger.unaccount(3..5, start + GRACE); + assert!(ledger.covers(0..3) && ledger.covers(5..10)); + assert!(!ledger.covers(3..5)); + ledger.expire(start + GRACE * 3 / 2); + assert!(!ledger.covers(3..5), "its grace runs from the reversal"); + ledger.expire(start + GRACE * 2); + assert!(ledger.covers(0..10)); + } + + /// A reopened gap that joins an older one is waited for a fresh grace as a whole, rather + /// than folding with the older gap's age. + #[test] + fn unaccount_restarts_the_older_gap_it_joins() { + let start = Instant::now(); + let mut ledger = Ledger::new(GRACE); + arrive(&mut ledger, 0..2, start); + arrive(&mut ledger, 4..6, start); + let reversed = start + GRACE * 9 / 10; + ledger.unaccount(1..2, reversed); + ledger.expire(start + GRACE); + assert!(!ledger.covers(1..2), "its grace runs from the reversal"); + ledger.expire(reversed + GRACE); + assert!(ledger.covers(0..6)); + } + #[test] fn demand_can_move_an_older_gap_after_a_newer_one() { let start = Instant::now(); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index ac5f3ee1ee8..02cb1211c2b 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1,6 +1,7 @@ use crate::driver::Transport; use crate::transport::poll::Session as _; +use crate::model::track::Hole; use crate::{SessionError, datagram, frame, group, origin, track}; use rangemap::{RangeMap, RangeSet}; use std::{ @@ -2557,6 +2558,7 @@ impl Request for SubscribeServe { }; let floor = cursor_floor(shared.version, bounds.groups().start(), latest); let mut run = TrackRun::new(sub, track, bounds, floor, steer_tx); + run.update = msg.update; run.datagrams = LegacyDatagrams::join(legacy, msg.id, msg.priority, timescale); if let Some(update) = update { run.update(update); @@ -4722,7 +4724,8 @@ struct GroupOutcomes { #[cfg(test)] opened: std::sync::atomic::AtomicU64, unencodable: Mutex>, - dropped: Mutex>, + /// Groups whose stream failed, with why, for the control stream to name. + dropped: Mutex>, } impl Subscription { @@ -4801,7 +4804,17 @@ struct TrackRun { // The first servable group, held until the source resolves where its feed starts. first: Option, // Groups given up before the start resolved, which it must not name, with why. - skipped: RangeMap, + skipped: RangeMap, + // The number of the SUBSCRIBE (0) or latest SUBSCRIBE_UPDATE, which a drop echoes. + update: u64, + // Lite-07 groups whose stream expired as stale, which every update drops again under its + // number: the subscriber reopens each stale drop then, and the stream already reset. + expired: RangeSet, + // The expired runs an update owes a drop again, sent as ranges on the next poll. + redrop: RangeSet, + // The expired groups an update re-offers, still cached and in range, each served again + // on its own stream as stream credit allows. + reoffered: std::collections::VecDeque, end_sent: bool, // Lite-07 with every range bounded: once every group in its ranges is given up, it is // answered without a group to resolve its start, rather than left unanswered. @@ -4812,6 +4825,12 @@ struct TrackRun { // cannot tell when the subscriber stopped waiting for it. // `None` on other versions. served: Option>, + // Lite-07: the part of `served` named in SUBSCRIBE_DROP, never served. A skip replaces a + // drop, so it is sent over one of these, but never over a group served. + dropped: RangeSet, + // Lite-07: the part of `dropped` named stale (OLD), which a final drop or a group replaces, + // as one a route the front switched to delivers after the update that reopens it. + dropped_stale: RangeSet, /// The one group waiting for stream credit. The rest of the backlog waits in the cache, /// which bills, ages, and evicts it, and the cursor's budget skips whatever went stale /// meanwhile: a machine per parked group would hold state for as long as the peer @@ -4870,8 +4889,14 @@ impl TrackRun { start: None, first: None, skipped: RangeMap::new(), + update: 0, + expired: RangeSet::new(), + redrop: RangeSet::new(), + reoffered: Default::default(), end_sent: false, served, + dropped: RangeSet::new(), + dropped_stale: RangeSet::new(), opening: None, children: kio::Tasks::new(), idle: None, @@ -4899,6 +4924,7 @@ impl TrackRun { // The update moves the cursor and the bounds the track side judges against, which // notifies nothing it parked on. Idle::disarm(&mut self.idle); + self.update = upd.update; // Every update restates the wanted group. Past our largest it is a join, which // repeats the pending ask; at or below it the group already exists. if upd @@ -4992,6 +5018,36 @@ impl TrackRun { } self.bounded = bounds.ends(self.ctx.version); self.frames = bounds.frames; + // Lite-07 reopens every stale drop on any update, judged under its new terms: what was + // reported stale is offered again, and dropped again under this update's number if + // still stale. + if self.ctx.version.has_skip() { + let reopened = self.track.reoffer_stale(); + self.owe(reopened); + // A group whose stream already reset as stale is judged again under the new terms: + // one still cached, in range, and fresh under the new budget is served again. The + // rest is dropped again, by run, on the next poll. A run an earlier update left to + // drop again is judged again too, since these terms may serve it, and so is a group + // one re-offered that has not opened yet, since they may no longer admit it. + for group in std::mem::take(&mut self.reoffered) { + if let Some(end) = group.sequence.checked_add(1) { + self.expired.insert(group.sequence..end); + } + } + let pending = std::mem::take(&mut self.redrop); + self.expired.extend(pending.iter().cloned()); + for run in std::mem::take(&mut self.expired).iter() { + let (groups, gone) = self.track.reoffer_groups(run.clone()); + self.owe( + groups + .iter() + .filter_map(|group| Some(group.sequence..group.sequence.checked_add(1)?)) + .collect(), + ); + self.reoffered.extend(groups); + self.redrop.extend(gone.iter().cloned()); + } + } if self.ctx.version.has_ranges() { // A range retired before its start resolved is owed no SUBSCRIBE_STARTED. // Both list Range IDs in ascending order, since each one assigned is the next. @@ -5151,8 +5207,34 @@ impl TrackRun { let dropped = std::mem::take(&mut *self.ctx.outcomes.dropped.lock().unwrap()); if !dropped.is_empty() { - for dropped in dropped { - writer.buffer(&lite::SubscribeResponse::Drop(dropped))?; + for (sequence, err) in dropped { + // A stream expired under this subscription's budget is a stale verdict. + let hole = match err { + StreamError::Old => { + if self.ctx.version.has_skip() + && let Some(end) = sequence.checked_add(1) + { + self.expired.insert(sequence..end); + self.sent_stale(sequence..end); + } + Hole::stale(self.track.budget()) + } + err => Hole::drop(err), + }; + writer.buffer(&self.hole(sequence..=sequence, &hole))?; + } + return Poll::Ready(Ok(ControlFlow::Continue(()))); + } + if !self.redrop.is_empty() { + let hole = Hole::stale(self.track.budget()); + for run in std::mem::take(&mut self.redrop).iter() { + writer.buffer(&self.hole(run.start..=run.end - 1, &hole))?; + self.expired.insert(run.clone()); + self.sent_stale(run.clone()); + // A re-offer an update owed again is given up now, so the end sweep never names it. + if let Some(served) = &mut self.served { + served.insert(run.clone()); + } } return Poll::Ready(Ok(ControlFlow::Continue(()))); } @@ -5217,6 +5299,26 @@ impl TrackRun { return Poll::Ready(Ok(ControlFlow::Continue(()))); } + // A group an update re-offers takes the next stream credit, ahead of new ones, unless a + // verdict sent since accounted for it. + if self.opening.is_none() + && let Some(group) = self.reoffered.pop_front() + { + let sequence = group.sequence; + if self.served.as_ref().is_some_and(|served| served.contains(&sequence)) { + return Poll::Ready(Ok(ControlFlow::Continue(()))); + } + match position_group(group, &self.frames) { + Some(group) => self.serve(group), + None => { + if let Some(end) = sequence.checked_add(1) { + self.drop_range(sequence..end, &Hole::drop(StreamError::NotFound), writer)?; + } + } + } + return Poll::Ready(Ok(ControlFlow::Continue(()))); + } + // One cursor drives the whole subscription: poll the cap-aware arrival-order group. let emit_boundary = self.emit_range && !self.end_sent; let groups = self.opening.is_none(); @@ -5226,15 +5328,19 @@ impl TrackRun { Recv::Group(group) => { // Never a second stream for a sequence this subscription already accounted // for (served or dropped): a group re-created after its - // DROP stays dropped, since the subscriber may settle before it arrives. - if self - .served - .as_ref() - .is_some_and(|served| served.contains(&group.sequence)) + // DROP stays dropped, since the subscriber may settle before it arrives. A + // stale drop a lite-07 update reopened is owed again (see `Self::owe`). A + // lite-07 stale drop is no final verdict, and a Group Stream wins over it, so a + // group handed out after one is served, as one a route the front switched to + // delivers: the update that reopens the drop would never re-offer it. + let sequence = group.sequence; + if self.dropped_stale.contains(&sequence) + && let Some(end) = sequence.checked_add(1) { + self.replace_stale(sequence..end); + } else if self.served.as_ref().is_some_and(|served| served.contains(&sequence)) { return Poll::Ready(Ok(ControlFlow::Continue(()))); } - let sequence = group.sequence; let Some(group) = position_group(group, &self.frames) else { // Its head is gone, and this subscriber didn't ask for a // partial group. Skip it rather than open a stream that can @@ -5243,13 +5349,13 @@ impl TrackRun { if self.emit_range && self.start.is_none() { // At u64::MAX no later group can resolve a start past this one. if let Some(end) = sequence.checked_add(1) { - self.skipped.insert(sequence..end, StreamError::NotFound); + self.skipped.insert(sequence..end, Hole::drop(StreamError::NotFound)); } } else if self.start.is_some() && let Some(end) = sequence.checked_add(1) { // Lite-05+ names it now; a no-op on other versions. - self.drop_range(sequence..end, &StreamError::NotFound, writer)?; + self.drop_range(sequence..end, &Hole::drop(StreamError::NotFound), writer)?; } return Poll::Ready(Ok(ControlFlow::Continue(()))); }; @@ -5341,18 +5447,31 @@ impl TrackRun { true => self.track.poll_drop(waiter), false => self.track.poll_queued_drop(waiter), }; - let Poll::Ready((range, err)) = polled else { + let Poll::Ready((range, hole)) = polled else { return Ok(false); }; match self.start { - None => self.skipped.insert(range, err), + None => self.skipped.insert(range, hole), Some(_) => { - self.drop_range(range, &err, writer)?; + self.forward_range(range, &hole, writer)?; } } Ok(true) } + /// Owe `reopened` again, what this subscription reported stale before an update: lite-07 + /// re-offers a group it still holds and drops again, under the new update, one it cannot + /// serve. Earlier versions' drops are final, so their cursor takes nothing back. + fn owe(&mut self, reopened: Vec>) { + if let Some(served) = &mut self.served { + for run in reopened { + self.dropped.remove(run.clone()); + self.dropped_stale.remove(run.clone()); + served.remove(run); + } + } + } + /// Record `sequence` as served, for [`Self::drop_unserved`]. fn mark_served(&mut self, sequence: u64) { let Some(served) = &mut self.served else { @@ -5363,12 +5482,60 @@ impl TrackRun { } } - /// Drop, with `err`, whatever part of `range` this subscription has not been served or - /// dropped yet. Returns whether any drop was buffered. + /// Forward a verdict the cursor reached on `range`, as [`Self::drop_range`] does. On lite-07 + /// it also replaces what it overrides: a skip any drop already sent, and a drop the wire + /// names final (any code but OLD) a stale one, which the next update would otherwise reopen. A reader the earlier drop ended stays + /// ended. + fn forward_range( + &mut self, + range: Range, + hole: &Hole, + writer: &mut Writer, + ) -> Result { + if self.ctx.version.has_skip() && self.served.is_some() { + let replaces = match hole { + Hole::Skip => Some(&self.dropped), + Hole::Drop { .. } if hole.error() == StreamError::Old => None, + Hole::Drop { .. } => Some(&self.dropped_stale), + }; + let replaced: Vec<_> = replaces + .into_iter() + .flat_map(|runs| runs.overlapping(&range)) + .map(|run| run.start.max(range.start)..run.end.min(range.end)) + .collect(); + for run in replaced { + self.replace_stale(run.clone()); + if let Some(served) = &mut self.served { + served.remove(run); + } + } + } + self.drop_range(range, hole, writer) + } + + /// Record `run` as named stale (OLD) in a lite-07 drop, which a later verdict or group + /// replaces; see [`Self::dropped_stale`]. + fn sent_stale(&mut self, run: Range) { + self.dropped.insert(run.clone()); + self.dropped_stale.insert(run); + } + + /// Forget that `run` went out as a drop: a verdict or group replaces it. A stream of this + /// subscription that expired there is no longer owed again on an update, since what + /// replaced it settles it. + fn replace_stale(&mut self, run: Range) { + self.dropped.remove(run.clone()); + self.dropped_stale.remove(run.clone()); + self.expired.remove(run.clone()); + self.redrop.remove(run); + } + + /// Give up, as `hole` says, whatever part of `range` this subscription has not been + /// served or given up yet. Returns whether any was buffered. fn drop_range( &mut self, range: Range, - err: &StreamError, + hole: &Hole, writer: &mut Writer, ) -> Result { let Some(served) = &mut self.served else { @@ -5377,18 +5544,57 @@ impl TrackRun { if range.is_empty() { return Ok(false); } - let gaps: Vec<_> = served.gaps(&range).collect(); - for gap in &gaps { - served.insert(gap.clone()); - let drop = lite::SubscribeDrop::new(gap.start..=gap.end - 1, err, self.ctx.version); - writer.buffer(&lite::SubscribeResponse::Drop(drop))?; + let gaps: RangeSet = served.gaps(&range).collect(); + if self.ctx.version.has_skip() { + for gap in gaps.iter() { + match hole { + Hole::Skip => { + self.dropped.remove(gap.clone()); + self.dropped_stale.remove(gap.clone()); + } + // The wire names every OLD drop stale, which the next update reopens. One the + // cursor gave up as final, such as one relayed from a version whose drops are + // all final, is dropped again by each update, as an expired stream is. + Hole::Drop { .. } if hole.error() == StreamError::Old => { + self.dropped.insert(gap.clone()); + self.dropped_stale.insert(gap.clone()); + if hole.stale_budget().is_none() { + self.expired.insert(gap.clone()); + } + } + Hole::Drop { .. } => { + self.dropped.insert(gap.clone()); + self.dropped_stale.remove(gap.clone()); + } + } + } + } + served.extend(gaps.iter().cloned()); + for gap in gaps.iter() { + let msg = self.hole(gap.start..=gap.end - 1, hole); + writer.buffer(&msg)?; } Ok(!gaps.is_empty()) } + /// The message naming `groups` given up as `hole`: lite-07 skips what was never + /// produced, and earlier versions drop it as not found. + fn hole(&self, groups: std::ops::RangeInclusive, hole: &Hole) -> lite::SubscribeResponse { + let version = self.ctx.version; + match hole { + Hole::Skip if version.has_skip() => lite::SubscribeResponse::Skip(lite::SubscribeSkip { + start: *groups.start(), + end: *groups.end(), + }), + hole => { + lite::SubscribeResponse::Drop(lite::SubscribeDrop::new(groups, &hole.error(), self.update, version)) + } + } + } + /// Name every sequence from the start to the end that this subscription never got. The - /// track has ended, so none of them will be served now: groups it never produced (a - /// skipped sequence), and groups the cursor passed over (stale, or missing their head). + /// track has ended, so none of them will be served now: groups the cursor passed over + /// (stale, or missing their head), and gaps the publisher never declared skipped. /// /// Returns whether any drop was buffered. fn drop_unserved( @@ -5410,9 +5616,12 @@ impl TrackRun { // Only the groups the ranges name: one between disjoint ranges is not owed, and // naming it would keep a later update widening over it from delivering it. let end = self.track.subscription().end().map_or(fin, |end| end.min(fin)); + // A sequence the publisher declared skipped went out as a skip already; one it never + // declared might have existed, so it is not found. + let hole = Hole::drop(StreamError::NotFound); let mut dropped = false; for run in self.track.runs_in_range(start, end) { - dropped |= self.drop_range(run, &StreamError::NotFound, writer)?; + dropped |= self.drop_range(run, &hole, writer)?; } Ok(dropped) } @@ -5509,8 +5718,8 @@ impl TrackRun { }))?; // The subscriber now owes itself everything from the start, so a range given up // above it is named as it would have been after the start. - for (range, err) in skipped.iter() { - self.drop_range(range.start.max(start)..range.end, err, writer)?; + for (range, hole) in skipped.iter() { + self.forward_range(range.start.max(start)..range.end, hole, writer)?; } Ok(()) } @@ -6110,9 +6319,12 @@ impl GroupServe { /// that never got a stream has none, so the control stream names it with that error. fn settle_failed(&self, err: &Error) { if self.ctx.version.has_track_stream() { - let drop = - lite::SubscribeDrop::new(self.sequence..=self.sequence, &StreamError::from(err), self.ctx.version); - self.ctx.outcomes.dropped.lock().unwrap().push(drop); + self.ctx + .outcomes + .dropped + .lock() + .unwrap() + .push((self.sequence, StreamError::from(err))); } } } @@ -6992,6 +7204,7 @@ mod serve_group_test { min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], order: Order::Descending, + update: 0, }; run.replace(update(Some(0))); assert_eq!(track.demand().wanted(), Some(0), "stale request is not a join"); @@ -7128,6 +7341,7 @@ mod serve_group_test { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }); { let mut poll = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); @@ -7287,6 +7501,7 @@ mod serve_group_test { min_delay: None, ranges, order: Order::Descending, + update: 0, } } @@ -7440,9 +7655,10 @@ mod serve_group_test { ); } - /// A skipped sequence is accounted for by DROP rather than a stream count. + /// A gap the publisher never declared might have existed, so the end names it with a + /// DROP rather than a SKIP: here the reader passed it, so it is stale. #[kio::test] - async fn lite07_end_drops_skipped_sequences() { + async fn lite07_end_drops_an_undeclared_gap() { let mut track = track::Producer::new( Arc::new(broadcast::Info::default()), "test", @@ -7458,12 +7674,115 @@ mod serve_group_test { track.finish().unwrap(); assert!(futures::poll!(run.as_mut()).is_pending(), "the stream stays open"); - // SUBSCRIBE_START with the range starting at 0, a DROP for the skipped group 1 - // with NOT_FOUND (0x33), which the end gives up on at once, and SUBSCRIBE_FINISH at 3. + // SUBSCRIBE_START with the range starting at 0, a DROP for group 1 with OLD (0x34) + // under update 0, which the end gives up on at once, and SUBSCRIBE_FINISH at 3. assert_eq!( *log.writes.lock().unwrap(), - [0, 4, 1, 0, 1, 1, 2, 3, 1, 1, 0x33, 1, 2, 3, 0] + [0, 4, 1, 0, 1, 1, 2, 4, 1, 1, 0x34, 0, 1, 2, 3, 0] + ); + } + + /// A sequence the publisher declared it will never produce is accounted for by SKIP + /// rather than a stream count. + #[kio::test] + async fn lite07_skips_a_declared_skip() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe(None); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + let mut run = std::pin::pin!(kio::wait(move |waiter| run.poll(&mut writer, waiter))); + + write_group(&mut track, 0, 0); + assert!(futures::poll!(run.as_mut()).is_pending()); + write_group(&mut track, 2, 2); + track.skip_groups(1..2); + track.finish().unwrap(); + assert!(futures::poll!(run.as_mut()).is_pending(), "the stream stays open"); + + // SUBSCRIBE_START with the range starting at 0, a SKIP for group 1, and + // SUBSCRIBE_FINISH at 3. + assert_eq!(*log.writes.lock().unwrap(), [0, 4, 1, 0, 1, 1, 3, 2, 1, 1, 1, 2, 3, 0]); + } + + /// A relay forwards a SKIP over a DROP it already forwarded for the same groups, since the + /// skip replaces it; lite-06, which names a skip as a drop, sends nothing more. + #[kio::test] + async fn a_skip_goes_out_over_a_forwarded_drop() { + for version in [Version::Lite07, Version::Lite06] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + track.received(); + let subscriber = track.subscribe(None); + let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert!(run.start.is_some(), "{version}: started"); + track.give_up(1..3, Hole::drop(StreamError::Cancel)); + write_group(&mut track, 3, 3); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + track.skip_groups(0..4); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + kio::wait(|waiter| writer.poll_flush(waiter)).await.unwrap(); + + let writes = log.writes.lock().unwrap().clone(); + let mut buf = writes.as_slice(); + let mut holes = Vec::new(); + while !buf.is_empty() { + let (msg, size) = lite::SubscribeResponse::decode_slice(buf, version).unwrap(); + buf = &buf[size..]; + match msg { + lite::SubscribeResponse::Skip(skip) => holes.push(("skip", skip.start, skip.end)), + lite::SubscribeResponse::Drop(drop) => holes.push(("drop", drop.start, drop.end)), + _ => {} + } + } + let expected = match version.has_skip() { + true => vec![("drop", 1, 2), ("skip", 1, 2)], + false => vec![("drop", 1, 2)], + }; + assert_eq!(holes, expected, "{version}"); + } + } + + /// A huge declared skip goes out as one SUBSCRIBE_SKIP, at a cost independent of its + /// length, and the end settles over it. + #[kio::test] + async fn lite07_skips_a_huge_run_at_once() { + const FAR: u64 = 1 << 61; + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), ); + let subscriber = track.subscribe(None); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + track.skip_groups(1..FAR); + write_group(&mut track, FAR, 1); + track.finish().unwrap(); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + kio::wait(|waiter| writer.poll_flush(waiter)).await.unwrap(); + + let writes = log.writes.lock().unwrap().clone(); + let mut buf = writes.as_slice(); + let (mut skips, mut drops) = (Vec::new(), 0); + while !buf.is_empty() { + let (msg, size) = lite::SubscribeResponse::decode_slice(buf, Version::Lite07).unwrap(); + buf = &buf[size..]; + match msg { + lite::SubscribeResponse::Skip(skip) => skips.push((skip.start, skip.end)), + lite::SubscribeResponse::Drop(_) => drops += 1, + _ => {} + } + } + assert_eq!((skips, drops), (vec![(1, FAR - 1)], 0)); } /// A relay that holds only the first group of a backlog answers with the largest its @@ -7518,46 +7837,512 @@ mod serve_group_test { drops } - /// A stale group already named in SUBSCRIBE_DROP stays dropped once SUBSCRIBE_UPDATE - /// widens the max age past it: the subscriber may have settled it already. + /// The SUBSCRIBE_DROP ranges a run loop's subscribe stream carried, once flushed, with the + /// update each names. + async fn drop_updates( + writer: &mut Writer, + log: &Log, + version: Version, + ) -> Vec<(u64, u64, u64)> { + kio::wait(|waiter| writer.poll_flush(waiter)).await.unwrap(); + let writes = log.writes.lock().unwrap().clone(); + let mut buf = writes.as_slice(); + let mut drops = Vec::new(); + while !buf.is_empty() { + let (msg, size) = lite::SubscribeResponse::decode_slice(buf, version).unwrap(); + buf = &buf[size..]; + if let lite::SubscribeResponse::Drop(drop) = msg { + drops.push((drop.start, drop.end, drop.update)); + } + } + drops + } + + /// A lite-07 stale (OLD) drop is no final verdict: a final drop of the same group replaces + /// it, and a group handed out after it is served, as when a front switches to a route that + /// delivers what the old route reported stale, before the update that reopens it. Earlier + /// versions' drops are final, so neither goes out there. #[kio::test] - async fn a_reported_stale_drop_survives_a_widened_max_age() { - for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + async fn a_final_drop_or_a_group_replaces_a_stale_drop_on_lite07() { + for version in [Version::Lite06, Version::Lite07] { let mut track = track::Producer::new( Arc::new(broadcast::Info::default()), "test", crate::track::Info::timed(), ); - let subscriber = track.subscribe( - track::Subscription::default() - .with_delay(track::Delay::default().with_max(Duration::from_millis(1500))), - ); + let subscriber = track.subscribe(None); let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); write_group(&mut track, 0, 0); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} - // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. + assert!(run.start.is_some(), "{version}: started"); + let stale = Hole::stale(Duration::from_secs(1)); + run.forward_range(1..3, &stale, &mut writer).unwrap(); + run.forward_range(1..2, &Hole::drop(StreamError::Cancel), &mut writer) + .unwrap(); + write_group(&mut track, 2, 2); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + + let (drops, opened) = match version.has_skip() { + true => (vec![(1, 2, 0), (1, 1, 0)], 2), + false => (vec![(1, 2, 0)], 1), + }; + assert_eq!(drop_updates(&mut writer, &log, version).await, drops, "{version}"); + assert_eq!(run.ctx.outcomes.opened.load(Ordering::Relaxed), opened, "{version}"); + } + } + + /// A group that wins over a lite-07 stale drop already sent retires it: the next update + /// neither sends the drop again nor reopens the served group, and the end names nothing + /// for it. Here a relay's upstream reported group 1 stale, then delivered it, as a front + /// that switched routes does. + #[kio::test] + async fn a_group_served_over_a_stale_drop_retires_it() { + let budget = Duration::from_millis(1500); + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = + track.subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + track.give_up(1..2, Hole::stale(budget)); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(1, 1, 0)]); + write_group(&mut track, 1, 1000); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!(run.ctx.outcomes.opened.load(Ordering::Relaxed), 2, "the group wins"); + + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 1, + max_delay: budget, + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update: 1, + }); + write_group(&mut track, 2, 2000); + track.finish().unwrap(); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!( + run.ctx.outcomes.opened.load(Ordering::Relaxed), + 3, + "group 1 is not served again" + ); + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + [(1, 1, 0)], + "no second OLD, and no NOT_FOUND at the end" + ); + } + + /// A lite-07 stream that expired as stale is a stale drop like any other: a final drop + /// the reader reaches for it later replaces it, and no update drops it again then. + #[kio::test] + async fn a_final_drop_replaces_an_expired_stream_on_lite07() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe(None); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + // Group 1's stream expired, as a blocked transport would leave it. + run.ctx.outcomes.dropped.lock().unwrap().push((1, StreamError::Old)); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + run.forward_range(1..2, &Hole::drop(StreamError::Cancel), &mut writer) + .unwrap(); + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 1, + max_delay: Duration::from_secs(3600), + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update: 1, + }); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + [(1, 1, 0), (1, 1, 0)], + "the final drop replaces the OLD, and the update drops nothing again" + ); + } + + /// A stale group already named in SUBSCRIBE_DROP stays dropped before lite-07 after any + /// SUBSCRIBE_UPDATE, since those subscribers settle a drop for good. Lite-07 reopens it on + /// every update: one that widens the max age past it serves it, and one that keeps the + /// budget drops it again under the new update's number. The groups served around it are + /// never resent. + #[kio::test] + async fn any_update_reoffers_a_reported_stale_drop_on_lite07_alone() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + for widen in [true, false] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let budget = Duration::from_millis(1500); + let subscriber = track + .subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. + for (sequence, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { + write_group(&mut track, sequence, millis); + } + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + let case = format!("{version:?}, widen={widen}"); + assert_eq!(run.ctx.outcomes.opened.load(Ordering::Relaxed), 3, "{case}"); + assert_eq!(drop_updates(&mut writer, &log, version).await, [(1, 2, 0)], "{case}"); + + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 0, + max_delay: if widen { Duration::from_secs(10) } else { budget }, + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update: 1, + }); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + let (opened, drops, why): (u64, &[(u64, u64, u64)], _) = match (version, widen) { + (Version::Lite07, true) => (5, &[(1, 2, 0)], "the stale groups are served, and nothing else"), + (Version::Lite07, false) => (3, &[(1, 2, 0), (1, 2, 1)], "still stale, so dropped again"), + _ => (3, &[(1, 2, 0)], "no stream and no drop again"), + }; + assert_eq!(run.ctx.outcomes.opened.load(Ordering::Relaxed), opened, "{case}: {why}"); + assert_eq!(drop_updates(&mut writer, &log, version).await, drops, "{case}: {why}"); + } + } + } + + /// A group stale at the track's max age bound is final under any budget, yet a lite-07 + /// update still reopens its drop at the subscriber, so it is dropped again under the new + /// update rather than left owed. + #[kio::test] + async fn an_update_drops_again_what_is_stale_at_the_bound() { + for widen in [false, true] { + let budget = Duration::from_millis(1500); + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed().with_max_age(budget), + ); + let subscriber = + track.subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} for (sequence, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { write_group(&mut track, sequence, millis); } while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} - assert_eq!(run.ctx.outcomes.opened.load(Ordering::Relaxed), 3, "{version:?}"); - assert_eq!(drops(&mut writer, &log, version).await, [(1, 2)], "{version:?}"); + assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(1, 2, 0)]); + let opened = run.ctx.outcomes.opened.load(Ordering::Relaxed); + + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 5, + max_delay: if widen { Duration::from_secs(10) } else { budget }, + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update: 1, + }); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!( + run.ctx.outcomes.opened.load(Ordering::Relaxed), + opened, + "widen={widen}: the bound keeps them stale" + ); + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + [(1, 2, 0), (1, 2, 1)], + "widen={widen}: dropped again under update 1" + ); + } + } + + /// A group whose stream expired as stale is dropped again under every later update on + /// lite-07, by run, since the subscriber reopens each stale drop and the stream already + /// reset. Earlier versions' drops are final. + #[kio::test] + async fn an_update_drops_expired_streams_again_on_lite07_alone() { + for version in [Version::Lite06, Version::Lite07] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe(None); + let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + // Two group streams expired, as a blocked transport would leave them. + run.ctx + .outcomes + .dropped + .lock() + .unwrap() + .extend([(1, StreamError::Old), (2, StreamError::Old)]); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 1, + max_delay: Duration::from_secs(3600), + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update: 1, + }); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + let expected: &[(u64, u64, u64)] = match version { + Version::Lite07 => &[(1, 1, 0), (2, 2, 0), (1, 2, 1)], + _ => &[(1, 1, 0), (2, 2, 0)], + }; + assert_eq!(drop_updates(&mut writer, &log, version).await, expected, "{version:?}"); + } + } + + /// A lite-07 group whose stream expired but that the track still caches is judged again + /// under an update: served again once the budget covers it, and dropped again under the + /// new update while it is still stale. + #[kio::test] + async fn an_update_reoffers_a_cached_expired_stream_on_lite07() { + for widen in [true, false] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let budget = Duration::from_millis(500); + let subscriber = + track.subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + write_group(&mut track, 1, 1000); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + let opened = run.ctx.outcomes.opened.load(Ordering::Relaxed); + // Group 2 lands at 3s, so group 0's successor is 2s behind the edge, past the 500ms + // budget, and its stream expired. + write_group(&mut track, 2, 3000); + run.ctx.outcomes.dropped.lock().unwrap().push((0, StreamError::Old)); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(0, 0, 0)]); run.replace(lite::SubscribeUpdate { wanted_group: None, priority: 0, - max_delay: Duration::from_secs(10), + max_delay: if widen { Duration::from_secs(10) } else { budget }, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], order: Order::Descending, + update: 1, }); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + // Group 2 is served either way. + assert_eq!( + run.ctx.outcomes.opened.load(Ordering::Relaxed), + opened + 1 + u64::from(widen), + "widen={widen}: the cached group is offered again on its own stream once fresh" + ); + let drops: &[(u64, u64, u64)] = match widen { + true => &[(0, 0, 0)], + false => &[(0, 0, 0), (0, 0, 1)], + }; + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + drops, + "widen={widen}: still stale, it is dropped again under update 1" + ); + } + } + + /// A group an update re-offers waits for stream credit, and is judged again by whatever + /// arrives meanwhile: a later update whose range no longer admits it, or a final drop the + /// reader reaches for it, leaves it unserved. + #[kio::test] + async fn a_queued_reoffer_is_judged_again_before_it_opens_on_lite07() { + for narrow in [true, false] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let budget = Duration::from_millis(500); + let subscriber = + track.subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + write_group(&mut track, 1, 1000); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + write_group(&mut track, 2, 3000); + run.ctx.outcomes.dropped.lock().unwrap().push((0, StreamError::Old)); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + let opened = run.ctx.outcomes.opened.load(Ordering::Relaxed); + let update = |id, start, update| lite::SubscribeUpdate { + wanted_group: None, + priority: 0, + max_delay: Duration::from_secs(10), + min_delay: None, + ranges: vec![(id, lite::Range::span(start, 0, None, None))], + order: Order::Descending, + update, + }; + // Widened: group 0 is fresh again, and waits its turn. + run.update(update(0, Some(0), 1)); + assert_eq!(run.reoffered.len(), 1); + match narrow { + true => run.update(update(1, Some(1), 2)), + false => { + run.forward_range(0..1, &Hole::drop(StreamError::Cancel), &mut writer) + .unwrap(); + } + } + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert_eq!( run.ctx.outcomes.opened.load(Ordering::Relaxed), - 3, - "{version:?}: no stream for a dropped group" + opened, + "narrow={narrow}: group 0 never opens again" ); + let drops: &[(u64, u64, u64)] = match narrow { + // Outside the range now, so it is dropped again under the narrowing update. + true => &[(0, 0, 0), (0, 0, 2)], + false => &[(0, 0, 0), (0, 0, 1)], + }; + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + drops, + "narrow={narrow}" + ); + } + } + + /// A queued re-offer that a later update drops again as stale counts as given up: the + /// track's end leaves that stale drop standing, rather than replacing it with a final one. + #[kio::test] + async fn a_redropped_reoffer_stays_stale_at_the_end_on_lite07() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let budget = Duration::from_millis(500); + let subscriber = + track.subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + write_group(&mut track, 1, 1000); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + write_group(&mut track, 2, 3000); + run.ctx.outcomes.dropped.lock().unwrap().push((0, StreamError::Old)); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + // Update 1 widens the budget, so group 0 waits its turn; update 2 restores it first. + for (update, max_delay) in [(1, Duration::from_secs(10)), (2, budget)] { + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 0, + max_delay, + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update, + }); } + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + track.finish_at(track::Position::group(3), None).unwrap(); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + [(0, 0, 0), (0, 0, 2)], + "group 0 is dropped again as stale under update 2, and never named again" + ); + } + + /// A drop the cursor gave up as final with OLD, as one relayed from a lite-06 upstream is, + /// goes out as a stale drop on lite-07, which the subscriber reopens on its next update, so + /// each update drops it again rather than leave it owed. + #[kio::test] + async fn a_final_old_drop_is_dropped_again_by_each_update_on_lite07() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe(None); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + run.forward_range(1..2, &Hole::drop(StreamError::Old), &mut writer) + .unwrap(); + for update in [1, 2] { + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 1, + max_delay: Duration::from_secs(3600), + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update, + }); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + } + assert_eq!( + drop_updates(&mut writer, &log, Version::Lite07).await, + [(1, 1, 0), (1, 1, 1), (1, 1, 2)] + ); + } + + /// A run one update left to drop again is judged again under the next update that + /// arrives before it goes out, rather than dropped under terms that may serve it. + #[kio::test] + async fn a_pending_redrop_is_judged_again_under_a_later_update() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let budget = Duration::from_millis(500); + let subscriber = + track.subscribe(track::Subscription::default().with_delay(track::Delay::default().with_max(budget))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + write_group(&mut track, 0, 0); + write_group(&mut track, 1, 1000); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + let opened = run.ctx.outcomes.opened.load(Ordering::Relaxed); + write_group(&mut track, 2, 3000); + run.ctx.outcomes.dropped.lock().unwrap().push((0, StreamError::Old)); + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(0, 0, 0)]); + + // Update 1 keeps the budget, so group 0 is left to drop again; update 2 widens it + // before that drop goes out. + for (update, max_delay) in [(1, budget), (2, Duration::from_secs(10))] { + run.update(lite::SubscribeUpdate { + wanted_group: None, + priority: 0, + max_delay, + min_delay: None, + ranges: vec![(0, lite::Range::span(None, 0, None, None))], + order: Order::Descending, + update, + }); + } + while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} + assert_eq!( + run.ctx.outcomes.opened.load(Ordering::Relaxed), + opened + 2, + "group 0 is served again under update 2, beside group 2" + ); + assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(0, 0, 0)]); } /// Lite-05 and later name every sequence below the end that the subscription never @@ -7705,6 +8490,7 @@ mod serve_group_test { max_delay: Duration::from_secs(60), min_delay: None, ranges: vec![lite::Range::span(Some(0), 0, None, None)], + update: 0, }); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert_eq!(drops(&mut writer, &log, version).await, [(2, 2)]); @@ -7780,6 +8566,8 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); + // A received copy, whose groups may land out of order. + track.received(); let subscriber = track.subscribe( track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(1))), ); @@ -8123,6 +8911,7 @@ mod serve_group_test { max_delay: Duration::from_secs(60), min_delay: None, ranges: vec![lite::Range::span(Some(0), 0, None, None)], + update: 0, }); { let mut run = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); @@ -8164,6 +8953,7 @@ mod serve_group_test { max_delay: Duration::from_secs(1), min_delay: None, ranges: vec![lite::Range::span(Some(0), 0, None, None)], + update: 0, }); { let mut run = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); @@ -8210,6 +9000,7 @@ mod serve_group_test { max_delay: Duration::from_secs(1), min_delay: None, ranges: vec![lite::Range::span(Some(start_group), 0, None, None)], + update: 0, }; run.replace(update(5)); { @@ -8256,6 +9047,7 @@ mod serve_group_test { max_delay: Duration::from_secs(10), min_delay: None, ranges: vec![lite::Range::span(Some(start_group), 0, None, None)], + update: 0, }; run.replace(update(5)); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} @@ -8302,6 +9094,7 @@ mod serve_group_test { max_delay: Duration::from_secs(60), min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }); write_group(&mut track, 5, 5); assert_eq!(settle(&mut run), 1, "served a group below the latest"); @@ -8332,6 +9125,7 @@ mod serve_group_test { max_delay: Duration::from_secs(60), min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }); write_group(&mut track, 0, 0); @@ -8469,6 +9263,7 @@ mod serve_group_test { max_delay: Duration::from_secs(30), min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }); relay.settle(); assert_eq!(relay.opened(), 1, "the finished group is the live edge"); @@ -8499,6 +9294,7 @@ mod serve_group_test { max_delay: Duration::from_secs(30), min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }); relay.settle(); assert_eq!(relay.opened(), 3, "group 0 is cached and now in range"); @@ -8526,6 +9322,7 @@ mod serve_group_test { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(Some(7), 0, None, None)], + update: 0, }; relay.run.replace(update); relay.settle(); @@ -8662,8 +9459,8 @@ mod serve_group_test { assert!(futures::poll!(run.as_mut()).is_pending(), "group 1 is still opening"); assert_eq!( *log.writes.lock().unwrap(), - // The DROP carries OLD (0x34). - [0, 4, 1, 0, 1, 1, 2, 3, 0, 0, 0x34, 1, 2, 2, 0], + // The DROP carries OLD (0x34) and the SUBSCRIBE's update, 0. + [0, 4, 1, 0, 1, 1, 2, 4, 0, 0, 0x34, 0, 1, 2, 2, 0], "the expired group is dropped and the end precedes stream credit" ); @@ -8675,7 +9472,7 @@ mod serve_group_test { assert!(futures::poll!(run.as_mut()).is_pending(), "the stream stays open"); assert_eq!( *log.writes.lock().unwrap(), - [0, 4, 1, 0, 1, 1, 2, 3, 0, 0, 0x34, 1, 2, 2, 0] + [0, 4, 1, 0, 1, 1, 2, 4, 0, 0, 0x34, 0, 1, 2, 2, 0] ); } } @@ -9398,6 +10195,7 @@ mod tests { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }; leave_request::(version, &subscribe, wait, close).await; @@ -9458,6 +10256,7 @@ mod tests { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) .unwrap(); @@ -9607,6 +10406,7 @@ mod tests { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, } } @@ -9937,6 +10737,7 @@ mod tests { max_delay: Duration::from_secs(60), min_delay: None, ranges: vec![lite::Range::span(start, 0, None, None)], + update: 0, }; let mut serve = SubscribeServe::start(&publisher.shared, msg, "video", broadcast.broadcast().consume()) @@ -10071,6 +10872,7 @@ mod tests { min_delay: None, // A relay asks from its cached group, so the SUBSCRIBE is not a live-edge join. ranges: vec![lite::Range::span(Some(1), 0, None, None)], + update: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) .unwrap(); @@ -10098,6 +10900,7 @@ mod tests { remove: Vec::new(), add: Vec::new(), }, + update: 0, }; session.push(&update(Some(2), 3).encode_bytes(version).unwrap()); session.push(&update(Some(2), 7).encode_bytes(version).unwrap()); @@ -10160,6 +10963,7 @@ mod tests { remove: Vec::new(), add: Vec::new(), }, + update: 0, } } @@ -10199,6 +11003,7 @@ mod tests { max_delay: Duration::from_secs(3600), min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) .unwrap(); @@ -10627,6 +11432,7 @@ mod tests { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }; let session = ScriptedSession::new(request.encode_bytes(version).unwrap().to_vec()); let log = session.log.clone(); @@ -10739,6 +11545,7 @@ mod tests { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }; let session = ScriptedSession::new(request.encode_bytes(version).unwrap().to_vec()); let log = session.log.clone(); @@ -10877,6 +11684,7 @@ mod tests { max_delay: Duration::ZERO, min_delay: None, ranges: vec![lite::Range::span(None, 0, None, None)], + update: 0, }; let mut script = Vec::new(); msg.encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index 87b9e1a21bb..fe9ee0215f3 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -30,6 +30,10 @@ pub struct Subscribe<'a> { /// Which pending group to send first. Lite07+ only; earlier wires decode as /// [`Order::Descending`] and cannot ask for anything else. pub order: Order, + /// The number of the terms in effect: the SUBSCRIBE is 0 and every SUBSCRIBE_UPDATE + /// carries a larger one, which a [`SubscribeDrop`] echoes. Lite07+ only; earlier wires + /// decode as 0 and drop it on encode. + pub update: u64, } impl Subscribe<'_> { @@ -45,6 +49,7 @@ impl Subscribe<'_> { min_delay: self.min_delay, ranges: self.ranges, order: self.order, + update: self.update, } } } @@ -142,6 +147,7 @@ impl<'a> Message<'a> for Subscribe<'a> { (order, max_delay, min_delay, decode_ranges(r, version)?) } }; + let update = decode_update(r, version)?; Ok(Self { id, @@ -153,6 +159,7 @@ impl<'a> Message<'a> for Subscribe<'a> { min_delay, ranges, order, + update, }) } @@ -173,11 +180,28 @@ impl<'a> Message<'a> for Subscribe<'a> { encode_ranges(w, version, &self.ranges)?; } } + encode_update(w, version, self.update)?; Ok(()) } } +/// Decode the `Update` number after the ranges. A wire without the field decodes as 0. +fn decode_update(r: &mut Decoder<'_>, version: Version) -> Result { + match version.has_skip() { + true => r.varint(), + false => Ok(0), + } +} + +/// Encode the `Update` number, dropping it on a wire without the field. +fn encode_update(w: &mut Encoder<'_>, version: Version, update: u64) -> Result<(), EncodeError> { + match version.has_skip() { + true => w.varint(update), + false => Ok(()), + } +} + /// Decode the `Subscriber Min Age` after the max age: milliseconds plus one, zero meaning not /// stated. A wire without the field decodes as not stated. fn decode_min_delay(r: &mut Decoder<'_>, version: Version) -> Result, DecodeError> { @@ -678,6 +702,8 @@ pub struct SubscribeUpdate { pub ranges: R, /// See [`Subscribe::order`]. pub order: Order, + /// See [`Subscribe::update`]: larger than the subscription's previous one. + pub update: u64, } impl SubscribeUpdate { @@ -690,6 +716,7 @@ impl SubscribeUpdate { min_delay: self.min_delay, ranges, order: self.order, + update: self.update, } } } @@ -722,6 +749,7 @@ impl<'a> Message<'a> for SubscribeUpdate { } false => RangeEdit::Replace(decode_ranges(r, version)?), }; + let update = decode_update(r, version)?; let wanted_group = match version.has_refresh() { true => r.varint()?.checked_sub(1), @@ -735,6 +763,7 @@ impl<'a> Message<'a> for SubscribeUpdate { min_delay, ranges, order, + update, }) } @@ -761,6 +790,7 @@ impl<'a> Message<'a> for SubscribeUpdate { (RangeEdit::Replace(ranges), false) => encode_ranges(w, version, ranges)?, _ => return Err(EncodeError::Version), } + encode_update(w, version, self.update)?; if version.has_refresh() { w.varint_opt(self.wanted_group)?; @@ -998,13 +1028,15 @@ impl<'a> Message<'a> for SubscribeStarted { } } -/// Indicates that one or more groups have been dropped. +/// Names groups that existed but will not be delivered: aborted or reset, evicted or +/// expired with no copy left, or stale. /// /// The range `[start, end]` is inclusive on both ends. For example, /// `start = 5, end = 7` means groups 5, 6, and 7 were dropped. /// -/// Lite03+; publishers send it on Lite05 and later. -#[derive(Clone, Debug)] +/// Lite03+; publishers send it on Lite05 and later. Before lite-07 it also names sequences +/// declared skipped, which lite-07 names in [`SubscribeSkip`]. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SubscribeDrop { /// The first absolute group sequence in the dropped range. pub start: u64, @@ -1015,12 +1047,18 @@ pub struct SubscribeDrop { /// Why the groups were given up: a stream error code on lite-07 and later, or an /// application-specific code before, where 0 means simply unavailable. pub error: u64, + + /// The number of the SUBSCRIBE or SUBSCRIBE_UPDATE in effect when the groups were + /// given up ([`Subscribe::update`]): a stale (OLD) drop under a superseded one is no + /// verdict. Lite07+ only; earlier wires decode as 0. + pub update: u64, } #[cfg(feature = "async")] impl SubscribeDrop { - /// Name `groups` given up with `err`, as `version` carries it. - pub fn new(groups: std::ops::RangeInclusive, err: &StreamError, version: Version) -> Self { + /// Name `groups` given up with `err` under the terms numbered `update`, as `version` + /// carries it. + pub fn new(groups: std::ops::RangeInclusive, err: &StreamError, update: u64, version: Version) -> Self { Self { start: *groups.start(), end: *groups.end(), @@ -1029,6 +1067,7 @@ impl SubscribeDrop { true => version.encode_stream_code(err).into(), false => 0, }, + update: if version.has_skip() { update } else { 0 }, } } @@ -1055,15 +1094,18 @@ impl<'a> Message<'a> for SubscribeDrop { _ => {} } - let msg = Self { - start: r.varint()?, - end: r.varint()?, - error: r.varint()?, - }; - if msg.start.saturating_sub(msg.end) > 1 { + let (start, end) = (r.varint()?, r.varint()?); + if start.saturating_sub(end) > 1 { return Err(DecodeError::InvalidValue); } - Ok(msg) + let error = r.varint()?; + let update = decode_update(r, version)?; + Ok(Self { + start, + end, + error, + update, + }) } fn encode_msg(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { @@ -1081,7 +1123,51 @@ impl<'a> Message<'a> for SubscribeDrop { w.varint(self.start)?; w.varint(self.end)?; w.varint(self.error)?; + encode_update(w, version, self.update)?; + + Ok(()) + } +} + +/// Names sequences the publisher declared it will never produce (see +/// `track::Producer::skip_groups`): final, with no error, ending no reader. +/// +/// The range `[start, end]` is inclusive on both ends. Lite07+ only; earlier versions name +/// them in a [`SubscribeDrop`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubscribeSkip { + /// The first absolute group sequence in the skipped range. + pub start: u64, + + /// The last absolute group sequence in the skipped range (inclusive). + pub end: u64, +} + +impl<'a> Message<'a> for SubscribeSkip { + const CONTROL: Option = Some(crate::stats::Message::Subscribe); + fn decode_msg(r: &mut Decoder<'a>, version: Version) -> Result { + if !version.has_skip() { + return Err(DecodeError::Version); + } + let msg = Self { + start: r.varint()?, + end: r.varint()?, + }; + if msg.start.saturating_sub(msg.end) > 1 { + return Err(DecodeError::InvalidValue); + } + Ok(msg) + } + fn encode_msg(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { + if !version.has_skip() { + return Err(EncodeError::Version); + } + if self.start.saturating_sub(self.end) > 1 { + return Err(EncodeError::InvalidState); + } + w.varint(self.start)?; + w.varint(self.end)?; Ok(()) } } @@ -1094,13 +1180,14 @@ impl<'a> Message<'a> for SubscribeDrop { /// - Lite05+: `0x0` SUBSCRIBE_START (SUBSCRIBE_OK on lite-07), `0x1` SUBSCRIBE_FINISH /// (SUBSCRIBE_END before lite-07), `0x2` SUBSCRIBE_DROP /// (SUBSCRIBE_OK was removed; acceptance is implicit). -/// - Lite07+: also `0x4` SUBSCRIBE_STARTED. +/// - Lite07+: also `0x3` SUBSCRIBE_SKIP and `0x4` SUBSCRIBE_STARTED. #[derive(Clone, Debug)] pub enum SubscribeResponse { Ok(SubscribeOk), Start(SubscribeStart), Finish(SubscribeFinish), Drop(SubscribeDrop), + Skip(SubscribeSkip), Started(SubscribeStarted), } @@ -1132,6 +1219,7 @@ impl Encode for SubscribeResponse { Self::Start(start) => encode_typed(w, 0, start, version)?, Self::Finish(end) => encode_typed(w, 1, end, version)?, Self::Drop(drop) => encode_typed(w, 2, drop, version)?, + Self::Skip(skip) => encode_typed(w, 3, skip, version)?, Self::Started(started) => encode_typed(w, 4, started, version)?, Self::Ok(_) => return Err(EncodeError::Version), }, @@ -1159,6 +1247,7 @@ impl<'a> Decode<'a, Version> for SubscribeResponse { 0 => Ok(Self::Start(SubscribeStart::decode(buf, version)?)), 1 => Ok(Self::Finish(SubscribeFinish::decode(buf, version)?)), 2 => Ok(Self::Drop(SubscribeDrop::decode(buf, version)?)), + 3 if version.has_skip() => Ok(Self::Skip(SubscribeSkip::decode(buf, version)?)), 4 if version.has_ranges() => Ok(Self::Started(SubscribeStarted::decode(buf, version)?)), _ => Err(DecodeError::InvalidMessage(typ)), } @@ -1182,7 +1271,8 @@ mod test { SubscribeDrop { start: 3, end: 1, - error: 0 + error: 0, + update: 0, } .encode_bytes(version) .is_err() @@ -1193,7 +1283,12 @@ mod test { (1 << 62) - 1 }; for (start, end) in [(2, 1), (max, max)] { - let msg = SubscribeDrop { start, end, error: 0 }; + let msg = SubscribeDrop { + start, + end, + error: 0, + update: 0, + }; let data = msg.encode_bytes(version).unwrap(); let (decoded, _) = SubscribeDrop::decode_slice(&data, version).unwrap(); assert_eq!((decoded.start, decoded.end), (start, end)); @@ -1313,11 +1408,13 @@ mod test { start: 1, end: 3, error: 7, + update: 0, }); let mut buf = Vec::new(); resp.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07) .unwrap(); - assert_eq!(buf, [2, 3, 1, 3, 7]); + // The drop names the SUBSCRIBE's update, 0. + assert_eq!(buf, [2, 4, 1, 3, 7, 0]); match crate::coding::decode_buf(&mut buf.as_slice(), Version::Lite07, |r, v| { SubscribeResponse::decode(r, v) }) @@ -1328,12 +1425,66 @@ mod test { } } + /// A drop names the update it was given up under on lite-07; earlier versions carry none. + #[test] + fn a_drop_names_its_update_on_lite07() { + let msg = SubscribeDrop { + start: 4, + end: 4, + error: 0x34, + update: 2, + }; + let buf = msg.encode_bytes(Version::Lite07).unwrap(); + assert_eq!(buf.as_ref(), [4, 4, 4, 0x34, 2]); + let (decoded, _) = SubscribeDrop::decode_slice(&buf, Version::Lite07).unwrap(); + assert_eq!(decoded, msg); + let msg = SubscribeDrop { error: 0, ..msg }; + let buf = msg.encode_bytes(Version::Lite06).unwrap(); + assert_eq!(buf.as_ref(), [3, 4, 4, 0]); + let (decoded, _) = SubscribeDrop::decode_slice(&buf, Version::Lite06).unwrap(); + assert_eq!(decoded.update, 0); + } + + /// SUBSCRIBE_SKIP is type 3 on lite-07 and unknown before it. + #[test] + fn subscribe_skip_is_lite07_only() { + let resp = SubscribeResponse::Skip(SubscribeSkip { start: 1, end: 3 }); + let buf = resp.encode_bytes(Version::Lite07).unwrap(); + assert_eq!(buf.as_ref(), [3, 2, 1, 3]); + match SubscribeResponse::decode_slice(&buf, Version::Lite07).unwrap().0 { + SubscribeResponse::Skip(skip) => assert_eq!(skip, SubscribeSkip { start: 1, end: 3 }), + other => panic!("expected Skip, got {other:?}"), + } + assert!(matches!( + SubscribeSkip::decode_slice(&[2, 3, 1], Version::Lite07), + Err(DecodeError::InvalidValue) + )); + assert!(resp.encode_bytes(Version::Lite06).is_err()); + assert!(matches!( + SubscribeResponse::decode_slice(&buf, Version::Lite06), + Err(DecodeError::InvalidMessage(3)) + )); + } + + /// SUBSCRIBE and SUBSCRIBE_UPDATE carry their update number on lite-07 alone. + #[test] + fn update_number_roundtrips_only_on_lite07() { + for version in [Version::Lite06, Version::Lite07] { + let mut msg = subscribe_sample(); + msg.update = 5; + let buf = encode(&msg, version).unwrap(); + let decoded = crate::coding::decode_from(&mut buf.as_slice(), version, Subscribe::decode_msg).unwrap(); + assert_eq!(decoded.update, if version.has_skip() { 5 } else { 0 }); + } + } + #[test] fn subscribe_drop_is_type_2_on_lite05() { let resp = SubscribeResponse::Drop(SubscribeDrop { start: 1, end: 3, error: 0, + update: 0, }); let mut buf = Vec::new(); resp.encode(&mut Encoder::new(&mut buf, Version::Lite05.into()), Version::Lite05) @@ -1354,6 +1505,7 @@ mod test { start: 1, end: 3, error: 0, + update: 0, }); let mut buf = Vec::new(); resp.encode(&mut Encoder::new(&mut buf, Version::Lite04.into()), Version::Lite04) @@ -1382,6 +1534,7 @@ mod test { start: absolute(7, 4), end: absolute(9, 3), }], + update: 0, } } @@ -1562,6 +1715,7 @@ mod test { min_delay: None, ranges: sent.edit(&msg.ranges).unwrap(), order: crate::track::Order::Ascending, + update: 0, }; let buf = update.encode_bytes(Version::Lite07).unwrap(); let got = crate::coding::decode_buf(&mut buf.as_ref(), Version::Lite07, |r, v| SubscribeUpdate::decode(r, v)) @@ -1607,6 +1761,7 @@ mod test { min_delay: None, ranges: edit, order: crate::track::Order::Descending, + update: 0, }; let buf = update.encode_bytes(version).unwrap(); let got = @@ -1708,9 +1863,9 @@ mod test { msg.ranges = vec![Range::OLDEST]; let mut buf = encode(&msg, version).unwrap(); - // The list is last: a count of one, then two anchors of type 0. - let at = buf.len() - 3; - assert_eq!(&buf[at..], [1, 0, 0]); + // The list is last but for the update number: a count of one, then two anchors of type 0. + let at = buf.len() - 4; + assert_eq!(&buf[at..], [1, 0, 0, 0]); buf[at] = 0; buf.truncate(at + 1); assert!(matches!( @@ -1719,7 +1874,7 @@ mod test { )); let mut buf = encode(&msg, version).unwrap(); - let last = buf.len() - 1; + let last = buf.len() - 2; buf[last] = 4; assert!(matches!( crate::coding::decode_from(&mut buf.as_slice(), version, Subscribe::decode_msg), @@ -1785,6 +1940,7 @@ mod test { min_delay: None, ranges: LiveRanges::new(version, &[Range::OLDEST]).unchanged(), order: crate::track::Order::Descending, + update: 0, }; let buf = update.encode_bytes(version).unwrap(); let got = crate::coding::decode_buf(&mut buf.as_ref(), version, |r, v| SubscribeUpdate::decode(r, v)) @@ -1818,6 +1974,7 @@ mod test { min_delay, ranges: LiveRanges::new(version, &[Range::OLDEST]).unchanged(), order: crate::track::Order::Descending, + update: 0, }; let buf = update.encode_bytes(version).unwrap(); let got = crate::coding::decode_buf(&mut buf.as_ref(), version, |r, v| SubscribeUpdate::decode(r, v)) @@ -1846,6 +2003,7 @@ mod test { min_delay: None, ranges: LiveRanges::new(version, &[Range::OLDEST]).unchanged(), order, + update: 0, }; let mut buf = Vec::new(); update diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 76db28b054d..e34ac33b7bf 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -19,6 +19,7 @@ use super::Version; use crate::tail::{self, Reading, Settle, Tail}; use kio::Lock; +use rangemap::RangeSet; mod fills; use fills::{FillAsk, RangeFills}; @@ -219,10 +220,16 @@ struct Parked { /// The headers still waiting on the accept, by ticket, in arrival order. waiting: Vec<(u64, lite::Group)>, /// What the accept opened for each, until its stream takes it. - opened: HashMap>, + opened: HashMap>, next: u64, } +/// What a group stream's header opened in its subscription's copy. +enum Open { + Group(Opened), + Duplicate(Duplicate), +} + /// A group stream's group, opened in its subscription's copy for the stream to fill. struct Opened { group: group::Producer, @@ -231,6 +238,16 @@ struct Opened { reading: Reading, } +/// A group stream duplicating a copy still arriving, such as a lite-07 re-offer that +/// overtook the reset of the stream it replaces. It waits unread, which its stream credit +/// bounds, and is discarded if that copy finishes or takes the group if it fails. +struct Duplicate { + arriving: group::Handle, + track: track::Producer, + /// Counted already, and kept open while held, since the stream may still carry the group. + reading: Reading, +} + impl Drop for TrackEntry { fn drop(&mut self) { // Groups the accept opened for streams that never took them end as cancelled with @@ -243,7 +260,9 @@ impl Drop for TrackEntry { Err(_) => return, }; for opened in opened.into_values().flatten() { - let _ = opened.group.abort(Error::Cancel); + if let Open::Group(opened) = opened { + let _ = opened.group.abort(Error::Cancel); + } } } } @@ -316,7 +335,7 @@ impl TrackEntry { } /// Take what the accept opened for the parked stream `ticket`. - fn unpark(&mut self, ticket: u64) -> Result { + fn unpark(&mut self, ticket: u64) -> Result { let Some(parked) = &self.parked else { return Err(Error::Cancel); }; @@ -347,7 +366,7 @@ impl TrackEntry { } Err(_) => return, }; - if let Some(Ok(opened)) = opened { + if let Some(Ok(Open::Group(opened))) = opened { let _ = opened.group.abort(err.clone()); } if done { @@ -356,8 +375,8 @@ impl TrackEntry { } /// Open the group of a stream whose header is `hdr` in the copy. - fn open(&mut self, hdr: &lite::Group, version: Version, now: crate::time::Instant) -> Result { - let Some(Accepted { producer, framing }) = &mut self.copy else { + fn open(&mut self, hdr: &lite::Group, version: Version, now: crate::time::Instant) -> Result { + let Some(Accepted { producer, .. }) = &mut self.copy else { return Err(Error::Cancel); }; let group_info = group::Info { sequence: hdr.sequence }; @@ -390,8 +409,38 @@ impl TrackEntry { } } } + self.place(hdr, version, received, reading) + } + + /// Open the group of a held stream again, once the copy it waited on failed. + fn reopen(&self, hdr: &lite::Group, version: Version, reading: Reading) -> Result { + let Some(Accepted { producer, .. }) = &self.copy else { + return Err(Error::Cancel); + }; + let received = producer.create_group_at(group::Info { sequence: hdr.sequence }, hdr.frame_start); + self.place(hdr, version, received, reading) + } + + /// Give a stream what the cache answered for its header. + fn place( + &self, + hdr: &lite::Group, + version: Version, + received: Result, + reading: Reading, + ) -> Result { + let Some(Accepted { producer, framing }) = &self.copy else { + return Err(Error::Cancel); + }; let group = match received { - Ok(group) => group, + Ok(track::Received::Group(group)) => group, + Ok(track::Received::Held(arriving)) => { + return Ok(Open::Duplicate(Duplicate { + arriving, + track: producer.clone(), + reading, + })); + } // The group is at or past the end the publisher declared, which no // later stream can repair. Before lite-05 only a local finish sets // an end, and lite-05 specified an inclusive one, so its last @@ -408,12 +457,12 @@ impl TrackEntry { } Err(err) => return Err(err), }; - Ok(Opened { + Ok(Open::Group(Opened { group, track: producer.clone(), framing: *framing, reading, - }) + })) } } @@ -1454,7 +1503,8 @@ impl GroupRecv { None => Poll::Ready(Ok(())), }); self.park_wait = None; - let (ietf, hdr) = header; + let ietf: bool = header.0; + let hdr: lite::Group = header.1; if let (Err(err), Some((_, ticket))) = (&accepted, &parked) && let Some(entry) = self.subscriber.subscribes.lock().get_mut(&hdr.subscribe) { @@ -1462,7 +1512,7 @@ impl GroupRecv { } accepted?; - let opened = { + let open = { let mut subs = self.subscriber.subscribes.lock(); let entry = subs.get_mut(&hdr.subscribe).ok_or(Error::Cancel)?; match &parked { @@ -1470,15 +1520,45 @@ impl GroupRecv { None => entry.open(&hdr, self.subscriber.version, self.subscriber.runtime.now()), } }; + let mut open: Option = Some(open?); + // A duplicate of a copy still arriving waits, unread, for that copy to settle. + let opened: Opened = wait!(Hold => loop { + let duplicate = match open.take() { + Some(Open::Group(opened)) => break Poll::Ready(Ok(opened)), + Some(Open::Duplicate(duplicate)) => duplicate, + None => break Poll::Ready(Err(Error::Cancel)), + }; + if let Poll::Ready(err) = self.reader.poll_reset(waiter) { + break Poll::Ready(Err(err)); + } + if let Poll::Ready(err) = duplicate.track.poll_aborted(waiter) { + break Poll::Ready(Err(err)); + } + match duplicate.arriving.poll_closed(waiter) { + Poll::Pending => { + *open = Some(Open::Duplicate(duplicate)); + break Poll::Pending; + } + // The copy holds every frame, so this one only duplicates it. + Poll::Ready(Ok(())) => break Poll::Ready(Err(Error::Duplicate)), + Poll::Ready(Err(_)) => { + let subs = self.subscriber.subscribes.lock(); + let Some(entry) = subs.get(&hdr.subscribe) else { + break Poll::Ready(Err(Error::Cancel)); + }; + match entry.reopen(hdr, self.subscriber.version, duplicate.reading) { + Ok(reopened) => *open = Some(reopened), + Err(err) => break Poll::Ready(Err(err)), + } + } + } + })?; let Opened { group, track, framing, reading, - } = match opened { - Ok(opened) => opened, - Err(err) => return Poll::Ready(Err(err)), - }; + } = opened; let (priority, order) = track .subscription() .map_or((0, Default::default()), |sub| (sub.priority, sub.order)); @@ -1507,7 +1587,10 @@ impl GroupRecv { let res = match res { Ok(()) => { - let _ = group.finish(); + // Only a lite-07 stale drop asks which groups arrived whole. + if group.finish().is_ok() && self.subscriber.version.has_skip() { + reading.finish(); + } Ok(()) } Err(err @ (Error::Cancel | Error::Stream(crate::StreamError::Cancel))) => { @@ -3080,6 +3163,8 @@ mod tests { fn feed_fetch(sequence: u64) -> (track::Producer, kio::Pending, track::Fill) { let info = track::Info::default().with_timescale(None).with_max_age(None); let track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", info); + // The relay's copy of the upstream feed, whose groups may land out of order. + track.received(); let dynamic = track.requests(); let fetching = track.consume().fetch_group(sequence, None); let req = dynamic.requested_fill().now_or_never().expect("queued").unwrap(); @@ -3113,6 +3198,8 @@ mod tests { wanted: None, unanswered: None, _working: crate::session::Work::default().start(), + update: 0, + stale: RangeSet::new(), }) } @@ -3268,6 +3355,8 @@ mod tests { wanted: None, unanswered: None, _working: crate::session::Work::default().start(), + update: 0, + stale: RangeSet::new(), }); if update { h.serve @@ -3310,6 +3399,8 @@ mod tests { ) -> (track::Producer, Vec>, Vec) { let info = track::Info::default().with_timescale(None).with_max_age(None); let track = track::Producer::new(Arc::new(crate::broadcast::Info::default()), "video", info); + // The relay's copy of the upstream feed, whose groups may land out of order. + track.received(); let dynamic = track.requests(); let mut callers = Vec::new(); let mut requests = Vec::new(); @@ -3734,6 +3825,8 @@ mod tests { wanted: None, unanswered: None, _working: crate::session::Work::default().start(), + update: 0, + stale: RangeSet::new(), }; assert_eq!( sub.owed(None), @@ -3848,6 +3941,8 @@ mod tests { wanted: None, unanswered: None, _working: crate::session::Work::default().start(), + update: 0, + stale: RangeSet::new(), }; assert_eq!(sub.fill_range(0), None, "the demand's range is no fill"); @@ -3914,6 +4009,8 @@ mod tests { wanted: None, unanswered: None, _working: crate::session::Work::default().start(), + update: 0, + stale: RangeSet::new(), }; assert_eq!(sub.owed(None), Some(vec![4..10])); @@ -4827,6 +4924,108 @@ mod tests { assert!(settle.poll(&kio::Waiter::noop(), |tail| tail.streams() >= 1).is_ready()); } + /// A lite-07 re-offer can overtake the reset of the stream it replaces: its stream waits, + /// unread, until that copy settles. It is discarded once the copy finishes, and takes the + /// group once the copy fails. + #[kio::test] + async fn a_duplicate_of_an_arriving_copy_waits_for_it_to_settle() { + use crate::transport::poll::Session as _; + + for first_fails in [false, true] { + let version = Version::Lite07; + let mut script = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); + lite::Group { + subscribe: 7, + sequence: 3, + frame_start: 0, + } + .encode(&mut encoder, version) + .unwrap(); + encoder.varint(crate::coding::varint::zigzag(300)).unwrap(); + encoder.bytes(b"again").unwrap(); + let session = crate::lite::test_transport::ScriptedSession::eof(script); + let clock = crate::time::Deadlines::new(crate::time::Instant::now()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: clock.clone(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + + let track = track::Producer::new( + std::sync::Arc::new(crate::broadcast::Info::default()), + "video", + crate::track::Info::timed(), + ); + let subscription = crate::track::Subscription::default() + .with_delay(crate::track::Delay::default().with_max(std::time::Duration::from_secs(10))) + .with_range(0..); + let mut reader = track.subscribe(subscription.clone()); + // The original copy, still arriving: one frame in, no end yet. + let Ok(track::Received::Group(mut first)) = track.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first + .write_frame(Timestamp::from_millis(300).unwrap(), &b"first"[..]) + .unwrap(); + let tail = kio::Producer::new(Tail::default()); + subscriber.subscribes.lock().insert( + 7, + TrackEntry::accepted( + track.clone(), + lite::Framing { + timescale: Some(Timescale::MILLI), + ietf: None, + compressed: false, + }, + tail.clone(), + ), + ); + + let (_, recv) = session.open_bi(Default::default()).await.unwrap(); + let mut duplicate = GroupRecv::new( + subscriber, + Reader::new(crate::transport::boxed::RecvStream::new(recv), version), + ); + let waiter = kio::Waiter::noop(); + assert!(duplicate.poll_serve(&waiter).is_pending(), "first fails: {first_fails}"); + assert_eq!(duplicate.state.name(), "Hold", "first fails: {first_fails}"); + // Counted and holding the end open while it waits, since it may carry the group. + assert_eq!(tail.read().streams(), 1); + + let mut original = reader.assert_group(); + if !first_fails { + first.finish().unwrap(); + let res = kio::wait(|waiter| duplicate.poll_serve(waiter)).await; + assert!(matches!(res, Err(Error::Duplicate)), "{res:?}"); + let frame = original.read_frame().await.unwrap().unwrap(); + assert_eq!(&frame.payload[..], b"first"); + assert!(original.read_frame().await.unwrap().is_none()); + continue; + } + + // The reset lands: the held stream takes the group. + first.abort(Error::Cancel).unwrap(); + let res = kio::wait(|waiter| duplicate.poll_serve(waiter)).await; + assert!(res.is_ok(), "{res:?}"); + // A reader already handed the original sees it fail, since no sequence is handed + // out twice; one that had not reached it reads the re-offer. + assert!(original.read_frame().await.is_err()); + let mut later = track.subscribe(subscription); + let mut group = later.assert_group(); + assert_eq!(group.sequence, 3); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"again"); + assert!(group.read_frame().await.unwrap().is_none()); + } + } + /// A SUBSCRIBE_FINISH below a group already received contradicts that group. lite-05 /// specified an inclusive end, and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it only /// costs the early boundary; later drafts abort the track. @@ -5065,6 +5264,426 @@ mod tests { assert!(matches!(reader.recv_group().await, Err(Error::ProtocolViolation))); } + /// A lite-07 subscription's serve loop with an active SUBSCRIBE under update 0 and the + /// max age `budget`, its broadcast, and the tail it accounts on. + async fn lite07_serving( + budget: Duration, + ) -> (TrackServe, ServeLoop, kio::Producer, crate::broadcast::Producer) { + let version = Version::Lite07; + let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: crate::auth::Link::new(false), + }); + let serve = TrackServe { + validity: Default::default(), + route: None, + source: None, + subscriber, + path: Path::from_static("room"), + instance: None, + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let request = broadcast.reserve_track("video").unwrap(); + let mut serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); + let tail = kio::Producer::new(Tail::default()); + serving.sub = Sub::Active(SubStream { + order: crate::track::Order::Descending, + stream: Stream::open(&crate::driver::Transport::new(session.clone()), version) + .await + .unwrap(), + id: 0, + max_delay: budget, + min_delay: None, + start: None, + priority: 0, + requested: None, + live: lite::LiveRanges::new(version, &[lite::Range::OLDEST]), + demand: Some(Subscription::default()), + fill_ids: Default::default(), + fill_groups: Default::default(), + tail: tail.clone(), + served: None, + started: false, + starts: Default::default(), + declared: None, + end: None, + wanted: None, + unanswered: None, + _working: crate::session::Work::default().start(), + update: 0, + stale: RangeSet::new(), + }); + (serve, serving, tail, broadcast) + } + + /// A SUBSCRIBE_DROP can only echo an update the subscriber sent: one naming a later update + /// is a protocol violation, and accounts for nothing. + #[kio::test] + async fn a_drop_naming_an_unsent_update_is_a_protocol_violation() { + let (serve, mut serving, tail, _broadcast) = lite07_serving(Duration::from_millis(500)).await; + let drop = lite::SubscribeResponse::Drop(lite::SubscribeDrop { + start: 1, + end: 2, + error: u64::from(crate::coding::StreamCodes::encode_stream_code( + &Version::Lite07, + &crate::StreamError::Old, + )), + update: 1, + }); + assert!(matches!( + serving.apply_response(&serve, drop), + Err(Error::ProtocolViolation) + )); + assert!(!tail.read().covers(1..3)); + } + + /// On lite-07 an OLD drop is a stale verdict judged under the budget this subscriber sent + /// with the update it names. Any SUBSCRIBE_UPDATE reopens every stale run in the tail, + /// whether or not it widens the budget, and an OLD drop naming a superseded update is no + /// verdict: still owed, and given up on no reader. Any other drop is final, whatever update + /// it names. + #[kio::test] + async fn an_update_reopens_every_stale_drop_and_supersedes_the_rest() { + let version = Version::Lite07; + let budget = Duration::from_millis(500); + let (serve, mut serving, tail, _broadcast) = lite07_serving(budget).await; + let mut reader = serving.serving.subscribe( + crate::track::Subscription::default() + .with_delay(crate::track::Delay::default().with_max(budget)) + .with_ranges([crate::track::Range::groups(0..)]), + ); + let waiter = kio::Waiter::noop(); + let code = |err: &crate::StreamError| u64::from(crate::coding::StreamCodes::encode_stream_code(&version, err)); + let drop = |start, end, err: &crate::StreamError, update| { + lite::SubscribeResponse::Drop(lite::SubscribeDrop { + start, + end, + error: code(err), + update, + }) + }; + let covers = |groups: std::ops::Range| tail.read().covers(groups); + + serving + .apply_response(&serve, drop(1, 2, &crate::StreamError::Old, 0)) + .unwrap(); + assert!(covers(1..3)); + assert_eq!( + reader.poll_drop(&waiter), + Poll::Ready((1..3, track::Hole::stale(budget))), + "judged under the budget sent with update 0" + ); + + // An update that keeps the budget and the ranges still reopens the stale run. + { + let Sub::Active(active) = &mut serving.sub else { + unreachable!("an active subscription"); + }; + let ranges = active.live.unchanged(); + buffer_update(active, ranges, &serving.serving, serve.subscriber.runtime.now()).unwrap(); + assert_eq!(active.update, 1); + assert!(active.stale.is_empty()); + } + assert!(!covers(1..3), "the publisher re-offers or drops it again"); + + // Still in flight from before the update: no verdict. + serving + .apply_response(&serve, drop(4, 4, &crate::StreamError::Old, 0)) + .unwrap(); + assert!(!covers(4..5), "still owed"); + assert!(reader.poll_drop(&waiter).is_pending(), "no reader settles on it"); + + // Any other drop is final, whatever update it names. + serving + .apply_response(&serve, drop(5, 5, &crate::StreamError::App(5), 0)) + .unwrap(); + assert!(covers(5..6)); + assert_eq!( + reader.poll_drop(&waiter), + Poll::Ready((5..6, track::Hole::drop(crate::StreamError::App(5)))) + ); + + // Under the current update, a stale drop is a verdict again. + serving + .apply_response(&serve, drop(6, 6, &crate::StreamError::Old, 1)) + .unwrap(); + assert!(covers(6..7)); + assert_eq!( + reader.poll_drop(&waiter), + Poll::Ready((6..7, track::Hole::stale(budget))) + ); + } + + /// A group that arrives over a lite-07 stale (OLD) drop wins over it, whichever lands first: + /// the publisher serves a group after its stale drop, as a front that switched routes does, + /// and the group's stream may overtake the drop. The next update owes it no more, and a + /// group still arriving is never aborted by the drop. + #[kio::test] + async fn a_group_wins_over_its_stale_drop() { + let version = Version::Lite07; + let (serve, mut serving, tail, _broadcast) = lite07_serving(Duration::from_millis(500)).await; + let now = || serve.subscriber.runtime.now(); + let code = u64::from(crate::coding::StreamCodes::encode_stream_code( + &version, + &crate::StreamError::Old, + )); + let old = |sequence| { + lite::SubscribeResponse::Drop(lite::SubscribeDrop { + start: sequence, + end: sequence, + error: code, + update: 0, + }) + }; + let frame = |group: &mut crate::group::Producer| { + group.write_frame(crate::Timestamp::from_millis(1).unwrap(), &b"x"[..])?; + group.finish() + }; + let arrive = |serving: &mut ServeLoop, sequence| { + let group = serving.serving.create_group(crate::group::Info { sequence }).unwrap(); + (group, Reading::open(&tail, Some(sequence), now())) + }; + + // OLD, then the group, which finishes. + serving.apply_response(&serve, old(1)).unwrap(); + let (mut one, reading) = arrive(&mut serving, 1); + frame(&mut one).unwrap(); + drop(reading); + // The group first, still arriving when its OLD lands. + let (mut two, _two) = arrive(&mut serving, 2); + serving.apply_response(&serve, old(2)).unwrap(); + // OLD, then the group, still arriving at the update. + serving.apply_response(&serve, old(3)).unwrap(); + let (mut three, _three) = arrive(&mut serving, 3); + // OLD, and no group. + serving.apply_response(&serve, old(4)).unwrap(); + + { + let Sub::Active(active) = &mut serving.sub else { + unreachable!("an active subscription"); + }; + let ranges = active.live.unchanged(); + buffer_update(active, ranges, &serving.serving, now()).unwrap(); + } + let covers = |groups: std::ops::Range| tail.read().covers(groups); + assert!(covers(1..4), "every group that arrived stays accounted for"); + assert!(!covers(4..5), "the group that never arrived is owed again"); + frame(&mut two).expect("the drop leaves a group still arriving"); + frame(&mut three).expect("the update leaves a group still arriving"); + } + + /// A group whose stream finished wins over a lite-07 stale drop that lands after the cache + /// evicted it: the publisher served it over that drop, so the next update owes it no more. + /// A stream that finished under an earlier update says nothing of a drop under the new one. + #[kio::test] + async fn a_finished_group_wins_over_a_late_stale_drop_once_evicted() { + let version = Version::Lite07; + let (serve, mut serving, tail, _broadcast) = lite07_serving(Duration::from_millis(500)).await; + let now = || serve.subscriber.runtime.now(); + let code = u64::from(crate::coding::StreamCodes::encode_stream_code( + &version, + &crate::StreamError::Old, + )); + let old = |sequence, update| { + lite::SubscribeResponse::Drop(lite::SubscribeDrop { + start: sequence, + end: sequence, + error: code, + update, + }) + }; + serve.subscriber.subscribes.lock().insert( + 0, + TrackEntry::accepted( + serving.serving.clone(), + lite::Framing { + timescale: Some(Timescale::MILLI), + ietf: None, + compressed: false, + }, + tail.clone(), + ), + ); + // A group stream that arrives whole, through the path that records it, then is evicted. + let arrive = async |serving: &mut ServeLoop, sequence| { + use crate::transport::poll::Session as _; + + let mut script = Vec::new(); + let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); + lite::Group { + subscribe: 0, + sequence, + frame_start: 0, + } + .encode(&mut encoder, version) + .unwrap(); + encoder.varint(crate::coding::varint::zigzag(1)).unwrap(); + encoder.bytes(b"x").unwrap(); + let session = crate::lite::test_transport::ScriptedSession::eof(script); + let (_, recv) = session.open_bi(Default::default()).await.unwrap(); + let mut group = GroupRecv::new( + serve.subscriber.clone(), + Reader::new(crate::transport::boxed::RecvStream::new(recv), version), + ); + kio::wait(|waiter| group.poll_serve(waiter)).await.unwrap(); + serving.serving.evict(sequence); + }; + let update = |serving: &mut ServeLoop| { + let Sub::Active(active) = &mut serving.sub else { + unreachable!("an active subscription"); + }; + let ranges = active.live.unchanged(); + buffer_update(active, ranges, &serving.serving, now()).unwrap(); + }; + let covers = |groups: std::ops::Range| tail.read().covers(groups); + + arrive(&mut serving, 1).await; + serving.apply_response(&serve, old(1, 0)).unwrap(); + update(&mut serving); + assert!(covers(1..2), "the evicted group that arrived stays accounted for"); + + // Finished under the first update, then judged stale under the second. + arrive(&mut serving, 2).await; + update(&mut serving); + serving.apply_response(&serve, old(2, 2)).unwrap(); + update(&mut serving); + assert!( + !covers(2..3), + "a drop under a later update is the publisher's to re-offer" + ); + } + + /// A SKIP, or any final drop, that follows an OLD drop of the same groups replaces it: an + /// update reopens only stale verdicts, so the groups stay accounted for, and the skip + /// stays final rather than being superseded with the OLD drop. An empty range names + /// nothing. + #[kio::test] + async fn a_skip_or_final_drop_replaces_an_earlier_stale_one() { + let version = Version::Lite07; + let budget = Duration::from_millis(500); + let (serve, mut serving, tail, _broadcast) = lite07_serving(budget).await; + let code = u64::from(crate::coding::StreamCodes::encode_stream_code( + &version, + &crate::StreamError::Old, + )); + let old = |start, end| { + lite::SubscribeResponse::Drop(lite::SubscribeDrop { + start, + end, + error: code, + update: 0, + }) + }; + let covers = |groups: std::ops::Range| tail.read().covers(groups); + + serving.apply_response(&serve, old(1, 2)).unwrap(); + serving.apply_response(&serve, old(4, 4)).unwrap(); + serving + .apply_response( + &serve, + lite::SubscribeResponse::Skip(lite::SubscribeSkip { start: 1, end: 2 }), + ) + .unwrap(); + serving + .apply_response( + &serve, + lite::SubscribeResponse::Drop(lite::SubscribeDrop::new(4..=4, &crate::StreamError::App(5), 0, version)), + ) + .unwrap(); + // Empty ranges. + serving.apply_response(&serve, old(6, 5)).unwrap(); + serving + .apply_response( + &serve, + lite::SubscribeResponse::Skip(lite::SubscribeSkip { start: 6, end: 5 }), + ) + .unwrap(); + + { + let Sub::Active(active) = &mut serving.sub else { + unreachable!("an active subscription"); + }; + let ranges = active.live.unchanged(); + buffer_update(active, ranges, &serving.serving, serve.subscriber.runtime.now()).unwrap(); + } + assert!(covers(1..3), "the skip is final"); + assert!(covers(4..5), "the drop is final"); + assert!( + matches!( + serving.serving.create_group(crate::group::Info { sequence: 1 }), + Err(Error::Skipped) + ), + "still skipped after the update" + ); + } + + /// A SKIP replaces an earlier final DROP of the same groups too: "never produced" is the + /// most authoritative verdict. A later DROP leaves the skip in place, and a SKIP naming a + /// group that arrived keeps the group, without ending the session. + #[kio::test] + async fn a_skip_replaces_a_final_drop_but_not_a_cached_group() { + let version = Version::Lite07; + let (serve, mut serving, tail, _broadcast) = lite07_serving(Duration::from_millis(500)).await; + let drop = |start, end, error: &crate::StreamError| { + lite::SubscribeResponse::Drop(lite::SubscribeDrop::new(start..=end, error, 0, version)) + }; + let skip = |start, end| lite::SubscribeResponse::Skip(lite::SubscribeSkip { start, end }); + let mut group = serving + .serving + .create_group(crate::group::Info { sequence: 5 }) + .unwrap(); + group + .write_frame(crate::Timestamp::from_millis(5).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); + + serving + .apply_response(&serve, drop(1, 2, &crate::StreamError::App(5))) + .unwrap(); + serving.apply_response(&serve, skip(1, 2)).unwrap(); + assert!( + matches!( + serving.serving.create_group(crate::group::Info { sequence: 2 }), + Err(Error::Skipped) + ), + "the skip replaced the final drop" + ); + serving + .apply_response(&serve, drop(2, 2, &crate::StreamError::NotFound)) + .unwrap(); + assert!( + matches!( + serving.serving.create_group(crate::group::Info { sequence: 2 }), + Err(Error::Skipped) + ), + "a later drop leaves the skip" + ); + + // Group 5 arrived, so the skip names only 4 and 6. + serving.apply_response(&serve, skip(4, 6)).unwrap(); + assert!(tail.read().covers(4..7)); + let mut reader = serving + .serving + .subscribe(crate::track::Subscription::default().with_ranges([crate::track::Range::groups(0..)])); + let waiter = kio::Waiter::noop(); + assert_eq!(reader.poll_drop(&waiter), Poll::Ready((1..3, track::Hole::Skip))); + assert_eq!(reader.poll_drop(&waiter), Poll::Ready((4..5, track::Hole::Skip))); + assert_eq!(reader.poll_drop(&waiter), Poll::Ready((6..7, track::Hole::Skip))); + assert!(reader.poll_drop(&waiter).is_pending()); + let group = reader.recv_group().await.unwrap().unwrap(); + assert_eq!(group.sequence, 5, "the group that arrived is kept"); + } + /// A SUBSCRIBE_DROP ends a group still arriving now, with its Error Code on lite-07. /// Earlier versions carry an application code there, so the same bytes read as /// OLD. @@ -5082,6 +5701,7 @@ mod tests { start: 2, end: 2, error: code.into(), + update: 0, }), ] { resp.encode( @@ -6035,7 +6655,9 @@ mod tests { Err(_) => panic!("the entry holds the parked streams"), }; assert_eq!(opened, [kept], "the withdrawn group is gone"); - let Opened { mut group, .. } = entry.unpark(kept).expect("the kept group"); + let Ok(Open::Group(Opened { mut group, .. })) = entry.unpark(kept) else { + panic!("the kept group"); + }; group.finish().unwrap(); track.finish().unwrap(); // Only the kept group reaches the subscriber: the withdrawn one ended, so nothing @@ -8797,6 +9419,12 @@ struct SubStream { /// without an instance resolves whichever one wins the path when it arrives, so one /// still unanswered once another instance takes the path may be answered by it. unanswered: Option, + /// The number of the latest SUBSCRIBE (0) or SUBSCRIBE_UPDATE sent: a stale drop given + /// up under an earlier one is no verdict on the current terms. + update: u64, + /// The runs `tail` accounts for by a stale drop under `update`: the next update reopens + /// them, since the publisher re-offers or drops them again under it. + stale: RangeSet, /// Holds the session busy for as long as the subscription is open. _working: crate::session::Working, } @@ -9401,8 +10029,12 @@ impl TrackServe { match request { Some(wire) => { match sub { - // Open an upstream SUBSCRIBE for the first subscriber. - Sub::None => Ok(Begin::Establish(self.prepare_establish(producer, wire, &fills.groups))), + // Open an upstream SUBSCRIBE for the first subscriber. Its terms are new, so a + // stale run an earlier subscription reported is no verdict on them. + Sub::None => { + producer.supersede_stale(judged(&wire.ranges)); + Ok(Begin::Establish(self.prepare_establish(producer, wire, &fills.groups))) + } Sub::Active(active) => { let Wire { subscription, @@ -9463,7 +10095,7 @@ impl TrackServe { } else if start_moved { let _ = producer.start_at(active.start); } - buffer_update(active, edit)?; + buffer_update(active, edit, producer, self.subscriber.runtime.now())?; } Ok(Begin::None) } @@ -9744,9 +10376,55 @@ fn wire_max_delay(max_delay: Option) -> std::time::Duration max_delay.unwrap_or(std::time::Duration::from_millis(super::MAX_SAFE_DELAY_MS)) } -/// Buffer the current SUBSCRIBE parameters, with `ranges` the change to the live ranges. -/// The caller flushes. -fn buffer_update(active: &mut SubStream, ranges: lite::RangeEdit) -> Result<(), Error> { +/// Every group `ranges` name, which a stale drop under them was judged against. A bound only +/// the publisher resolves is taken at its widest: a start from group 0, an end past the live +/// edge (an exclusive end of `u64::MAX`). +fn judged<'a>(ranges: impl IntoIterator) -> Option> { + let mut judged = RangeSet::new(); + for range in ranges { + let start = match range.start { + Some(lite::Anchor::Absolute(start)) => start.group, + _ => 0, + }; + let end = match range.end { + Some(lite::Anchor::Absolute(end)) => end.group.saturating_add(u64::from(end.frame > 0)), + _ => u64::MAX, + }; + if start < end { + judged.insert(start..end); + } + } + Some(judged) +} + +/// Buffer the current SUBSCRIBE parameters as the next update, with `ranges` the change to +/// the live ranges. The caller flushes. +/// +/// A stale drop given up under an earlier update may still be in flight, and this one may +/// move the range or budget it was judged against, so the runs `serving` holds as stale +/// become unknown again: a reader fills them by range rather than trust an old verdict. +/// +/// A lite-07 publisher re-offers every group it reported stale on any update, and drops +/// again under the new number what is still stale, so the tail owes every stale run again, +/// from `now`. A group that arrived since its stale drop won over it: the publisher served +/// it, so it is neither re-offered nor owed again. +fn buffer_update( + active: &mut SubStream, + ranges: lite::RangeEdit, + serving: &track::Producer, + now: crate::time::Instant, +) -> Result<(), Error> { + active.update += 1; + let held = serving.supersede_stale(judged(active.live.iter().map(|(_, range)| range))); + let recorded = std::mem::take(&mut active.stale); + if let Ok(mut tail) = active.tail.write() { + tail.updated(); + for run in recorded.iter() { + for reopened in held.overlapping(run) { + tail.unaccount(reopened.start.max(run.start)..reopened.end.min(run.end), now); + } + } + } active.stream.writer.buffer(&lite::SubscribeUpdate { wanted_group: active.wanted, priority: active.priority, @@ -9754,6 +10432,7 @@ fn buffer_update(active: &mut SubStream, ranges: lite::RangeEdit) -> Result<(), min_delay: active.min_delay, ranges, order: active.order, + update: active.update, }) } @@ -9818,6 +10497,7 @@ impl Establish { min_delay: self.subscription.delay.min, ranges: self.ranges.clone(), order: self.subscription.order, + update: 0, }; stream = stream.with_stats(self.serve.subscriber.origin.stats()); stream.writer.buffer(&lite::ControlType::Subscribe)?; @@ -9876,6 +10556,8 @@ impl Establish { declared: None, end: None, wanted: None, + update: 0, + stale: RangeSet::new(), _working: self.session.work().start(), } } @@ -11113,26 +11795,89 @@ impl ServeLoop { } // The publisher will never send these groups, so they // are accounted for without a stream, and a reader of - // one still arriving ends with the drop's error now. + // one still arriving ends with the drop's error now. A + // lite-07 stale drop leaves a group whose stream arrived, + // since the stream wins over it. lite::SubscribeResponse::Drop(dropped) => { + // It can only echo an update this subscriber sent, even when it names nothing. + if dropped.update > active.update { + return Err(Error::ProtocolViolation); + } let groups = dropped.start..dropped.end.saturating_add(1); - let err = dropped.error(serve.subscriber.version); - if let Ok(mut tail) = active.tail.write() { - tail.account(groups.clone(), serve.subscriber.runtime.now()); + // An empty range (Group Start = Group End + 1) names nothing. + if groups.is_empty() { + return Ok(()); } - // A group a fill asked for and the live budget dropped as stale is recovered on - // its own, so the copy does not record it given up meanwhile. - let mut given = rangemap::RangeSet::new(); - given.insert(groups.clone()); - if err == crate::StreamError::Old { - for asked in active.fill_groups.overlapping(&groups) { - given.remove(asked.clone()); + let error = dropped.error(serve.subscriber.version); + // On lite-07 an OLD drop is a stale verdict, which the next update reopens; every + // other drop is final, whatever update it names. + let stale = serve.subscriber.version.has_skip() && error == crate::StreamError::Old; + // One given up under an update since replaced says nothing of the current terms: + // the publisher re-offers it or drops it again under the new number, so it is + // still owed, and a reader that wants it fills it by range meanwhile. + if !(stale && dropped.update < active.update) { + if let Ok(mut tail) = active.tail.write() { + tail.account(groups.clone(), serve.subscriber.runtime.now()); + } + match stale { + // Judged under the budget this subscriber sent with that update. A + // sequence already skipped stays skipped, and one whose group arrived + // keeps it, so no update reopens either. A group a fill asked for is + // recovered on its own, so the copy does not record it given up meanwhile. + true => { + let mut given = rangemap::RangeSet::new(); + given.insert(groups.clone()); + for asked in active.fill_groups.overlapping(&groups) { + given.remove(asked.clone()); + } + // A group whose stream finished under this update was served over + // the drop, so it won, even once the cache evicted it. + for finished in active.tail.read().finished(groups.clone()) { + given.remove(finished); + } + for given in given.iter() { + let recorded = self + .serving + .give_up(given.clone(), track::Hole::stale(active.max_delay)); + active.stale.extend(recorded); + } + } + // Final, so no update reopens it, even where it replaces a stale drop. + false => { + active.stale.remove(groups.clone()); + self.serving.give_up(groups.clone(), track::Hole::drop(error.clone())); + } } + self.range_fills.refuse(groups, error); + } + } + // Never produced, so nothing is waited for and no reader ends. A skip is final and + // replaces any drop of the same groups, stale or final, so no update reopens them; + // a reader a drop already ended stays ended. + lite::SubscribeResponse::Skip(skipped) => { + let groups = skipped.start..skipped.end.saturating_add(1); + // An empty range (Group Start = Group End + 1) names nothing. + if groups.is_empty() { + return Ok(()); } - for given in given.iter() { - self.serving.drop_groups(given.clone(), err.clone()); + if let Ok(mut tail) = active.tail.write() { + tail.account(groups.clone(), serve.subscriber.runtime.now()); + } + active.stale.remove(groups.clone()); + // A fill owing them is answered: the source never had them. + self.range_fills.refuse(groups.clone(), crate::StreamError::NotFound); + // A group that arrived was produced after all, so the skip contradicts it: it + // keeps the group, and the session goes on, since the draft requires no more. + let cached = self.serving.skip_received(groups); + if let Some(first) = cached.iter().next() { + tracing::warn!( + track = %serve.name, + start = skipped.start, + end = skipped.end, + cached = first.start, + "protocol error: SUBSCRIBE_SKIP names a group that arrived; keeping the group" + ); } - self.range_fills.refuse(groups, err); } // OK just resolves the range (the producer already orders // groups). @@ -11399,7 +12144,9 @@ impl ServeLoop { { active.wanted = Some(wanted); let ranges = active.live.unchanged(); - if let Err(err) = buffer_update(active, ranges) { + if let Err(err) = + buffer_update(active, ranges, &self.serving, serve.subscriber.runtime.now()) + { return Poll::Ready(ServeEnd::GiveBack(err)); } continue; diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 20f33e84d05..dc915fc6169 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -298,6 +298,17 @@ impl Version { ) } + /// Whether holes split into SUBSCRIBE_SKIP, for sequences declared never produced, and + /// SUBSCRIBE_DROP, for groups that existed, which names the `Update` in effect when it was + /// given up; SUBSCRIBE and SUBSCRIBE_UPDATE carry that `Update` number. Added in lite-07; + /// earlier versions name every hole in SUBSCRIBE_DROP. + pub(crate) fn has_skip(self) -> bool { + !matches!( + self, + Self::Lite01 | Self::Lite02 | Self::Lite03 | Self::Lite04 | Self::Lite05 | Self::Lite06 + ) + } + /// Whether SUBSCRIBE_FINISH carries the error the track ended with, so an errored end /// keeps its history across a hop. Added in lite-07; earlier versions can only reset. pub(crate) fn has_end_error(self) -> bool { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f97c45212c0..2575aea9e15 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -393,6 +393,7 @@ impl Consumer { groups_read: false, out: Vec::new(), drops: RangeMap::new(), + reported_stale: RangeSet::new(), declared: None, declared_error: None, refused: false, @@ -881,6 +882,20 @@ impl Subscriber { reader.window.covered_by(from, cap, accounted) } + /// Fresh reads of the groups in `run` the serving route's copy still caches and judges + /// fresh, and the runs still given up; see [`track::Subscriber::reoffer_groups`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(super) fn reoffer_groups(&mut self, run: std::ops::Range) -> (Vec, RangeSet) { + self.reader().reoffer_groups(run) + } + + /// Take back every stale run this reader queued or the serving route's copy reported; + /// see [`track::Subscriber::reoffer_stale`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(super) fn reoffer_stale(&mut self) -> Vec> { + self.reader().reoffer_stale() + } + /// Poll for sequences the serving copy gave up that no copy may still deliver; see /// [`track::Subscriber::poll_drop`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -889,7 +904,7 @@ impl Subscriber { delivered: &Delivered, probe: bool, waiter: &kio::Waiter, - ) -> Poll<(std::ops::Range, StreamError)> { + ) -> Poll<(std::ops::Range, track::Hole)> { self.reader().poll_drop(delivered, probe, waiter) } @@ -1041,7 +1056,12 @@ struct Reader { out: Vec<(u64, Weak<()>)>, /// Runs the serving copy gave up that the reader has yet to report; see /// [`Reader::poll_drop`]. - drops: RangeMap, + /// Each with whether it is a received stale run the serving copy held, which new terms + /// can supersede. + drops: RangeMap, + /// The stale runs reported since the last [`Reader::reoffer_stale`], whichever copy gave + /// them up: an update reopens each, and the copy that reported it may no longer serve. + reported_stale: RangeSet, } impl Reader { @@ -1401,6 +1421,11 @@ impl Reader { fn hand_out(&mut self, index: usize, group: group::Consumer) -> group::Consumer { let sequence = group.sequence; self.newest = self.newest.max(Some(sequence)); + // The group won over any stale report on it, whichever copy made it, so no update + // reopens it. + if let Some(end) = sequence.checked_add(1) { + self.reported_stale.remove(sequence..end); + } // Another copy holding the group, now or once it lands there, is where a failure // continues it, so it stays while the group is read; see [`Copy::retains`]. Recorded // even with one copy, since a later route's copy may come to hold it. @@ -1712,6 +1737,48 @@ impl Reader { } } + /// Fresh reads of the groups in `run` the serving copy still caches and judges fresh, + /// handed out as any group of that copy is, and the runs still given up: all of `run` + /// while no copy serves. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + fn reoffer_groups(&mut self, run: std::ops::Range) -> (Vec, RangeSet) { + let Poll::Ready(Some(sub)) = self.poll_serving(&kio::Waiter::noop()) else { + let mut given_up = RangeSet::new(); + given_up.insert(run); + return (Vec::new(), given_up); + }; + let (groups, given_up) = sub.reoffer_groups(run); + // The serving copy is the newest one. + let index = self.copies.len() - 1; + let groups = groups.into_iter().map(|group| self.hand_out(index, group)).collect(); + (groups, given_up) + } + + /// Take back every stale run: those queued here, those this reader reported, and those + /// the serving copy reported, which its cursor owes again. What this reader reported + /// counts whichever copy gave it up, since a route change since leaves the serving copy + /// knowing nothing of it, though the update reopened it. Only the serving copy is + /// judged again: a replaced copy may no longer deliver, so what it gave up is the new + /// route's to deliver or give up. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + fn reoffer_stale(&mut self) -> Vec> { + let mut runs = std::mem::take(&mut self.reported_stale); + let queued: Vec<_> = self + .drops + .iter() + .filter(|(_, (hole, _))| hole.stale_budget().is_some()) + .map(|(run, _)| run.clone()) + .collect(); + for run in queued { + self.drops.remove(run.clone()); + runs.insert(run); + } + if let Poll::Ready(Some(sub)) = self.poll_serving(&kio::Waiter::noop()) { + runs.extend(sub.reoffer_stale()); + } + runs.into_iter().collect() + } + /// The next run the serving copy gave up that `delivered` lacks and no other copy may /// still deliver. Only the serving copy's verdicts count, and a route failure is none: /// another route may still deliver the group, and a failed group it buffered continues @@ -1723,21 +1790,38 @@ impl Reader { delivered: &Delivered, probe: bool, waiter: &kio::Waiter, - ) -> Poll<(std::ops::Range, StreamError)> { + ) -> Poll<(std::ops::Range, track::Hole)> { + let queued: Vec<_> = self + .drops + .iter() + .filter(|(_, (_, received))| *received) + .map(|(run, _)| run.clone()) + .collect(); let mut given_up = Vec::new(); + let mut superseded = Vec::new(); if let Poll::Ready(Some(sub)) = self.poll_serving(waiter) { + // A received stale run queued here that new terms superseded since is unknown again: + // the copy no longer holds it, and its upstream re-offers or drops it again. + for run in queued { + superseded.extend(sub.stale_held(run.clone()).gaps(&run)); + } loop { let polled = match probe { true => sub.poll_drop(waiter), false => sub.poll_queued_drop(waiter), }; - let Poll::Ready(run) = polled else { break }; - given_up.push(run); + let Poll::Ready((run, hole)) = polled else { break }; + // A stale run the copy holds came from its upstream, which new terms may supersede. + let received = hole.stale_budget().is_some() && !sub.stale_held(run.clone()).is_empty(); + given_up.push((run, hole, received)); } } - for (run, err) in given_up { - if !Error::Stream(err.clone()).retryable() { - self.drops.insert(run, err); + for run in superseded { + self.drops.remove(run); + } + for (run, hole, received) in given_up { + if !Error::Stream(hole.error()).retryable() { + self.drops.insert(run, (hole, received)); } } let covered: Vec<_> = self @@ -1771,7 +1855,7 @@ impl Reader { feeding.insert(from..until); } } - let next = self.drops.iter().find_map(|(run, err)| { + let next = self.drops.iter().find_map(|(run, (err, _))| { let run = run.start.max(start)..run.end.min(end.unwrap_or(u64::MAX)); let open = delivered .gaps(run) @@ -1785,6 +1869,11 @@ impl Reader { // Reported once. As on a plain cursor, the report is not final: a miss (a fill given // up) may still land, and is handed out if it does. self.drops.remove(open.clone()); + match &err { + err if err.stale_budget().is_some() => self.reported_stale.insert(open.clone()), + // A skip or final drop is final, so no update reopens a stale report it replaced. + track::Hole::Skip | track::Hole::Drop { .. } => self.reported_stale.remove(open.clone()), + } Poll::Ready((open, err)) } @@ -2572,6 +2661,13 @@ mod test { ) } + /// A route's copy received from a remote publisher, whose groups may land out of order. + fn received_copy() -> track::Producer { + let copy = copy(); + copy.received(); + copy + } + /// The logical track `routes` serves. fn logical(routes: &Producer) -> track::Producer { track::Request::new(Arc::new(broadcast::Info::default()), "video") @@ -3725,8 +3821,8 @@ mod test { /// Every run the reader gives up right now. fn dropped(sub: &mut track::Subscriber) -> Vec<(std::ops::Range, StreamError)> { let mut runs = Vec::new(); - while let Poll::Ready(run) = sub.poll_drop(&kio::Waiter::noop()) { - runs.push(run); + while let Poll::Ready((run, hole)) = sub.poll_drop(&kio::Waiter::noop()) { + runs.push((run, hole.error())); } runs } @@ -3755,6 +3851,34 @@ mod test { assert_eq!(dropped(&mut sub), []); } + /// A skip of a run the serving copy already dropped replaces the drop, so the resumed + /// reader reports it again, as a skip, and only once. + #[kio::test] + async fn a_resumed_reader_reports_a_skip_over_a_reported_drop() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + let holes = |sub: &mut track::Subscriber| { + let mut runs = Vec::new(); + while let Poll::Ready(run) = sub.poll_drop(&kio::Waiter::noop()) { + runs.push(run); + } + runs + }; + + a.drop_groups(1..3, StreamError::NotFound); + assert_eq!(holes(&mut sub), [(1..3, track::Hole::drop(StreamError::NotFound))]); + a.skip_groups(1..3); + assert_eq!(holes(&mut sub), [(1..3, track::Hole::Skip)]); + assert_eq!(holes(&mut sub), []); + write(&a, 3); + assert_eq!(drain(&mut sub, false), [3]); + } + /// A run the serving copy gave up waits while a copy it replaced may still deliver it /// under its bound, and goes out once that copy is past the reader's budget. #[kio::test] @@ -3786,6 +3910,120 @@ mod test { assert_eq!(drain(&mut sub, false), Vec::::new()); } + /// A received stale run the reader held back is unknown again once new terms supersede it + /// on the serving copy: its upstream re-offers or drops it again, so it is not reported + /// meanwhile. + #[kio::test] + async fn a_held_back_stale_run_new_terms_supersede_is_not_reported() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + // The new route's upstream judged groups 1 and 2 stale; the old route may still deliver them. + b.give_up(1..3, track::Hole::stale(budget)); + assert_eq!(dropped(&mut sub), []); + write(&b, 3); + assert_eq!(drain(&mut sub, false), [3]); + + // The route's subscription updates before the old copy is past the budget. + b.supersede_stale(None); + kio::clock::sleep(budget).await; + assert_eq!(dropped(&mut sub), [], "no verdict under the new terms"); + } + + /// A front's reader re-offers what its serving route's copy still caches and judges fresh, + /// and gives up the rest, so a relay's publisher re-serves an expired stream through it. + #[test] + fn a_resumed_reader_reoffers_a_group_its_serving_copy_still_caches() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + let (groups, given_up) = sub.reoffer_groups(0..3); + assert_eq!(groups.iter().map(|group| group.sequence).collect::>(), [0]); + assert_eq!(given_up, std::iter::once(1..3).collect::>()); + } + + /// A front's reader takes back the stale runs its serving route's copy reported, so a + /// relay's publisher re-offers them under a new update rather than keep them reported. + #[test] + fn a_resumed_reader_takes_back_its_stale_reports() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + a.give_up(1..3, track::Hole::stale(HOUR)); + assert_eq!(dropped(&mut sub), [(1..3, StreamError::Old)]); + assert_eq!(sub.reoffer_stale(), vec![1..3]); + assert_eq!(sub.reoffer_stale(), Vec::>::new()); + } + + /// A stale run one route reported is still taken back once another serves: the update + /// reopened it downstream, and the new route's copy knows nothing of it, so the groups it + /// delivers there afterwards are owed rather than kept reported. + #[test] + fn a_resumed_reader_takes_back_a_replaced_routes_stale_reports() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + a.give_up(1..3, track::Hole::stale(HOUR)); + assert_eq!(dropped(&mut sub), [(1..3, StreamError::Old)]); + + let b = copy(); + routes.serve(b.consume()); + assert_eq!(dropped(&mut sub), []); + assert_eq!( + sub.reoffer_stale(), + vec![1..3], + "reopened, though the new route never reported it" + ); + assert_eq!(sub.reoffer_stale(), Vec::>::new()); + write(&b, 1); + write(&b, 2); + assert_eq!(drain(&mut sub, false), [1, 2]); + } + + /// A group the new route delivers over a stale run the old route reported retires that + /// report, and a final drop that follows it replaces it: only what is still stale is taken + /// back, so an update reopens no group already served. + #[test] + fn a_resumed_reader_retires_a_stale_report_its_group_or_final_drop_replaces() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + a.give_up(1..4, track::Hole::stale(HOUR)); + assert_eq!(dropped(&mut sub), [(1..4, StreamError::Old)]); + a.drop_groups(2..3, StreamError::App(5)); + assert_eq!(dropped(&mut sub), [(2..3, StreamError::App(5))]); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + assert_eq!(sub.reoffer_stale(), vec![3..4], "only group 3 is still stale"); + } + /// A run a replaced route held back is dropped with that route: the route serving next /// may still deliver it. #[test] @@ -4720,7 +4958,7 @@ mod test { fn a_clean_end_waits_for_a_group_the_replaced_copy_still_carries() { let routes = Producer::new(); let logical = logical(&routes); - let a = copy(); + let a = received_copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(1)); write(&a, 0); @@ -4748,7 +4986,7 @@ mod test { for ordered in [false, true] { let routes = Producer::new(); let logical = logical(&routes); - let a = copy(); + let a = received_copy(); let pool = a.broadcast().pool.clone(); let opened = a.consume().now(); let noop = kio::Waiter::noop(); @@ -4802,7 +5040,7 @@ mod test { fn a_read_waiting_past_a_drained_copy_wakes_when_the_route_changes() { let routes = Producer::new(); let logical = logical(&routes); - let a = copy(); + let a = received_copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(1)); write(&a, 0); diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index 7236e9a7f3c..e2277091071 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -456,6 +456,8 @@ impl Model { let requests = self.origin.forward(PATH, route, None).unwrap(); let broadcast = broadcast::Info::new().produce(); let track = broadcast.create_track("video", track::Info::timed()).unwrap(); + // The route's live feed, received from its publisher: its groups land in any order. + track.received(); let upstream = broadcast.consume(); let serving = kio::clock::sim::spawn(async move { while let Some(request) = kio::wait(|waiter| requests.poll_requested(waiter)).await { @@ -719,7 +721,7 @@ impl Model { if reader.reading.contains_key(&group) { return Err(fail( index, - format!("dropped {range:?} ({err}) after handing out {group}"), + format!("dropped {range:?} ({err:?}) after handing out {group}"), )); } } diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index b39073a1bd2..e726c33773d 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -67,6 +67,19 @@ impl Skipped { widened } + /// The skipped runs within `groups`, clipped to them, lowest first; each stays until its + /// group is handed out or skipped again. + pub fn within(&self, groups: RangeInclusive) -> Vec> { + let Some(skipped) = &self.0 else { + return Vec::new(); + }; + skipped + .runs + .overlapping(&groups) + .map(|(run, _)| *run.start().max(groups.start())..=*run.end().min(groups.end())) + .collect() + } + /// Whether no run is recorded. pub fn is_empty(&self) -> bool { self.0.as_ref().is_none_or(|skipped| skipped.runs.is_empty()) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 58d862f3b0b..0ee302943ad 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -32,9 +32,11 @@ pub(super) use demand::Preferences; use demand::Subscriptions; mod fill; +mod hole; mod miss; pub use fill::Fill; use fill::FillState; +pub(crate) use hole::Hole; #[cfg_attr(not(feature = "async"), allow(unused_imports, reason = "async-only"))] pub(crate) use fill::{GroupFill, Readers as FillReaders}; @@ -506,12 +508,24 @@ pub(crate) struct TrackState { // budget, but neither this copy's timestamps nor a fill can judge an older group against // a reader's budget. A widened budget reaches lower groups through the feed's re-offer. fills_from_start: bool, - // Sequences the upstream publisher gave up, with the error each was given up with: an - // incoming SUBSCRIBE_DROP or a producer cut. Kept apart from the cache so reclaiming a - // slot cannot erase a disposition a relay must forward, and as ranges so a wide DROP - // costs one entry. - drops: RangeMap, - // Bumped on every insert into `drops`, so a cursor rescans it only after a change. + // Sequences the upstream publisher gave up, with why: an incoming SUBSCRIBE_SKIP or + // SUBSCRIBE_DROP, or a producer cut. Kept apart from the cache so reclaiming a slot + // cannot erase a disposition a relay must forward, and as ranges so a wide one costs + // one entry. + drops: RangeMap, + // What this track's own publisher produced since its last `append_group`, which skips + // every sequence below it that never arrived and was not given up: the producer moved on. + // Arrivals cannot say so, since a gap folds into them once nobody waits for it. Only + // sequences at or above `appended` are kept, so it holds the runs created since then. + produced: RangeSet, + // One past the last `append_group`: every sequence below it arrived, was given up, or + // was skipped. + appended: Option, + // Every group the ranges of a received copy's current upstream terms name, which its + // stale runs were judged against; `None` when unknown, which counts as every group. + judged: Option>, + // Bumped on every insert into `drops`, and when `judged` changes, so a cursor rescans + // it only after a change. drops_revision: u64, drops_changes: ledger::Changes, // How many copies the cache took in that the live feed didn't deliver: fetched backfill, @@ -1138,18 +1152,102 @@ impl TrackState { } } - /// Record sequences the upstream publisher gave up, with `err`. - fn drop_groups(&mut self, groups: ops::Range, err: StreamError) { + /// Record sequences the upstream publisher gave up, with why. A skip is final and the + /// most authoritative verdict, so a drop never replaces one; see [`Self::skip_groups`]. + /// Returns the runs recorded. + fn drop_groups(&mut self, groups: ops::Range, hole: Hole) -> RangeSet { + let mut runs = RangeSet::new(); if groups.is_empty() { - return; + return runs; + } + runs.insert(groups.clone()); + if hole != Hole::Skip { + for (run, given) in self.drops.overlapping(&groups) { + if *given == Hole::Skip { + runs.remove(run.clone()); + } + } + } + self.cache.wakes().arrived(groups); + for run in runs.iter() { + self.change_drops(run.clone(), Some(hole.clone())); + } + runs + } + + /// Whether the publisher declared `sequence` skipped: never produced, which is final. + fn skipped(&self, sequence: u64) -> bool { + self.drops.get(&sequence).is_some_and(|hole| *hole == Hole::Skip) + } + + /// Skip what of `groups` holds no cached group, replacing any drop of it, final or stale: + /// "never produced" is the most authoritative verdict on a sequence. A reader a drop + /// already ended stays ended, and a relay forwards the skip over the drop it reported. + /// A group that arrived is never overwritten, since it was produced after all. Returns + /// the cached sequences left alone. See [`Producer::skip_groups`]. + fn skip_groups(&mut self, groups: ops::Range) -> RangeSet { + let mut cached = RangeSet::new(); + if groups.is_empty() { + return cached; + } + let mut left = RangeSet::new(); + left.insert(groups.clone()); + for (sequence, _) in self.lookup.range(groups.clone()) { + if let Some(end) = sequence.checked_add(1) { + left.remove(*sequence..end); + cached.insert(*sequence..end); + } + } + // Already skipped: nothing changes, so no reader rescans. + for (run, hole) in self.drops.overlapping(&groups) { + if *hole == Hole::Skip { + left.remove(run.clone()); + } + } + for run in left.iter().cloned().collect::>() { + self.drop_groups(run, Hole::Skip); } - self.cache.wakes().arrived(groups.clone()); - self.change_drops(groups, Some(err)); + cached + } + + /// Settle an append at `sequence`: skip every sequence below it that this track's own + /// publisher never produced, from just past its previous append (before the first, from + /// the lowest it produced), since appending says it will not go back. The skip replaces a + /// drop given there, as [`Self::skip_groups`] does. + fn appended(&mut self, sequence: u64) { + if let Some(from) = self.appended.or(self.produced.iter().next().map(|run| run.start)) + && from < sequence + { + let mut left = RangeSet::new(); + left.insert(from..sequence); + for run in self.produced.iter() { + left.remove(run.clone()); + } + for run in left.iter().cloned().collect::>() { + self.skip_groups(run); + } + } + self.produced.clear(); + self.appended = sequence.checked_add(1); + } + + /// Forget every run held as stale; see [`Producer::supersede_stale`]. + fn supersede_stale(&mut self) -> RangeSet { + let stale: RangeSet = self + .drops + .iter() + .filter(|(_, hole)| hole.stale_budget().is_some()) + .map(|(run, _)| run.clone()) + .collect(); + for run in stale.iter() { + self.change_drops(run.clone(), None); + } + stale } /// Keep the change index aligned with the range map, including the neighbours an /// insertion coalesces and the two pieces a removal can leave. - fn change_drops(&mut self, groups: ops::Range, err: Option) { + fn change_drops(&mut self, groups: ops::Range, hole: Option) { let mut affected = groups.start.saturating_sub(1)..groups.end.saturating_add(1); let old: Vec<_> = self.drops.overlapping(&affected).map(|(run, _)| run.clone()).collect(); for run in old { @@ -1157,8 +1255,8 @@ impl TrackState { affected.start = affected.start.min(run.start); affected.end = affected.end.max(run.end); } - match err { - Some(err) => self.drops.insert(groups, err), + match hole { + Some(hole) => self.drops.insert(groups, hole), None => self.drops.remove(groups), } for (run, _) in self.drops.overlapping(&affected) { @@ -1175,7 +1273,7 @@ impl TrackState { && let Some(err) = slot.group.cut_error() && let Some(end) = sequence.checked_add(1) { - self.drop_groups(sequence..end, StreamError::from(err)); + self.drop_groups(sequence..end, Hole::drop(StreamError::from(err))); } } @@ -1184,7 +1282,7 @@ impl TrackState { if let Some(err) = self.lookup.get(&sequence).and_then(Slot::cut) { return Some(err.clone()); } - self.drops.get(&sequence).cloned().map(Error::Stream) + self.drops.get(&sequence).map(|hole| Error::Stream(hole.error())) } /// Whether a Producer was minted, committed or not. @@ -2049,7 +2147,7 @@ impl TrackState { .filter_map(|(&sequence, _)| Some(sequence..sequence.checked_add(1)?)) .collect(); for groups in open { - self.drop_groups(groups, dropped.clone()); + self.drop_groups(groups, Hole::drop(dropped.clone())); } self.lookup .retain(|_, slot| slot.copies().any(group::Handle::is_finished)); @@ -2085,6 +2183,12 @@ impl TrackState { /// Record that the live feed delivered `sequence`, then fold the gaps that aged out. fn record_arrival(&mut self, sequence: u64, now: crate::time::Instant) { self.arrivals.arrive(sequence, now); + if !self.received + && self.appended.is_none_or(|appended| sequence >= appended) + && let Some(end) = sequence.checked_add(1) + { + self.produced.insert(sequence..end); + } // A feed still delivering keeps the gap above its last arrival open. if self.tail.is_some() { self.tail = Some(now); @@ -2703,10 +2807,12 @@ impl TrackState { self.readable() || self.unfed && window.end().is_some() } - /// Whether the live edge reached `fin`, or a received copy's tail decides it instead: - /// what it still misses up to `fin` is waited for only as long as the grace. + /// Whether the live edge reached `fin`, counting a gap up to it that was wholly given up + /// (skipped or dropped), or a received copy's tail decides it instead: what it still + /// misses up to `fin` is waited for only as long as the grace. fn reaches(&self, fin: u64) -> bool { - self.edge() >= fin || self.tail.is_some() + let edge = self.edge(); + edge >= fin || self.tail.is_some() || self.drops.gaps(&(edge..fin)).next().is_none() } /// Whether the largest can no longer advance while a producer is still held: the track @@ -3045,6 +3151,10 @@ impl TrackState { { return Err(Error::Closed); } + // Never produced, so no fill holds it either. + if self.skipped(sequence) { + return Err(Error::Skipped); + } // Adopt the supplied info only if the track hasn't been accepted yet. Groups // created here charge the same account as any other, so backfill written @@ -3377,6 +3487,16 @@ fn largest_ended(state: &TrackState, waiter: &kio::Waiter) -> Poll<()> { } } +/// Where a received copy of a group goes. +pub(crate) enum Received { + /// A new group, for the copy to fill. + Group(group::Producer), + /// Another copy of the sequence that is still arriving: whether this one is needed + /// depends on how that one settles. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + Held(group::Handle), +} + /// A producer for a track, used to create new groups. #[derive(Clone)] pub struct Producer { @@ -3464,28 +3584,47 @@ impl Producer { /// Create a new group with the given sequence number. /// + /// It says nothing about the sequences around it: groups may be created out of order, and + /// an aborted or evicted sequence created again. A gap it leaves is skipped only once the + /// producer says it will not fill it, with [`Self::skip_groups`] or [`Self::append_group`]. + /// A sequence already skipped is refused with [`Error::Skipped`]. + /// /// Readers and group demand reach it at once, but the track counts it as content (its /// live edge, or the successor that ages the group before it) only once its first frame /// opens or it finishes. pub fn create_group(&self, group: group::Info) -> Result { - self.create_group_inner(group, None) + match self.create_group_inner(group, None)? { + Received::Group(group) => Ok(group), + Received::Held(_) => unreachable!("only a received copy waits on another"), + } } /// A received copy starts at its wire offset before any reader can see it. + /// + /// A copy that already answers from `start` holds the sequence: a finished one refuses + /// this copy with [`Error::Duplicate`], and one still arriving is returned to wait on. #[cfg(feature = "async")] - pub(crate) fn create_group_at(&self, group: group::Info, start: u64) -> Result { + pub(crate) fn create_group_at(&self, group: group::Info, start: u64) -> Result { self.create_group_inner(group, Some(start)) } - fn create_group_inner(&self, info: group::Info, received: Option) -> Result { + fn create_group_inner(&self, info: group::Info, received: Option) -> Result { let mut state = self.modify()?; if state.sealed || state.final_sequence().is_some_and(|fin| info.sequence >= fin) { return Err(Error::Closed); } + // A skip says the sequence will never be produced, so it is final. A drop is not: a + // publisher may re-create a group it cut or aborted. + if state.skipped(info.sequence) { + return Err(Error::Skipped); + } let start = received.unwrap_or(0); let content = if received.is_some() { - if state.holding_group(info.sequence, start).is_some() { - return Err(Error::Duplicate); + if let Some(holding) = state.holding_group(info.sequence, start) { + return match holding.is_settled() { + true => Err(Error::Duplicate), + false => Ok(Received::Held(holding.clone())), + }; } state.content_for_copy(info.sequence)? } else { @@ -3505,11 +3644,17 @@ impl Producer { let deferred = state.deferred(); drop(state); deferred.wake(); - Ok(group) + Ok(Received::Group(group)) } /// Create a new group with the next sequence number: one past this producer's own writes. /// + /// Appending says the producer will not go back, so it skips every sequence below the new + /// group that it never produced, from just past its previous append (before the first, + /// from the lowest group it created), as [`Self::skip_groups`] would, replacing any drop + /// there. Those sequences are then final: creating one is refused. Appending alone never + /// leaves such a gap; one left by out-of-order [`Self::create_group`] calls is what it names. + /// /// The broadcast remembers nothing of an earlier producer of the name, so a replacement /// starts at 0. Since a name always means the same content, a replacement continues past /// its predecessor's groups with [`Self::create_group`]. @@ -3521,6 +3666,9 @@ impl Producer { { return Err(Error::Closed); } + if state.skipped(sequence) { + return Err(Error::Skipped); + } let track = state.info.clone().unwrap(); @@ -3528,6 +3676,7 @@ impl Producer { group::Producer::new(group::Info { sequence }, track, state.cache.clone()).with_meter(self.stats.meter()); state.cap(&group.handle())?; state.commit_group(&group.handle()); + state.appended(sequence); let deferred = state.deferred(); drop(state); deferred.wake(); @@ -3697,31 +3846,118 @@ impl Producer { /// Record that the publisher gave up `groups` with `err`: none will arrive, so a reader /// waiting on one moves past it, a group still arriving in them ends with `err`, the /// declared end does not wait for them, and a relay serving this track forwards the drop. - pub fn drop_groups(&self, groups: ops::Range, err: StreamError) { + #[cfg(any(test, feature = "fuzz"))] + pub(crate) fn drop_groups(&self, groups: ops::Range, err: StreamError) { + self.give_up(groups, Hole::drop(err)); + } + + /// Declare that this producer will never produce `groups`: none will arrive, so a reader + /// waiting on one moves past it, the declared end does not wait for them, and a relay + /// serving this track forwards the skip (lite-07 SUBSCRIBE_SKIP). Unlike a drop, it ends + /// no reader. + /// + /// This and [`Self::append_group`] are the only sources of a skip: a gap between created + /// groups says nothing, since groups may be created out of order, so only the producer + /// knows a sequence will never exist. A skip is final: creating a group at a skipped + /// sequence is refused with [`Error::Skipped`], and a later drop of it is ignored. + /// + /// A skip replaces any drop of the same sequences, whatever its error: a reader the drop + /// already ended stays ended, and every later reader and relay sees the skip. A group + /// already cached keeps what it is, since it was produced after all. + pub fn skip_groups(&self, groups: ops::Range) { + self.skip_received(groups); + } + + /// [`Self::skip_groups`], returning the cached sequences it left alone: a skip naming a + /// group that arrived contradicts it, which a subscriber reports. + pub(crate) fn skip_received(&self, groups: ops::Range) -> RangeSet { + let Ok(mut state) = self.modify() else { + return RangeSet::new(); + }; + let cached = state.skip_groups(groups); + let deferred = state.deferred(); + drop(state); + deferred.wake(); + cached + } + + /// Forget every run held as stale: the terms a received copy's upstream judged them + /// under were replaced, so they are unknown again, and a reader that wants one fills it. + /// `judged` is every group the new terms' ranges name, `None` when unknown. Returns the + /// runs forgotten; a group that arrived since its stale drop replaced it, so it is not + /// among them. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn supersede_stale(&self, judged: Option>) -> RangeSet { + let Ok(mut state) = self.modify() else { + return RangeSet::new(); + }; + if state.judged != judged { + state.judged = judged; + state.drops_revision += 1; + } + state.supersede_stale() + } + + /// Record `groups` given up as `hole` describes, returning the runs recorded: a drop + /// leaves a skipped sequence skipped, and a stale verdict leaves a group that arrived and + /// was not aborted, since a group stream wins over its drop. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn give_up(&self, groups: ops::Range, hole: Hole) -> RangeSet { + let error = match &hole { + Hole::Drop { error, .. } => error.clone(), + Hole::Skip => { + let mut runs = RangeSet::new(); + runs.insert(groups.clone()); + for cached in self.skip_received(groups).iter() { + runs.remove(cached.clone()); + } + return runs; + } + }; if groups.is_empty() { - return; + return RangeSet::new(); } - let open: Vec = match self.modify() { + let (runs, open): (_, Vec) = match self.modify() { Ok(mut state) => { - state.drop_groups(groups.clone(), err.clone()); // Only the live feed's copy: a drop judges the subscription, so a fetched head // or fetch-only backfill may still complete. - state + let open: Vec<_> = state .lookup - .range(groups) - .filter(|(_, slot)| slot.visible) - .map(|(_, slot)| &slot.group) - .filter(|group| !group.is_aborted()) - .cloned() - .collect() + .range(groups.clone()) + .filter(|(_, slot)| slot.visible && !slot.group.is_aborted()) + .map(|(&sequence, slot)| (sequence, slot.group.clone())) + .collect(); + match hole.stale_budget() { + // A lite-07 publisher may serve a group after its stale drop, and the group's + // stream may land first: the group wins, still arriving or not. + Some(_) => { + let mut left = RangeSet::new(); + left.insert(groups); + for (sequence, _) in &open { + if let Some(end) = sequence.checked_add(1) { + left.remove(*sequence..end); + } + } + let mut runs = RangeSet::new(); + for run in left.iter() { + runs.extend(state.drop_groups(run.clone(), hole.clone()).iter().cloned()); + } + (runs, Vec::new()) + } + None => ( + state.drop_groups(groups, hole), + open.into_iter().map(|(_, group)| group).collect(), + ), + } } - Err(_) => return, + Err(_) => return RangeSet::new(), }; // Settle the readers of a group still arriving now, rather than once its stream's // reset lands, with the track guard released. for group in open { - group.abort_open(Error::Stream(err.clone())); + group.abort_open(Error::Stream(error.clone())); } + runs } /// Readers get nothing from the cache until [`Self::set_live`]: the upstream @@ -6057,6 +6293,57 @@ struct Drift { /// The budget is the track's max age bound, which no widening passes: its verdicts are final. bounded: bool, edge: Edge, + /// The groups this reader reads; see [`Coverage`]. + coverage: Coverage, +} + +/// The groups a reader reads, and the exclusive end it stops at, or `None` before its ranges +/// resolve. +type Coverage = Option<(Layered, Option)>; + +/// Whether a reader judging under `budget` with `coverage` is settled by `hole` over a run +/// from `start`, on a track whose upstream judged stale runs against ranges naming the groups +/// in `judged` (`None` for every group). Any hole but a received stale one settles every +/// reader. A stale verdict holds for a reader no wider than the budget it was judged under +/// that reads every group the upstream's ranges name above the run's start: its own successor +/// and edge are then no older than those the verdict was judged against, since a group's +/// verdict looks only above it. Any other reader treats the run as unknown, and fills it. +fn settles(budget: Duration, coverage: &Coverage, start: u64, hole: &Hole, judged: Option<&RangeSet>) -> bool { + let Some(stale) = hole.stale_budget() else { + return true; + }; + let Some((groups, cap)) = coverage else { + return false; + }; + let Some(above) = start.checked_add(1) else { + return budget <= stale; + }; + // Whether the reader reads every group of `run`, an exclusive end of `u64::MAX` past the + // live edge. + let reads = |run: ops::Range| { + let open = run.end == u64::MAX; + if cap.is_some_and(|cap| open || run.end > cap) { + return false; + } + groups.first(run.start, run.start).is_some_and(|held| match open { + true => *held.end() == u64::MAX, + false => *held.end() >= run.end - 1, + }) + }; + let covers = match judged { + Some(judged) => judged + .overlapping(&(above..u64::MAX)) + .all(|run| reads(run.start.max(above)..run.end)), + None => reads(above..u64::MAX), + }; + budget <= stale && covers +} + +impl Drift { + /// Whether `hole`, over a run from `start`, settles this reader on `state`; see [`settles`]. + fn settles(&self, state: &TrackState, start: u64, hole: &Hole) -> bool { + settles(self.budget, &self.coverage, start, hole, state.judged.as_ref()) + } } /// Keeps one handed-out group tied to the subscription whose cursor selected it. @@ -6792,13 +7079,35 @@ struct Heads { filled: BTreeMap, } -/// The sequences a [`DropCursor`] reports from: a cursor's span, narrowed to its window. +/// The sequences a [`DropCursor`] reports from: a cursor's span, narrowed to its window, and +/// the drift that says which received stale runs settle it. struct Within<'a> { span: ops::Range, window: &'a Window, revision: u64, + drift: Option<&'a Drift>, +} + +/// A run a [`DropCursor`] has yet to report, and whether it was copied from the track's +/// runs, which a rescan folds in afresh: one the track forgot, such as a stale run new terms +/// superseded, leaves nothing behind. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Pending { + hole: Hole, + received: bool, +} + +impl Pending { + /// A verdict this cursor reached itself. + fn own(hole: Hole) -> Self { + Self { hole, received: false } + } } +/// The [`TrackState::drops_revision`], span, window revision, and the budget and coverage +/// that settle received stale runs (see [`Drift::settles`]) a [`DropCursor`] folded in. +type Scan = (u64, ops::Range, u64, Option<(Duration, Coverage)>); + /// What a wire publisher's cursor gave up, found as it changes rather than by rescanning /// the cache on every poll. struct DropCursor { @@ -6806,15 +7115,30 @@ struct DropCursor { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] cuts: super::aborts::Watch, /// Sequences given up and not reported yet, with why: stale skips, cuts, and upstream - /// drops. - pending: RangeMap, - /// The [`TrackState::drops_revision`], span, and window revision last folded into - /// `pending`. + /// skips and drops. + pending: RangeMap, + /// What `pending` last folded in; see [`Scan`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - scanned: Option<(u64, ops::Range, u64)>, + scanned: Option, + /// The budget and coverage of the last drift judged, which settle received stale runs + /// until the next: a queue-only poll judges none, and must not unsettle them. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + drift: Option<(Duration, Coverage)>, /// Every sequence this cursor already reported. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] reported: RangeSet, + /// The reported runs judged stale, for a wire whose update re-offers them; see + /// [`Subscriber::reoffer_stale`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + reported_stale: RangeSet, + /// The part of [`Self::reported_stale`] an upstream's received stale run decided, which + /// is the upstream's to judge again once an update reopens it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + reported_received: RangeSet, + /// The reported runs skipped. A skip replaces a drop, so a run reported dropped is + /// reported again once it is skipped, and only a skip already reported stops it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + reported_skip: RangeSet, } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -6825,36 +7149,75 @@ impl DropCursor { cuts: state.cache.watch_cuts(), pending: RangeMap::new(), scanned: None, + drift: None, reported: RangeSet::new(), + reported_stale: RangeSet::new(), + reported_received: RangeSet::new(), + reported_skip: RangeSet::new(), + } + } + + /// Take back every stale run, reported or not: an update reopens their verdict, so they + /// are judged again, and reported again under the new update if still stale. Also returns + /// the part an upstream's received stale run decided. + fn take_stale(&mut self) -> (Vec>, RangeSet) { + let mut stale = RangeSet::new(); + let mut received = std::mem::take(&mut self.reported_received); + let queued: Vec<_> = self + .pending + .iter() + .filter(|(_, pending)| pending.hole.stale_budget().is_some()) + .map(|(run, pending)| (run.clone(), pending.received)) + .collect(); + for (run, from_upstream) in queued { + self.pending.remove(run.clone()); + if from_upstream { + received.insert(run.clone()); + } + stale.insert(run); + } + for run in std::mem::take(&mut self.reported_stale).iter() { + self.reported.remove(run.clone()); + stale.insert(run.clone()); + } + // A received stale run is folded in again by the next scan, if the track still holds it. + self.scanned = None; + (stale.into_iter().collect(), received) + } + + /// Retire a stale report on `sequence` once its group is handed out: the group won over + /// the drop, so no update reopens it. + fn retire(&mut self, sequence: u64) { + if let Some(end) = sequence.checked_add(1) { + self.reported_stale.remove(sequence..end); + self.reported_received.remove(sequence..end); } } - /// Record a run this cursor's fills gave up on as a miss: why it is gone is unknown, so - /// a reason already pending, such as the track's own drop, is kept. - fn give_up(&mut self, run: ops::Range) { + /// Record a run this cursor's fills gave up on as a miss, as `hole` says; a reason + /// already pending, such as the track's own drop, is kept. + fn give_up(&mut self, run: ops::Range, hole: Hole) { let gaps: Vec<_> = self.pending.gaps(&run).collect(); for gap in gaps { - self.pending.insert(gap, StreamError::NotFound); + self.pending.insert(gap, Pending::own(hole.clone())); } } - /// Record `slot`, cached at `sequence`, if it holds a cut. + /// Record `slot`, cached at `sequence`, if it holds a cut no fetched copy covers: a + /// copy still serving the group's end delivers it, so it is never also dropped. fn cut(&mut self, sequence: u64, slot: &Slot) { if slot.visible + && !slot.serves_end() && let Some(err) = slot.group.cut_error() && let Some(end) = sequence.checked_add(1) { - self.pending.insert(sequence..end, StreamError::from(err)); + self.pending + .insert(sequence..end, Pending::own(Hole::drop(StreamError::from(err)))); } } /// The lowest unreported run of `within` given up, now marked reported, with why. - fn next( - &mut self, - state: &TrackState, - cut: BTreeSet, - within: &Within<'_>, - ) -> Option<(ops::Range, StreamError)> { + fn next(&mut self, state: &TrackState, cut: BTreeSet, within: &Within<'_>) -> Option<(ops::Range, Hole)> { let (span, window) = (&within.span, within.window); // A cut already reclaimed from the cache is in the track's drops instead. for sequence in cut { @@ -6862,24 +7225,49 @@ impl DropCursor { self.cut(sequence, slot); } } - // Rescan only after the drops change or the window moves; out-of-window runs were - // skipped and may be owed now. - let scan = (state.drops_revision, span.clone(), within.revision); + // Rescan only after the drops change, the window moves, or what settles a stale run + // does; out-of-window and unsettling runs were skipped and may be owed now. + if let Some(drift) = within.drift { + self.drift = Some((drift.budget, drift.coverage.clone())); + } + let scan = (state.drops_revision, span.clone(), within.revision, self.drift.clone()); if self.scanned.as_ref() != Some(&scan) { self.scanned = Some(scan); - for (dropped, err) in state.drops.overlapping(span.clone()) { - self.pending - .insert(dropped.start.max(span.start)..dropped.end.min(span.end), err.clone()); + let stale: Vec<_> = self + .pending + .iter() + .filter(|(_, pending)| pending.received) + .map(|(run, _)| run.clone()) + .collect(); + for run in stale { + self.pending.remove(run); + } + for (dropped, hole) in state.drops.overlapping(span.clone()) { + let run = dropped.start.max(span.start)..dropped.end.min(span.end); + // A received stale run that does not settle this reader is unknown to it. + let settled = match &self.drift { + Some((budget, coverage)) => settles(*budget, coverage, run.start, hole, state.judged.as_ref()), + None => hole.stale_budget().is_none(), + }; + if settled { + self.pending.insert( + run, + Pending { + hole: hole.clone(), + received: true, + }, + ); + } } for (&sequence, slot) in state.lookup.range(span.clone()) { self.cut(sequence, slot); } } - while let Some((range, err)) = self + while let Some((range, pending)) = self .pending .iter() .next() - .map(|(range, err)| (range.clone(), err.clone())) + .map(|(range, pending)| (range.clone(), pending.clone())) { self.pending.remove(range.clone()); // Only what this subscriber asked for is owed: the first run of its ranges in it, @@ -6888,14 +7276,43 @@ impl DropCursor { continue; }; if clipped.end < range.end { - self.pending.insert(clipped.end..range.end, err.clone()); + self.pending.insert(clipped.end..range.end, pending.clone()); + } + // A skip goes out over a drop already reported, and a final drop over a stale one, + // since each replaces what it goes out over. + let mut blocked = match pending.hole { + Hole::Skip => self.reported_skip.overlapping(&clipped).cloned().collect(), + Hole::Drop { .. } => self.reported.overlapping(&clipped).cloned().collect::>(), + }; + if matches!(pending.hole, Hole::Drop { stale: None, .. }) { + for stale in self.reported_stale.overlapping(&clipped) { + blocked.remove(stale.clone()); + } } - if let Some(gap) = self.reported.gaps(&clipped).next() { + if let Some(gap) = blocked.gaps(&clipped).next() { if gap.end < range.end { - self.pending.insert(gap.end..range.end, err.clone()); + self.pending.insert(gap.end..range.end, pending.clone()); } + let hole = pending.hole; self.reported.insert(gap.clone()); - return Some((gap, err)); + match &hole { + hole if hole.stale_budget().is_some() => { + self.reported_stale.insert(gap.clone()); + match pending.received { + true => self.reported_received.insert(gap.clone()), + false => self.reported_received.remove(gap.clone()), + } + } + // Final, so no update reopens a stale report it replaced. + hole => { + if matches!(hole, Hole::Skip) { + self.reported_skip.insert(gap.clone()); + } + self.reported_stale.remove(gap.clone()); + self.reported_received.remove(gap.clone()); + } + } + return Some((gap, hole)); } } None @@ -6984,22 +7401,18 @@ impl Cursor { /// The next run given up, judging the live edge first only when `probe`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop( - &mut self, - taken: &Delivered, - probe: bool, - waiter: &kio::Waiter, - ) -> Poll<(ops::Range, StreamError)> { + fn poll_drop(&mut self, taken: &Delivered, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { self.sync(); if self.drops.is_none() { let mut drops = Box::new(DropCursor::new(&self.state.read())); if let Some(misses) = &self.misses { for run in misses.given_up().iter() { - drops.give_up(run.clone()); + drops.give_up(run.clone(), Hole::missed(None)); } } self.drops = Some(drops); } + let mut judged = None; if probe && self.filter.refused.is_none() && let Poll::Ready(Ok(drift)) = self.poll_drift(&self.servable, waiter) @@ -7007,6 +7420,7 @@ impl Cursor { // From where the reader reads next: an ordered one asks nothing below what it took. let from = self.next_sequence.unwrap_or(u64::MAX).max(self.floor()); self.probe(from, &drift, taken, waiter); + judged = Some(drift); } // A cap at or below the floor leaves nothing to report, not a reversed span. let floor = self.floor(); @@ -7014,6 +7428,7 @@ impl Cursor { span: floor..self.end_sequence.unwrap_or(u64::MAX).max(floor), window: &self.filter.window, revision: self.filter.revision, + drift: judged.as_ref(), }; let drops = match &mut self.drops { Some(drops) => drops, @@ -7157,26 +7572,33 @@ impl Cursor { return false; } let given_up = self.misses.as_ref().map(|misses| misses.given_up()); + let coverage = self.coverage(); let state = self.state.read(); - let accounted = |gap: ops::Range| { - let mut left = RangeSet::new(); - left.insert(gap.clone()); - for run in self.filter.written_off.overlapping(&gap) { - left.remove(run.clone()); - } - if let Some(given_up) = given_up { - for run in given_up.overlapping(&gap) { + let budget = clamp_max_delay(self.subscription.read().budget(), state.max_age_bound()); + let accounted = + |gap: ops::Range| { + let mut left = RangeSet::new(); + left.insert(gap.clone()); + for run in self.filter.written_off.overlapping(&gap) { left.remove(run.clone()); } - } - for (run, _) in state.drops.overlapping(&gap) { - left.remove(run.clone()); - } - for sequence in self.seek_pending.range(gap).map(|(sequence, _)| *sequence) { - left.remove(sequence..sequence + 1); - } - left.is_empty() - }; + if let Some(given_up) = given_up { + for run in given_up.overlapping(&gap) { + left.remove(run.clone()); + } + } + // A received stale run that does not settle this reader is unknown to it, so it + // still wants those groups. + for (run, _) in state.drops.overlapping(&gap).filter(|(run, hole)| { + settles(budget, &coverage, run.start.max(gap.start), hole, state.judged.as_ref()) + }) { + left.remove(run.clone()); + } + for sequence in self.seek_pending.range(gap).map(|(sequence, _)| *sequence) { + left.remove(sequence..sequence + 1); + } + left.is_empty() + }; self.filter .window .runs(self.filter.window.floor().unwrap_or(0)) @@ -7356,30 +7778,77 @@ impl Cursor { } /// Owe again the groups this cursor skipped as stale under a budget narrower than - /// `budget`: only a widening reverses a stale verdict. Costs the skipped runs, never - /// the cache, and nothing when the budget did not widen. + /// `budget`: only a widening reverses a stale verdict it has not reported. Costs the + /// skipped runs, never the cache, and nothing when the budget did not widen. fn reoffer(&mut self, budget: Duration) { for run in self.skipped.widened(budget) { - self.filter - .written_off - .remove(*run.start()..run.end().saturating_add(1)); - // Judged again, so a drop not reported yet is withdrawn; a reported one stands. - if let Some(drops) = &mut self.drops { - let span = *run.start()..run.end().saturating_add(1); - let old: Vec<_> = drops - .pending - .overlapping(span.clone()) - .filter(|(_, err)| **err == StreamError::Old) - .map(|(range, _)| range.start.max(span.start)..range.end.min(span.end)) - .collect(); - for range in old { - drops.pending.remove(range); + self.reowe(run); + } + } + + /// Owe again every run this cursor gave up as stale, reported or not: an update reopens + /// the verdict. A group still cached is owed as a widened skip is, a missing one is + /// missed again. Returns the runs taken back; whatever is still stale under the new + /// terms is given up and reported again. + fn reoffer_stale(&mut self) -> Vec> { + let Some(drops) = &mut self.drops else { + return Vec::new(); + }; + let (runs, received) = drops.take_stale(); + // What neither a skip nor a miss offers again: a group written off at the track's max + // age bound, which no budget reverses. It is still stale under the new terms. What an + // upstream's stale run decided is the upstream's to judge again: the next scan folds + // it in again if the track still holds it, and it stays owed if new terms superseded it. + let mut kept = RangeSet::new(); + for run in &runs { + kept.insert(run.clone()); + for decided in received.overlapping(run) { + kept.remove(decided.clone()); + } + if let Some(misses) = &mut self.misses { + for missed in misses.given_up().overlapping(run) { + kept.remove(missed.clone()); + } + misses.retry(run.clone()); + } + if let Some(last) = run.end.checked_sub(1) { + for skipped in self.skipped.within(run.start..=last) { + kept.remove(*skipped.start()..skipped.end().saturating_add(1)); + self.reowe(skipped); } } - for owed in self.filter.admitted.overlapping(run.clone()) { - self.filter.owed.insert(owed); + } + let budget = { + let state = self.state.read(); + clamp_max_delay(self.subscription.read().budget(), state.max_age_bound()) + }; + if let Some(drops) = &mut self.drops { + for run in kept.iter() { + drops.give_up(run.clone(), Hole::stale(budget)); + } + } + runs + } + + /// Owe `run` of skipped groups again, judged anew, withdrawing a stale drop of it not + /// reported yet; a reported one stands. + fn reowe(&mut self, run: ops::RangeInclusive) { + let span = *run.start()..run.end().saturating_add(1); + self.filter.written_off.remove(span.clone()); + if let Some(drops) = &mut self.drops { + let old: Vec<_> = drops + .pending + .overlapping(span.clone()) + .filter(|(_, pending)| pending.hole.stale_budget().is_some()) + .map(|(range, _)| range.start.max(span.start)..range.end.min(span.end)) + .collect(); + for range in old { + drops.pending.remove(range); } } + for owed in self.filter.admitted.overlapping(run) { + self.filter.owed.insert(owed); + } } // A helper to automatically apply Dropped if the state is closed without an error. @@ -7427,6 +7896,11 @@ impl Cursor { std::mem::take(&mut self.stale) } + /// The groups this reader reads, and its cap; see [`Coverage`]. + fn coverage(&self) -> Coverage { + Some((self.filter.window.groups()?.clone(), self.end_sequence)) + } + /// This subscriber's clamped drift budget and the live edge to measure against, /// resolved once per poll. /// @@ -7443,12 +7917,14 @@ impl Cursor { max_delay = subscription.budget(); Poll::<()>::Pending }); + let coverage = self.coverage(); self.poll(waiter, |state| { let bound = state.max_age_bound(); Poll::Ready(Ok(Drift { budget: clamp_max_delay(max_delay, bound), bounded: bound.is_some_and(|bound| max_delay >= bound), edge: state.drift_edge(servable), + coverage: coverage.clone(), })) }) } @@ -7691,8 +8167,8 @@ impl Cursor { misses.expire(&self.state.read(), drift, &mut given_up, waiter); } if let Some(drops) = &mut self.drops { - for run in given_up { - drops.give_up(run); + for (run, stale) in given_up { + drops.give_up(run.clone(), Hole::missed(stale)); } } // A refusal taken in may move a time end walking history past it, widening what @@ -7710,8 +8186,8 @@ impl Cursor { let mut given_up = Vec::new(); let group = misses.next_filled(&self.state.read(), order, &mut given_up); if let Some(drops) = &mut self.drops { - for run in given_up { - drops.give_up(run); + for (run, stale) in given_up { + drops.give_up(run.clone(), Hole::missed(stale)); } } group @@ -7847,7 +8323,9 @@ impl Cursor { if let Some(drops) = &mut self.drops && let Some(end) = sequence.checked_add(1) { - drops.pending.insert(sequence..end, StreamError::Old); + drops + .pending + .insert(sequence..end, Pending::own(Hole::stale(drift.budget))); } match drift.bounded { // A re-offered group stale at the bound is final. @@ -7994,9 +8472,10 @@ impl Cursor { for run in given_up { misses.give_up(run, &mut runs); } + // A head the cursor found missing belonged to a group that existed. if let Some(drops) = &mut self.drops { - for run in runs { - drops.give_up(run); + for (run, _) in runs { + drops.give_up(run, Hole::drop(StreamError::NotFound)); } } } @@ -8594,6 +9073,12 @@ impl Subscriber { cursor.prune_skipped(); } self.delivered.insert(sequence); + // The group won over any stale report on it; a resumed reader retires its own. + if let Inner::Plain(cursor) = &mut self.inner + && let Some(drops) = &mut cursor.drops + { + drops.retire(sequence); + } let mut delivered = std::mem::take(&mut self.delivered); self.forget_past_budget(&mut delivered); self.delivered = delivered; @@ -8866,7 +9351,7 @@ impl Subscriber { /// Stale skips are recorded only once this has been polled, so a reader that never /// forwards drops keeps none. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + pub(crate) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { self.poll_drop_probing(true, waiter) } @@ -8874,12 +9359,12 @@ impl Subscriber { /// the last judgement gave up, such as a receive that probed after a drop poll found /// nothing. That judgement parked `waiter` on everything that could change it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn poll_queued_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + pub(crate) fn poll_queued_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { self.poll_drop_probing(false, waiter) } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { match &mut self.inner { Inner::Plain(cursor) => cursor.poll_drop(&self.delivered, probe, waiter), Inner::Resume(resume, _) => resume.poll_drop(&self.delivered, probe, waiter), @@ -8968,6 +9453,18 @@ impl Subscriber { self } + /// Re-offer what this reader gave up as stale through [`Self::poll_drop`], as a wire + /// whose every update reopens its stale drops does: each run is judged again under the + /// new terms. Returns the runs taken back, which are owed again. A front's reader takes + /// back what it queued and what its serving route's copy reported. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn reoffer_stale(&mut self) -> Vec> { + match &mut self.inner { + Inner::Plain(cursor) => cursor.reoffer_stale(), + Inner::Resume(resume, _) => resume.reoffer_stale(), + } + } + pub(crate) fn set_range(&mut self, groups: impl RangeBounds) { let (start, end) = super::subscription::sequence_bounds(groups); match &mut self.inner { @@ -9088,6 +9585,73 @@ impl Subscriber { } } + /// Fresh reads of the groups in `run` this subscriber's track still caches, its range + /// still admits, and its current budget no longer judges stale, each expiring against + /// that budget, and the runs still given up: no longer cached, or still stale. For a wire + /// whose update re-offers a group whose stream it gave up as stale. A front's reader asks + /// its serving route's copy. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn reoffer_groups(&mut self, run: ops::Range) -> (Vec, RangeSet) { + let cursor = match &mut self.inner { + Inner::Plain(cursor) => cursor, + Inner::Resume(resume, _) => return resume.reoffer_groups(run), + }; + let mut given_up = RangeSet::new(); + given_up.insert(run.clone()); + let budget = cursor.subscription.read().budget(); + let fresh: Vec<_> = { + let state = cursor.state.read(); + let budget = clamp_max_delay(budget, state.max_age_bound()); + let edge = state.drift_edge(&cursor.servable); + state + .lookup + .range(run) + .filter(|&(&sequence, slot)| { + slot.serves_end() && cursor.admits(sequence) && !state.is_stale(sequence, &edge, budget) + }) + .map(|(&sequence, slot)| (sequence, slot.serving().consume())) + .collect() + }; + let mut groups = Vec::new(); + for (sequence, group) in fresh { + if let Some(end) = sequence.checked_add(1) { + given_up.remove(sequence..end); + group.cache_refresh(); + groups.push(cursor.deliver(group)); + } + } + (groups, given_up) + } + + /// The parts of `run` this subscriber's track still holds as a received stale run: one + /// new terms superseded is gone (see [`Producer::supersede_stale`]). + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn stale_held(&self, run: ops::Range) -> RangeSet { + match &self.inner { + Inner::Plain(cursor) => cursor + .state + .read() + .drops + .overlapping(&run) + .filter(|(_, hole)| hole.stale_budget().is_some()) + .map(|(held, _)| held.start.max(run.start)..held.end.min(run.end)) + .collect(), + // A front's reader checks its own routes' copies. + Inner::Resume(..) => RangeSet::new(), + } + } + + /// The budget this subscriber judges staleness under: its max delay, clamped to the + /// track's max age bound. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn budget(&self) -> Duration { + let budget = self.subscription().budget(); + match &self.inner { + Inner::Plain(cursor) => clamp_max_delay(budget, cursor.state.read().max_age_bound()), + Inner::Resume(..) => budget, + } + } + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn latest_sequence(&self) -> Option { match &self.inner { @@ -11711,7 +12275,7 @@ mod test { assert_eq!(read_now(&mut reader, true), None, "the fresh suffix is still owed"); assert_eq!( reader.poll_drop(&waiter), - Poll::Ready((0..3, StreamError::NotFound)), + Poll::Ready((0..3, Hole::drop(StreamError::NotFound))), "the unjudgeable prefix is given up" ); assert!(fill.wanted().is_some(), "the fresh suffix stays wanted"); @@ -11819,7 +12383,10 @@ mod test { "a late live arrival may still answer" ); elapse(&producer, ledger::GRACE); - assert_eq!(reader.poll_drop(&waiter), Poll::Ready((1..3, StreamError::NotFound))); + assert_eq!( + reader.poll_drop(&waiter), + Poll::Ready((1..3, Hole::drop(StreamError::NotFound))) + ); assert!(reader.poll_drop(&waiter).is_pending(), "each drop is reported once"); if bounded { assert_eq!(read_now(&mut reader, ordered), Some(None), "nothing remains owed"); @@ -13530,6 +14097,7 @@ mod test { budget: Duration::ZERO, bounded: false, edge: state.drift_edge(&Servable::ALL), + coverage: None, }; assert!( state.is_stale(0, &drift.edge, drift.budget), @@ -18015,6 +18583,20 @@ mod test { ); } + /// A group the publisher cut while a fetched copy still serves its end is delivered, + /// and never also reported dropped. + #[kio::test] + #[cfg(feature = "async")] + async fn a_cut_a_fetched_copy_covers_is_not_reported() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay()); + let (live, _head) = live_group_with_head(&producer).await; + live.abort(Error::Stream(StreamError::App(5))).unwrap(); + let waiter = kio::Waiter::noop(); + assert!(subscriber.poll_drop(&waiter).is_pending(), "the copy covers the cut"); + assert_eq!(drain_groups(&mut subscriber), [0]); + } + /// Reads land on the fetched head, so they keep the slot from expiring even though /// the live copy beside it sits idle. #[kio::test] @@ -19288,9 +19870,9 @@ mod test { let mut fetched = pending.await.unwrap(); subscriber.assert_no_group(); - let mut live = producer - .create_group_at(0u64.into(), 0) - .expect("the feed takes the slot"); + let Ok(Received::Group(mut live)) = producer.create_group_at(0u64.into(), 0) else { + panic!("the feed takes the slot"); + }; live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut group = subscriber.assert_group(); @@ -20957,6 +21539,8 @@ mod test { #[cfg(feature = "async")] fn a_reader_forgets_misses_behind_its_window() { let mut producer = track_producer("test", Info::timed()); + // A received copy, whose missing groups may still arrive. + producer.received(); let _dynamic = producer.requests(); let ascending = |groups| Subscription::default().with_range(groups).with_order(Order::Ascending); let mut subscriber = producer.subscribe(ascending(0..)); @@ -21057,6 +21641,8 @@ mod test { #[cfg(feature = "async")] fn an_end_ahead_of_the_edge_keeps_a_late_group() { let mut producer = track_producer("test", Info::timed()); + // A received copy, whose missing groups may still arrive. + producer.received(); let mut subscriber = producer.subscribe(replay().with_range(0..)); for sequence in [0, 1, 2, 4, 5] { write_group(&producer, sequence); @@ -22175,47 +22761,644 @@ mod test { } assert_eq!( dropped, - vec![(2..4, StreamError::NotFound), (6..8, StreamError::NotFound)] + vec![ + (2..4, Hole::drop(StreamError::NotFound)), + (6..8, Hole::drop(StreamError::NotFound)) + ] ); } - /// A report describes a miss, not a delivery: a late group is still handed out once. + /// A received stale run settles at once only a reader no wider than the budget it was + /// judged under, whose range covers it and everything above it; any other reader treats + /// it as unknown. A skip settles every reader. #[test] #[cfg(feature = "async")] - fn a_reported_drop_still_allows_a_late_group() { + fn a_received_stale_drop_settles_only_the_readers_it_covers() { let producer = track_producer("test", Info::timed()); - let mut subscriber = producer.subscribe(replay()); + // A received copy's serving session keeps a handler, so an unsettled run is asked + // rather than given up as unfillable. + let _dynamic = producer.requests(); + let budget = Duration::from_millis(500); + let reader = |max: Duration, groups: Range| { + producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(max)) + .with_ranges([groups]), + ) + }; + let mut covering = reader(budget, Range::groups(0..)); + let mut narrower = reader(Duration::from_millis(100), Range::groups(0..)); + let mut wider = reader(Duration::from_secs(5), Range::groups(0..)); + let mut capped = reader(budget, Range::groups(0..4)); + let mut above = reader(budget, Range::groups(3..)); + producer.give_up(1..3, Hole::stale(budget)); + producer.skip_groups(3..4); let waiter = kio::Waiter::noop(); - producer.drop_groups(2..4, StreamError::NotFound); + let dropped = |subscriber: &mut Subscriber| { + let mut runs = Vec::new(); + while let Poll::Ready(run) = subscriber.poll_drop(&waiter) { + runs.push(run); + } + runs + }; + let settled = vec![(1..3, Hole::stale(budget)), (3..4, Hole::Skip)]; + assert_eq!(dropped(&mut covering), settled); + assert_eq!(dropped(&mut narrower), settled, "a narrower budget is settled too"); + assert_eq!(dropped(&mut wider), [(3..4, Hole::Skip)], "a wider budget waits"); assert_eq!( - subscriber.poll_drop(&waiter), - Poll::Ready((2..4, StreamError::NotFound)) + dropped(&mut capped), + [(3..4, Hole::Skip)], + "a capped range may judge otherwise" + ); + assert_eq!( + dropped(&mut above), + [(3..4, Hole::Skip)], + "the stale run is outside its range" ); - assert!(subscriber.poll_drop(&waiter).is_pending()); - write_group(&producer, 3); - assert_eq!(drain_groups(&mut subscriber), [3]); - assert!(subscriber.poll_drop(&waiter).is_pending()); - assert!(drain_groups(&mut subscriber).is_empty()); } - /// A drop ends a group still arriving with the drop's error at once, for the reader - /// already holding it, before any stream reset or track end. A group that finished here - /// stays whole. + /// A queue-only poll, which judges no drift, still reports the received stale runs the + /// last probing poll found settled, rather than forgetting them until the next probe. #[test] #[cfg(feature = "async")] - fn a_drop_ends_a_held_group_with_its_error() { + fn a_queued_drop_poll_keeps_the_last_drift() { let producer = track_producer("test", Info::timed()); - let mut subscriber = producer.subscribe(replay()); - let mut open = producer.create_group(group::Info { sequence: 0 }).unwrap(); - open.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); - let mut finished = producer.create_group(group::Info { sequence: 1 }).unwrap(); - finished.write_frame(Timestamp::ZERO, &b"b"[..]).unwrap(); - finished.finish().unwrap(); - - let mut held = subscriber.assert_group(); + let budget = Duration::from_millis(500); + let mut subscriber = producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..)]), + ); + producer.give_up(1..2, Hole::stale(budget)); + producer.give_up(3..4, Hole::stale(budget)); + let waiter = kio::Waiter::noop(); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..2, Hole::stale(budget)))); assert_eq!( - &held.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[..], - b"a" + subscriber.poll_queued_drop(&waiter), + Poll::Ready((3..4, Hole::stale(budget))) + ); + assert!(subscriber.poll_queued_drop(&waiter).is_pending()); + } + + /// A stale run that settled a reader is no verdict once the reader's budget widens past + /// the one it was judged under: the wider reader asks a fill for it, as one that was + /// always that wide does. + #[test] + #[cfg(feature = "async")] + fn a_widened_budget_asks_a_fill_for_a_stale_run() { + let mut producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + producer.start_at(3).unwrap(); + write_group(&producer, 3); + let budget = Duration::from_millis(500); + producer.give_up(0..3, Hole::stale(budget)); + let narrow = Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..)]); + let mut subscriber = producer.subscribe(narrow.clone()); + let waiter = kio::Waiter::noop(); + assert_eq!(drain_groups(&mut subscriber), [3]); + assert!( + dynamic.poll_requested_fill(&waiter).is_pending(), + "the stale run settles the reader" + ); + + subscriber + .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(60)))) + .unwrap(); + assert!(subscriber.poll_next_group(&waiter).is_pending()); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("the wider budget asks for the stale run"); + }; + assert_eq!((fill.start().group, fill.end()), (0, 3)); + } + + /// Every group a set of ranges names, as a received copy's upstream terms judge them. + #[cfg(feature = "async")] + fn judged(runs: impl IntoIterator) -> Option> { + Some(runs.into_iter().map(|(start, end)| start..end).collect()) + } + + /// A stale verdict settles a reader that reads every range the upstream judged it + /// against, however many there are, with nothing received and no fill asked. Mirrors JS. + #[test] + #[cfg(feature = "async")] + fn a_stale_verdict_settles_a_reader_of_every_range_it_judged() { + let producer = track_producer("test", Info::timed()); + producer.received(); + let dynamic = producer.requests(); + let budget = Duration::from_millis(500); + // The upstream judged 0..2 and 4..6 under 500ms, and gave up both of them as stale. + producer.supersede_stale(judged([(0, 2), (4, 6)])); + producer.give_up(0..2, Hole::stale(budget)); + producer.give_up(4..6, Hole::stale(budget)); + let mut subscriber = producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..2), Range::groups(4..6)]), + ); + let waiter = kio::Waiter::noop(); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((0..2, Hole::stale(budget)))); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((4..6, Hole::stale(budget)))); + assert!( + matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None))), + "nothing is owed" + ); + assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "nothing is asked"); + } + + /// A reader whose ranges only partly overlap the ones a stale verdict was judged against + /// is settled where it reads every judged group above the run, and asks a fill for the + /// rest, even with nothing received: no update brings it, since its ranges add nothing to + /// the upstream's terms. Mirrors JS. + #[test] + #[cfg(feature = "async")] + fn a_stale_verdict_settles_a_partly_overlapping_reader_only_where_it_reads_above() { + let producer = track_producer("test", Info::timed()); + producer.received(); + let dynamic = producer.requests(); + let budget = Duration::from_millis(500); + producer.supersede_stale(judged([(0, 2), (4, 6)])); + producer.give_up(0..2, Hole::stale(budget)); + producer.give_up(4..6, Hole::stale(budget)); + let reader = |ranges: [Range; 2]| { + producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges(ranges), + ) + }; + let waiter = kio::Waiter::noop(); + // Reads every judged group above each run, and more past them: settled. + let mut wider = reader([Range::groups(1..2), Range::groups(4..8)]); + assert_eq!(wider.poll_drop(&waiter), Poll::Ready((1..2, Hole::stale(budget)))); + assert_eq!(wider.poll_drop(&waiter), Poll::Ready((4..6, Hole::stale(budget)))); + assert!(wider.poll_drop(&waiter).is_pending()); + assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "nothing is asked"); + // Stops at 4, which the upstream judged against 5: its own newest, which no verdict + // convicts. Both runs are unknown to it, and asked for. + let mut narrower = reader([Range::groups(0..2), Range::groups(4..5)]); + assert!(narrower.poll_recv_group(&waiter).is_pending()); + assert!(narrower.poll_drop(&waiter).is_pending()); + let mut asked = Vec::new(); + while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { + asked.push((fill.start().group, fill.end())); + } + asked.sort(); + assert_eq!(asked, [(0, 2), (4, 5)]); + } + + /// A fill taken before the last handler closed still answers an unsettled stale run: a + /// reader that asks afterwards joins it while it runs, and reads its groups from the cache + /// once it is done. + #[test] + #[cfg(feature = "async")] + fn a_taken_fill_answers_an_unsettled_stale_run_after_its_handler_closes() { + let producer = track_producer("test", Info::timed()); + producer.received(); + let dynamic = producer.requests(); + let budget = Duration::from_millis(500); + producer.supersede_stale(judged([(0, 2), (4, 6)])); + producer.give_up(0..2, Hole::stale(budget)); + producer.give_up(4..6, Hole::stale(budget)); + // Stops at 2, though the upstream judged 0..2 against 4..6 too: the run is unknown to it. + let reader = || { + producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..2)]), + ) + }; + let waiter = kio::Waiter::noop(); + let mut first = reader(); + assert!(first.poll_recv_group(&waiter).is_pending()); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { + panic!("the unsettled stale run is asked for"); + }; + assert_eq!((fill.start().group, fill.end()), (0, 2)); + drop(dynamic); + let mut joined = reader(); + assert!(joined.poll_recv_group(&waiter).is_pending(), "it joins the taken fill"); + for sequence in 0..2 { + let mut group = fill.accept(sequence, None).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); + } + drop(fill); + let mut late = reader(); + for subscriber in [&mut first, &mut joined, &mut late] { + let mut got = drain_groups(subscriber); + got.sort_unstable(); + assert_eq!(got, [0, 1]); + assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); + } + } + + /// Creating a group past the next sequence says nothing about the gap: groups may be + /// created out of order, so a late one in it still reaches readers, and nothing is skipped. + #[test] + #[cfg(feature = "async")] + fn creating_a_group_past_a_gap_skips_nothing() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + let waiter = kio::Waiter::noop(); + write_group(&producer, 2); + write_group(&producer, 5); + assert!(producer.state.read().drops.is_empty()); + assert!(subscriber.poll_drop(&waiter).is_pending()); + assert_eq!(drain_groups(&mut subscriber), [2, 5]); + + write_group(&producer, 4); + assert_eq!(drain_groups(&mut subscriber), [4]); + assert!(producer.state.read().drops.is_empty()); + } + + /// Appending says the producer will not go back: every sequence below the new group it + /// never produced since its first group is skipped, and readers are told at once. + #[test] + #[cfg(feature = "async")] + fn appending_skips_the_holes_below_it() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + let waiter = kio::Waiter::noop(); + write_group(&producer, 2); + write_group(&producer, 5); + write_group(&producer, 7); + producer.append_group().unwrap().finish().unwrap(); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((3..5, Hole::Skip))); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((6..7, Hole::Skip))); + assert_eq!(drain_groups(&mut subscriber), [2, 5, 7, 8]); + + // The next append only names what was left since this one. + write_group(&producer, 10); + producer.append_group().unwrap().finish().unwrap(); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((9..10, Hole::Skip))); + let skipped: Vec<_> = producer + .state + .read() + .drops + .iter() + .map(|(run, hole)| (run.clone(), hole.clone())) + .collect(); + assert_eq!(skipped, [(3..5, Hole::Skip), (6..7, Hole::Skip), (9..10, Hole::Skip)]); + } + + /// An append's skip replaces a drop below it, whatever its error: the sequence was never + /// produced, which is the most authoritative verdict. + #[test] + #[cfg(feature = "async")] + fn appending_replaces_a_drop_with_a_skip() { + let producer = track_producer("test", Info::timed()); + write_group(&producer, 0); + write_group(&producer, 4); + producer.give_up(1..2, Hole::drop(StreamError::Cancel)); + producer.append_group().unwrap().finish().unwrap(); + let drops: Vec<_> = producer + .state + .read() + .drops + .iter() + .map(|(run, hole)| (run.clone(), hole.clone())) + .collect(); + assert_eq!(drops, [(1..4, Hole::Skip)]); + } + + /// A skip is final: creating or filling a group at a sequence an append or `skip_groups` + /// skipped is refused, while a dropped one may be created again. + #[test] + #[cfg(feature = "async")] + fn creating_a_skipped_sequence_is_refused() { + let producer = track_producer("test", Info::timed()); + write_group(&producer, 0); + write_group(&producer, 3); + producer.append_group().unwrap().finish().unwrap(); + assert!(matches!( + producer.create_group(group::Info { sequence: 1 }), + Err(Error::Skipped) + )); + + producer.skip_groups(5..7); + assert!(matches!( + producer.create_group(group::Info { sequence: 6 }), + Err(Error::Skipped) + )); + // The next sequence, 5, was skipped too. + assert!(matches!(producer.append_group(), Err(Error::Skipped))); + // A later drop leaves a skipped sequence skipped. + producer.give_up(1..2, Hole::drop(StreamError::Cancel)); + assert!(matches!( + producer.create_group(group::Info { sequence: 1 }), + Err(Error::Skipped) + )); + // No fill holds a skipped sequence either. + assert!(matches!( + producer + .state + .write() + .ok() + .unwrap() + .insert_group_request(6, group::Bounds::default(), None), + Err(Error::Skipped) + )); + // A dropped sequence may be created again. + producer.give_up(9..10, Hole::drop(StreamError::Cancel)); + write_group(&producer, 9); + } + + /// Skips are kept and reported as runs: a huge `skip_groups` or append gap costs one + /// entry and one report, whatever its length. + #[test] + #[cfg(feature = "async")] + fn a_huge_skip_costs_one_run() { + const FAR: u64 = 1 << 61; + let waiter = kio::Waiter::noop(); + // Presented just after group 0, since its sequence in milliseconds overflows. + let write_far = |producer: &Producer| { + let mut group = producer.create_group(group::Info { sequence: FAR }).unwrap(); + group + .write_frame(Timestamp::from_millis(1).unwrap(), &b"x"[..]) + .unwrap(); + group.finish().unwrap(); + }; + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + write_group(&producer, 0); + producer.skip_groups(1..FAR); + assert_eq!(producer.state.read().drops.iter().count(), 1); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..FAR, Hole::Skip))); + write_far(&producer); + assert_eq!(drain_groups(&mut subscriber), [0, FAR]); + assert!(subscriber.poll_drop(&waiter).is_pending()); + + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + write_group(&producer, 0); + write_far(&producer); + producer.append_group().unwrap().finish().unwrap(); + assert_eq!(producer.state.read().drops.iter().count(), 1); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..FAR, Hole::Skip))); + assert_eq!(drain_groups(&mut subscriber), [0, FAR, FAR + 1]); + } + + /// A dropped sequence may be created again, as a group stream wins over its drop, such as + /// one whose group was aborted: only a skip is final. + #[test] + #[cfg(feature = "async")] + fn an_aborted_sequence_is_created_again() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + let group = producer.create_group(group::Info { sequence: 0 }).unwrap(); + group.abort(Error::Stream(StreamError::Cancel)).unwrap(); + write_group(&producer, 0); + producer.append_group().unwrap().finish().unwrap(); + assert!(producer.state.read().drops.is_empty()); + let mut read = drain_groups(&mut subscriber); + read.dedup(); + assert_eq!(read.last(), Some(&1)); + } + + /// `skip_groups` leaves a cached group alone, and replaces every drop, final or stale: the + /// skip is final, so new terms never supersede it. + #[test] + #[cfg(feature = "async")] + fn skipping_replaces_every_drop_but_leaves_a_cached_group() { + let producer = track_producer("test", Info::timed()); + write_group(&producer, 1); + producer.give_up(3..4, Hole::drop(StreamError::Cancel)); + producer.give_up(4..5, Hole::stale(Duration::from_secs(1))); + producer.skip_groups(0..5); + producer.supersede_stale(None); + let drops: Vec<_> = producer + .state + .read() + .drops + .iter() + .map(|(run, hole)| (run.clone(), hole.clone())) + .collect(); + assert_eq!(drops, [(0..1, Hole::Skip), (2..5, Hole::Skip)]); + } + + /// A skip after a drop already reported goes out again, replacing it, while a reader that + /// moved past the drop is not handed anything again; one already reported as a skip is + /// not reported twice, and a later drop leaves it skipped. + #[test] + #[cfg(feature = "async")] + fn a_skip_is_reported_over_a_reported_drop() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + let waiter = kio::Waiter::noop(); + write_group(&producer, 0); + producer.give_up(1..3, Hole::drop(StreamError::Cancel)); + producer.give_up(4..5, Hole::drop(StreamError::NotFound)); + write_group(&producer, 5); + assert_eq!( + subscriber.poll_drop(&waiter), + Poll::Ready((1..3, Hole::drop(StreamError::Cancel))) + ); + assert_eq!( + subscriber.poll_drop(&waiter), + Poll::Ready((4..5, Hole::drop(StreamError::NotFound))) + ); + assert!(subscriber.poll_drop(&waiter).is_pending()); + assert_eq!(drain_groups(&mut subscriber), [0, 5]); + + // Group 0 arrived, so the skip leaves it; both drops and the gap between become one skip. + producer.skip_groups(0..5); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..5, Hole::Skip))); + assert!(subscriber.poll_drop(&waiter).is_pending()); + assert!(drain_groups(&mut subscriber).is_empty(), "nothing is delivered again"); + + producer.give_up(2..3, Hole::drop(StreamError::Cancel)); + assert!(subscriber.poll_drop(&waiter).is_pending(), "the skip stands"); + assert!(producer.state.read().skipped(2)); + } + + /// Appending and a first group leave no gap, so they skip nothing; nor is anything below + /// a track's first group skipped. + #[test] + #[cfg(feature = "async")] + fn appending_and_a_first_group_skip_nothing() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + write_group(&producer, 3); + producer.append_group().unwrap().finish().unwrap(); + assert!(producer.state.read().drops.is_empty()); + assert!(drained_drops(&mut subscriber).is_empty()); + } + + /// A received copy's groups land in any order, so a gap above its newest is still owed. + #[test] + #[cfg(feature = "async")] + fn a_received_gap_is_not_skipped() { + let producer = track_producer("test", Info::timed()); + producer.received(); + let mut subscriber = producer.subscribe(replay().with_range(0..)); + write_group(&producer, 2); + write_group(&producer, 5); + assert!(producer.state.read().drops.is_empty()); + assert!(drained_drops(&mut subscriber).is_empty()); + write_group(&producer, 3); + assert_eq!(drain_groups(&mut subscriber), [2, 5, 3]); + } + + /// A received stale run superseded by new range terms is unknown again, and a group a + /// widening re-offers wins over its drop. + #[test] + #[cfg(feature = "async")] + fn superseded_stale_runs_are_unknown_again() { + let producer = track_producer("test", Info::timed()); + let budget = Duration::from_millis(500); + let mut subscriber = producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..)]), + ); + let waiter = kio::Waiter::noop(); + producer.give_up(1..2, Hole::stale(budget)); + producer.supersede_stale(None); + assert!( + subscriber.poll_drop(&waiter).is_pending(), + "no verdict under the new terms" + ); + assert!(producer.state.read().drops.is_empty()); + write_group(&producer, 1); + assert_eq!(drain_groups(&mut subscriber), [1]); + } + + /// A superseded stale run leaves nothing a cursor still holds to report, even the part + /// of it queued behind an earlier report. + #[test] + #[cfg(feature = "async")] + fn superseding_withdraws_a_partly_reported_stale_run() { + let producer = track_producer("test", Info::timed()); + let budget = Duration::from_millis(500); + let mut subscriber = producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..)]), + ); + let waiter = kio::Waiter::noop(); + producer.drop_groups(4..5, StreamError::NotFound); + assert_eq!( + subscriber.poll_drop(&waiter), + Poll::Ready((4..5, Hole::drop(StreamError::NotFound))) + ); + producer.give_up(3..7, Hole::stale(budget)); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((3..4, Hole::stale(budget)))); + producer.supersede_stale(None); + assert!(subscriber.poll_drop(&waiter).is_pending(), "5 and 6 are unknown again"); + } + + /// A received stale run a reader reported and the relay's own update then superseded is + /// unknown again: the reader's update owes it until the upstream serves or drops it again, + /// rather than dropping it again as if the reader had judged it stale itself. + #[test] + #[cfg(feature = "async")] + #[expect(clippy::single_range_in_vec_init, reason = "the reopened runs are a list, here one")] + fn a_superseded_received_stale_report_stays_owed_after_an_update() { + let producer = track_producer("test", Info::timed()); + let budget = Duration::from_millis(500); + let mut subscriber = producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_ranges([Range::groups(0..)]), + ); + let waiter = kio::Waiter::noop(); + producer.give_up(1..3, Hole::stale(budget)); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..3, Hole::stale(budget)))); + producer.supersede_stale(None); + assert_eq!(subscriber.reoffer_stale(), [1..3]); + assert!( + subscriber.poll_drop(&waiter).is_pending(), + "no verdict under the new terms" + ); + // The upstream drops it again under the new terms: reported again. + producer.give_up(1..3, Hole::stale(budget)); + assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..3, Hole::stale(budget)))); + } + + /// A bounded reader is settled by a received stale run only when it reads every group the + /// upstream's ranges name above the run; otherwise it still wants the groups the run names. + #[test] + #[cfg(feature = "async")] + fn a_received_stale_run_settles_a_bounded_reader_only_through_the_judged_ranges() { + let producer = track_producer("test", Info::timed()); + producer.received(); + // As its serving session does, so an unsettled run is asked rather than given up. + let _dynamic = producer.requests(); + let bounded = |max: Duration| { + producer.subscribe( + Subscription::default() + .with_delay(Delay::default().with_max(max)) + .with_ranges([Range::groups(0..2)]), + ) + }; + let mut wider = bounded(Duration::from_secs(5)); + let mut matching = bounded(Duration::from_millis(500)); + let waiter = kio::Waiter::noop(); + // The upstream judged its own range of 0..2 under 500ms. + producer.supersede_stale(judged([(0, 2)])); + producer.give_up(0..2, Hole::stale(Duration::from_millis(500))); + assert!( + wider.poll_recv_group(&waiter).is_pending(), + "a wider budget still wants the groups" + ); + assert_eq!( + matching.poll_drop(&waiter), + Poll::Ready((0..2, Hole::stale(Duration::from_millis(500)))) + ); + assert!( + matches!(matching.poll_recv_group(&waiter), Poll::Ready(Ok(None))), + "nothing is owed" + ); + // Judged against an open range, a capped reader may see an older edge. + producer.supersede_stale(None); + producer.give_up(0..2, Hole::stale(Duration::from_millis(500))); + let mut capped = bounded(Duration::from_millis(500)); + assert!(capped.poll_drop(&waiter).is_pending()); + write_group(&producer, 0); + assert_eq!(drain_groups(&mut wider), [0]); + } + + /// A report describes a miss, not a delivery: a late group is still handed out once. + #[test] + #[cfg(feature = "async")] + fn a_reported_drop_still_allows_a_late_group() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay()); + let waiter = kio::Waiter::noop(); + producer.drop_groups(2..4, StreamError::NotFound); + assert_eq!( + subscriber.poll_drop(&waiter), + Poll::Ready((2..4, Hole::drop(StreamError::NotFound))) + ); + assert!(subscriber.poll_drop(&waiter).is_pending()); + write_group(&producer, 3); + assert_eq!(drain_groups(&mut subscriber), [3]); + assert!(subscriber.poll_drop(&waiter).is_pending()); + assert!(drain_groups(&mut subscriber).is_empty()); + } + + /// A drop ends a group still arriving with the drop's error at once, for the reader + /// already holding it, before any stream reset or track end. A group that finished here + /// stays whole. + #[test] + #[cfg(feature = "async")] + fn a_drop_ends_a_held_group_with_its_error() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay()); + let mut open = producer.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut finished = producer.create_group(group::Info { sequence: 1 }).unwrap(); + finished.write_frame(Timestamp::ZERO, &b"b"[..]).unwrap(); + finished.finish().unwrap(); + + let mut held = subscriber.assert_group(); + assert_eq!( + &held.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[..], + b"a" ); assert!( held.read_frame().now_or_never().is_none(), @@ -22358,7 +23541,7 @@ mod test { let waiter = kio::Waiter::noop(); assert!(matches!( relay.poll_drop(&waiter), - Poll::Ready((range, StreamError::App(5))) if range == (0..1) + Poll::Ready((range, Hole::Drop { error: StreamError::App(5), .. })) if range == (0..1) )); } @@ -22534,6 +23717,9 @@ mod test { }]; for dropped in [false, true] { let producer = track_producer("test", Info::timed()); + // A received copy whose wire carries media time, so group 1 may still arrive. + producer.received(); + producer.anchored(); let mut zero = producer.create_group(group::Info { sequence: 0 }).unwrap(); zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); let mut two = producer.create_group(group::Info { sequence: 2 }).unwrap(); @@ -24067,6 +25253,8 @@ mod test { #[cfg(feature = "async")] async fn a_bounded_read_ends_after_its_fill() { let (producer, _pool) = pooled_producer(1 << 40); + // A received copy, whose missing groups may still arrive. + producer.received(); let dynamic = producer.requests(); producer.create_group(0u64.into()).unwrap().finish().unwrap(); producer.create_group(2u64.into()).unwrap().finish().unwrap(); diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index f813dc71a36..e06dfccb910 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -914,8 +914,8 @@ impl Fill { /// frames written keep the indices they have in the group. `info` sets the track's /// [`Info`] if it hasn't been accepted yet (a fetch with no live subscription), and is /// ignored otherwise. Errors with [`Error::InvalidRange`] outside the fill, - /// [`Error::Duplicate`] if the group was already answered or cached, or the track's abort - /// error. + /// [`Error::Duplicate`] if the group was already answered or cached, [`Error::Skipped`] if + /// the track skipped it, or the track's abort error. pub fn accept(&self, sequence: u64, info: impl Into>) -> Result { self.inner.accept(sequence, info.into()) } @@ -1479,7 +1479,7 @@ impl TrackState { } } for groups in dropped.iter() { - self.drop_groups(groups.clone(), err.clone()); + self.drop_groups(groups.clone(), super::Hole::drop(err.clone())); } } } diff --git a/rs/moq-net/src/model/track/hole.rs b/rs/moq-net/src/model/track/hole.rs new file mode 100644 index 00000000000..52312784100 --- /dev/null +++ b/rs/moq-net/src/model/track/hole.rs @@ -0,0 +1,62 @@ +use std::time::Duration; + +use crate::StreamError; + +/// Why a group sequence will not be delivered. +/// +/// lite-07 carries the two kinds in their own messages, SUBSCRIBE_SKIP and SUBSCRIBE_DROP; +/// earlier versions name both in SUBSCRIBE_DROP. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Hole { + /// Never produced, as the publisher declared (see `track::Producer::skip_groups`): final, + /// and it ends no reader. Never inferred from a gap, since groups may be created out of order. + Skip, + /// Produced but not delivered, ending any reader of it with `error`. + Drop { + error: StreamError, + /// The budget a staleness verdict was judged under, `None` for a drop that does not + /// depend on it. Only a widening past it reverses the verdict. + stale: Option, + }, +} + +impl Hole { + /// A drop with `error` that no budget reverses. + pub fn drop(error: StreamError) -> Self { + Self::Drop { error, stale: None } + } + + /// A group judged stale under `budget`. + pub fn stale(budget: Duration) -> Self { + Self::Drop { + error: StreamError::Old, + stale: Some(budget), + } + } + + /// A run a reader's misses gave up: stale under `stale` when that is the verdict, + /// otherwise simply not found. Never a skip, which only the publisher declares. + pub fn missed(stale: Option) -> Self { + match stale { + Some(budget) => Self::stale(budget), + None => Self::drop(StreamError::NotFound), + } + } + + /// The error a reader waiting on the sequence ends with: a sequence never produced is + /// simply not found. + pub fn error(&self) -> StreamError { + match self { + Self::Skip => StreamError::NotFound, + Self::Drop { error, .. } => error.clone(), + } + } + + /// The budget a staleness verdict was judged under, if this is one. + pub fn stale_budget(&self) -> Option { + match self { + Self::Drop { stale, .. } => *stale, + Self::Skip => None, + } + } +} diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 6ef6769d703..42dbabcc797 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -96,8 +96,9 @@ pub(super) struct Misses { refilled: RangeSet, } -/// A run of misses the reader gives up, for the drop cursor to name. -pub(super) type GivenUp = Vec>; +/// A run of misses the reader gives up, for the drop cursor to name, with the budget it was +/// judged stale under when that is why. +pub(super) type GivenUp = Vec<(ops::Range, Option)>; /// The joins whose fills changed their answer, recorded by each join's own waiter before it /// wakes the reader's tasks. @@ -625,7 +626,18 @@ impl Misses { if state.start_pending || !state.readable() && bound.is_none() { return asks; } - let Some(newest) = state.newest_produced.max(bound) else { + // A received stale run this reader cannot trust is still asked for, though nothing above + // it arrived: only an update brings it back otherwise, and a reader narrower than the + // upstream's terms changes none. See `TrackState::unfed`. Only runs above the newest + // group need a look. + let unknown = state + .drops + .iter() + .rev() + .take_while(|(run, _)| state.newest_produced.is_none_or(|newest| run.end - 1 > newest)) + .find(|(run, hole)| !drift.settles(state, run.start, hole)) + .map(|(run, _)| run.end - 1); + let Some(newest) = state.newest_produced.max(bound).max(unknown) else { return asks; }; let at = ( @@ -715,6 +727,16 @@ impl Misses { for fresh in self.judged.gaps(run) { candidates.insert(fresh); } + // A received stale run settles this reader only under the terms it judged, so + // what it settled is judged again under new ones. + for (dropped, hole) in state.drops.overlapping(run) { + if hole.stale_budget().is_some() { + let dropped = dropped.start.max(run.start)..dropped.end.min(run.end); + for judged in self.judged.overlapping(&dropped) { + candidates.insert(judged.start.max(dropped.start)..judged.end.min(dropped.end)); + } + } + } } } else { // A moving read floor removes only its retired prefix. New production extends the @@ -898,7 +920,7 @@ impl Misses { self.filled.extend(held.clone()); } for hole in holes(state, run.clone()) { - for hole in self.unclaimed(state, hole) { + for hole in self.unclaimed(state, hole, drift) { let (unfed, awaited) = state.unfed(hole, now); for run in unfed.iter() { asks.insert(run.clone()); @@ -956,7 +978,7 @@ impl Misses { .collect(); for run in stale { asks.remove(run.clone()); - self.give_up(run, given_up); + self.give_up_stale(run, drift.budget, given_up); } asks } @@ -1076,8 +1098,12 @@ impl Misses { released.insert(run.clone()); } self.release(&released); + // Below an old cache's floor is a boundary, not a drift verdict; above it, a stretch + // went stale under this reader's budget. + let floor = state.cache_floor(drift.budget); for run in stale.iter() { - self.give_up(run.clone(), given_up); + self.give_up(run.start..run.end.min(floor), given_up); + self.give_up_stale(run.start.max(floor)..run.end, drift.budget, given_up); } } @@ -1204,13 +1230,16 @@ impl Misses { } } - /// The part of `hole` nobody has a verdict on yet. - fn unclaimed(&self, state: &TrackState, hole: ops::Range) -> Vec> { + /// The part of `hole` nobody has a verdict on yet: a received stale run that does not + /// settle this reader is no verdict for it (see `Drift::settles`). + fn unclaimed(&self, state: &TrackState, hole: ops::Range, drift: &Drift) -> Vec> { let mut left = RangeSet::new(); left.insert(hole.clone()); // Only what overlaps the hole, so judging every run costs each one the same. - for (run, _) in state.drops.overlapping(&hole) { - left.remove(run.clone()); + for (run, gone) in state.drops.overlapping(&hole) { + if drift.settles(state, run.start.max(hole.start), gone) { + left.remove(run.clone()); + } } for claimed in [&self.given_up, &self.wanted] { for run in claimed.overlapping(&hole) { @@ -1244,13 +1273,31 @@ impl Misses { } } + /// Judge `groups` again: an update reopened the stale verdict they were given up with. + /// They are no longer judged, and the next probe judges its whole span afresh, so it + /// asks a fill for them or gives them up again. + pub(super) fn retry(&mut self, groups: ops::Range) { + self.given_up.remove(groups.clone()); + self.judged.remove(groups); + self.at = None; + } + pub(super) fn give_up(&mut self, groups: ops::Range, given_up: &mut GivenUp) { + self.give_up_judged(groups, None, given_up); + } + + /// Give up `groups` as stale under `budget`, a verdict a wider budget reverses. + fn give_up_stale(&mut self, groups: ops::Range, budget: Duration, given_up: &mut GivenUp) { + self.give_up_judged(groups, Some(budget), given_up); + } + + fn give_up_judged(&mut self, groups: ops::Range, stale: Option, given_up: &mut GivenUp) { if groups.is_empty() { return; } self.given_up.insert(groups.clone()); self.newly_gone.insert(groups.clone()); - given_up.push(groups); + given_up.push((groups, stale)); } } @@ -1380,22 +1427,35 @@ impl TrackState { unfed.insert(hole); return (unfed, awaited); } - let mut from = hole.start; let grace = self.miss_grace(); let mut gap = |groups: ops::Range, unfed: &mut RangeSet| match self.arrivals.folds_at(groups.start, grace) { Some(at) if at <= now => unfed.insert(groups), folds => awaited.push((groups, folds)), }; - for arrived in self.arrivals.accounted(from..hole.end) { - if from < arrived.start { - gap(from..arrived.start, &mut unfed); + // The upstream accounted for a received stale run, so the feed brings none of it unless + // an update supersedes the run. + let mut fed = RangeSet::new(); + fed.insert(hole.clone()); + for (run, given) in self.drops.overlapping(&hole) { + if given.stale_budget().is_some() { + let run = run.start.max(hole.start)..run.end.min(hole.end); + fed.remove(run.clone()); + unfed.insert(run); } - from = arrived.end; - unfed.insert(arrived); } - if from < hole.end { - gap(from..hole.end, &mut unfed); + for hole in fed.iter() { + let mut from = hole.start; + for arrived in self.arrivals.accounted(hole.clone()) { + if from < arrived.start { + gap(from..arrived.start, &mut unfed); + } + from = arrived.end; + unfed.insert(arrived); + } + if from < hole.end { + gap(from..hole.end, &mut unfed); + } } (unfed, awaited) } @@ -1545,7 +1605,9 @@ mod tests { .unwrap(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); - for sequence in [0, 4] { + // Newest first, so the publisher skips nothing below it: group 2 is a live gap a fill + // may answer. + for sequence in [4, 0] { track .create_group(crate::group::Info { sequence }) .unwrap() @@ -1561,6 +1623,7 @@ mod tests { budget, bounded: false, edge: state.drift_edge(&Servable::ALL), + coverage: None, }; let asks = misses.judge( &state, @@ -1687,7 +1750,9 @@ mod tests { .unwrap(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); - for sequence in [0, 4] { + // Newest first, so the publisher skips nothing below it: group 2 is a live gap a fill + // may answer. + for sequence in [4, 0] { track .create_group(crate::group::Info { sequence }) .unwrap() @@ -1703,6 +1768,7 @@ mod tests { budget, bounded: false, edge: state.drift_edge(&Servable::ALL), + coverage: None, }; let asks = misses.judge( &state, @@ -1753,12 +1819,14 @@ mod tests { &waiter, ); assert!(!asks.contains(&2), "a refused hole is not asked again"); - assert_eq!(given_up, vec![2..3]); + assert_eq!(given_up, vec![(2..3, None)], "a refusal is no stale verdict"); } #[kio::test] async fn unchanged_probes_and_late_arrivals_keep_other_gap_deadlines() { let broadcast = crate::broadcast::Info::default().produce(); let track = broadcast.create_track("sparse", crate::track::Info::default()).unwrap(); + // A received copy, whose live gaps are filled or waited for, not skipped. + track.received(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); for sequence in [0, 2, 4] { @@ -1776,6 +1844,7 @@ mod tests { budget, bounded: false, edge: state.drift_edge(&Servable::ALL), + coverage: None, }; assert!( misses @@ -1852,8 +1921,37 @@ mod tests { &waiter, ); assert!(asks.is_empty(), "the successor has aged past this reader's budget too"); - assert_eq!(given_up, vec![3..4]); + assert_eq!( + given_up, + vec![(3..4, Some(drift.budget))], + "judged stale under the reader's budget" + ); assert!(misses.deferred.runs.is_empty()); + + // An update reopens the verdict: retried, the gap is judged afresh, and a wider budget + // asks a fill for it. + misses.retry(3..4); + let wider = Drift { + budget: Duration::from_secs(3600), + bounded: false, + edge: state.drift_edge(&Servable::ALL), + coverage: None, + }; + given_up.clear(); + let asks = misses.judge( + &state, + (0, 0, None), + None, + None, + None, + |span| [span].into_iter().collect(), + &Delivered::default(), + &wider, + &mut given_up, + &waiter, + ); + assert!(asks.contains(&3), "a retried gap is asked again"); + assert!(given_up.is_empty()); } /// A grace change with nothing deferred has no deadline to move, so an unchanged probe @@ -1864,6 +1962,8 @@ mod tests { let track = broadcast .create_track("grace", crate::track::Info::default().with_max_age(None)) .unwrap(); + // A received copy, whose live gaps are filled or waited for, not skipped. + track.received(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); for sequence in 0..3 { @@ -1881,6 +1981,7 @@ mod tests { budget, bounded: false, edge: state.drift_edge(&Servable::ALL), + coverage: None, }; let full = |span| [span].into_iter().collect(); assert!( @@ -2061,6 +2162,7 @@ mod tests { budget, bounded: false, edge: state.drift_edge(&servable), + coverage: None, }; let expected = each(&misses, &state, &drift); let found = misses.expired(&state, &drift, &kio::Waiter::noop()); @@ -2125,15 +2227,15 @@ mod tests { let mut track = broadcast.create_track("moved", crate::track::Info::default()).unwrap(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); - // A feed with holes, some of them under where it starts. - for sequence in 0..SEQUENCES { - if next(3) > 0 { - track - .create_group(crate::group::Info { sequence }) - .unwrap() - .finish() - .unwrap(); - } + // A feed with holes, some of them under where it starts, created newest first so the + // publisher skips none of them. + let sequences: Vec<_> = (0..SEQUENCES).filter(|_| next(3) > 0).collect(); + for &sequence in sequences.iter().rev() { + track + .create_group(crate::group::Info { sequence }) + .unwrap() + .finish() + .unwrap(); } if next(2) == 0 { track.start_at(next(SEQUENCES / 2)).unwrap(); @@ -2174,6 +2276,7 @@ mod tests { budget, bounded: false, edge: state.drift_edge(&Servable::ALL), + coverage: None, }; let asks = misses.judge( &state, diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 05f2f1600cc..a5cb88c1778 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -12,6 +12,8 @@ use std::{ops::Range, task::Poll, time::Duration}; +use rangemap::RangeSet; + pub(crate) use crate::ledger::{GRACE, grace}; use crate::{ ledger::Ledger, @@ -28,6 +30,10 @@ pub(crate) struct Tail { // The publisher's FIN ended the track, its only end before lite-05, so a session that // dies before the tail settles declares that end before aborting. ended: bool, + // Groups whose stream finished since the last update was sent. A lite-07 stale drop under + // that update names one only if it was sent before the publisher served the group, so the + // group won over it, even once the cache evicted it. + finished: RangeSet, } impl Default for Tail { @@ -44,6 +50,7 @@ impl Tail { streams: 0, active: 0, ended: false, + finished: RangeSet::new(), } } @@ -71,6 +78,12 @@ impl Tail { self.expire(now); } + /// Owe `groups` again from `now`: a stale drop that accounted for them was reversed by a + /// widening, so the publisher serves or drops them anew. + pub fn unaccount(&mut self, groups: Range, now: Instant) { + self.ledger.unaccount(groups, now); + } + /// Restart the age of every gap reaching into `groups`, which the demand newly asks for. pub fn demand(&mut self, groups: Range, now: Instant) { self.ledger.demand(groups, now); @@ -97,6 +110,17 @@ impl Tail { self.ledger.folds_at(group, self.ledger.grace()) } + /// The runs of `groups` whose stream finished since the last update was sent. + pub fn finished(&self, groups: Range) -> impl Iterator> + '_ { + self.finished.overlapping(groups).cloned() + } + + /// Forget which streams finished, for an update being sent: a stale drop under an earlier + /// one is no verdict, so none needs them. + pub fn updated(&mut self) { + self.finished = RangeSet::new(); + } + /// Whether a data stream whose header arrived is still being read. pub fn is_reading(&self) -> bool { self.active > 0 @@ -111,6 +135,7 @@ impl Tail { /// A data stream whose header arrived, holding its subscription's end open until dropped. pub(crate) struct Reading { tail: Option>, + group: Option, active: bool, } @@ -120,6 +145,7 @@ impl Reading { let Ok(mut state) = tail.write() else { return Self { tail: None, + group, active: false, }; }; @@ -130,10 +156,20 @@ impl Reading { } Self { tail: Some(tail.clone()), + group, active: true, } } + /// Record that the stream's group arrived whole, once it finished. + pub fn finish(&self) { + if let (Some(tail), Some(group)) = (&self.tail, self.group) + && let Ok(mut state) = tail.write() + { + state.finished.insert(group..group.saturating_add(1)); + } + } + /// Stop holding the end open while the stream waits on another one, which holds it /// open itself if its header arrived. pub fn park(&mut self) { diff --git a/swift/Sources/Moq/Track.swift b/swift/Sources/Moq/Track.swift index d2d58cda5c9..8ed81f912ef 100644 --- a/swift/Sources/Moq/Track.swift +++ b/swift/Sources/Moq/Track.swift @@ -163,6 +163,12 @@ public final class TrackProducer: Sendable { GroupProducer(try ffi.createGroup(sequence: sequence)) } + /// Declare that groups `start` up to the exclusive `end` will never be produced. + /// Readers move past them, and creating a group there then throws. + public func skipGroups(start: UInt64, end: UInt64) throws { + try ffi.skipGroups(start: start, end: end) + } + /// Write a single-frame group with a presentation timestamp, or untimed when nil. public func writeFrame(_ payload: Data, timestamp: Timestamp? = nil) throws { try ffi.writeFrame(frame: Frame(payload: payload, timestamp: timestamp))