diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 2a53f2a5f45..6fd5378f77c 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -77,6 +77,12 @@ jobs: run: nix develop --command just test stream-limits shell: bash -leo pipefail {0} + - name: Stream credit in browsers + id: stream-credit + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test stream-credit + shell: bash -leo pipefail {0} + - name: Relay fills id: relay-fill if: ${{ !cancelled() && steps.setup.outcome == 'success' }} @@ -227,6 +233,7 @@ jobs: "Optional publisher retention": "${{ steps.max-age.outcome }}", "Datagram tracks": "${{ steps.datagram-tracks.outcome }}", "Stream limits over WebSocket": "${{ steps.stream-limits.outcome }}", + "Stream credit in browsers": "${{ steps.stream-credit.outcome }}", "Relay fills": "${{ steps.relay-fill.outcome }}", "Receive credit recovery": "${{ steps.receive-credit.outcome }}", "Harness regressions": "${{ steps.harness.outcome }}", diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 58ce3bd9c6b..d3f4724bc50 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -886,6 +886,7 @@ The GROUP message's `Frame Start` gives the index of the first FRAME on the stre A publisher MUST NOT send two Group Streams for the same Group Sequence on one subscription, except to re-offer a Group whose stream it reset (see [SUBSCRIBE_DROP](#subscribe-drop)), or to [continue](#yield) one. The re-offer can arrive before that reset: a subscriber holds a Group Stream for a sequence whose earlier stream is still arriving unread until that stream ends, then discards it if the earlier stream completed the Group, or reads it if the earlier stream was reset. +Since the publisher reset the earlier stream, the subscriber MAY instead end that stream at once and read the later one, ignoring the frames it holds; a browser can lose a reset that reaches a stream before its application does. A subscriber assembling a Group from more than one publisher does so across separate subscriptions, and is responsible for concatenating the runs in index order and for ignoring any frame it has already received. Both the publisher and subscriber MAY reset the stream at any time. @@ -899,7 +900,8 @@ A Group Stream carrying the `ietf` extension's object records is never reset wit A Group whose stream was reset with YIELDED is paused, not given up: the subscriber keeps the frames it received, and the publisher MUST later continue the Group on another Group Stream or name it in [SUBSCRIBE_DROP](#subscribe-drop). A continuation is a Group Stream for the paused Group whose `Frame Start` is the reset stream's own `Frame Start`, since a reset can discard frames the transport acknowledged. The subscriber ignores the frames it already holds, including the part of a frame the reset cut short, and appends the rest; a continuation that starts past the frames it holds whole is malformed. -A continuation MAY overtake the earlier stream's reset, and waits unread until that stream ends. +A continuation MAY overtake the earlier stream's reset, and waits unread until that stream ends, or ends it, as for a re-offer. +A subscriber whose transport reports a Group Stream's reset without its error code, as a browser can, pauses the Group as for YIELDED: the publisher's continuation or SUBSCRIBE_DROP settles it. ## Datagrams QUIC datagrams provide unreliable, unordered delivery for latency-sensitive content that does not need retransmission. @@ -2470,6 +2472,7 @@ An application-specific payload. - Expiration also measures a group's arrival age, the time since its successor arrived at this hop, so a group of an untimed Track expires by arrival age alone. - A publisher resets the Group Stream of an expired, unfinished group with OLD rather than finishing it, even after sending every frame it holds. - Added [YIELDED](#yield) (0x3C): a publisher out of stream credit resets the lowest-priority Group Stream with it, which pauses the Group for a later Group Stream to continue from where the reset stream started. +- A subscriber MAY end a Group Stream once a later stream for its Group arrives, rather than wait for its reset, and pauses a Group whose reset arrives without its error code as for YIELDED. - FRAME leads with `Payload Length`, the payload size plus one, in place of the trailing `Message Length`; 0 is malformed. - A publisher SHOULD drop a datagram rather than block when the transport has no room for it, where it previously sent one only if the congestion controller could transmit it immediately. - Added an optional `Subscriber Max Age` to FETCH, encoded as milliseconds plus one with zero meaning absent. It can tighten, but never relax, `Publisher Max Age`. diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 607c4133f33..0e5900e591d 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -445,7 +445,7 @@ test("a tail forgets its finished streams under debt", () => { expect(pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); // A stale drop inside what the fold forgot is owed again only where the ledger had not accounted // for it: there a stream may have finished and overtaken the drop. - tail.open(2n)(); + tail.open(2n).done(); expect(tail.overtaken(0n, 4n)).toEqual([{ start: 2n, end: 3n }]); expect(tail.overtaken(2n * runs, 2n * runs + 1n)).toEqual([]); tail.updated(); @@ -453,6 +453,30 @@ test("a tail forgets its finished streams under debt", () => { tail.close(); }); +// A tail's fragmented final drops bill its track's account too. Under the pool's debt they fold into +// the one span they reach, which still names every dropped sequence, so a stream a drop overtook is +// never held as owed, and the gaps between them are excused at worst. +test("a tail folds its final drops under debt", () => { + const runs = 4000n; + const pool = new cache.Pool({ capacity: 1 << 30 }); + const producer = pooled(pool); + hooks.receiveTrack(producer); + const tail = new Tail({ bill: hooks.billTrack(producer) }); + for (let run = 0n; run < runs; run++) tail.drop(2n * run, 2n * run + 1n); + expect(tail.dropped(2n)).toBe(true); + expect(tail.dropped(3n)).toBe(false); + expect(pool.used).toBeGreaterThanOrEqual(Number(runs) * cache.READER_RUN_OVERHEAD); + + const capacity = 64 * cache.RUN_OVERHEAD; + pool.resize(capacity); + for (let i = 0; i < 64; i++) finished(producer, 16 * 1024); + tail.drop(2n * runs, 2n * runs + 1n); + expect(pool.used).toBeLessThanOrEqual(capacity + 2 * (16 * 1024 + cache.ENTRY_OVERHEAD)); + for (let run = 0n; run <= runs; run++) expect(tail.dropped(2n * run)).toBe(true); + expect(tail.dropped(2n * runs + 1n)).toBe(false); + tail.close(); +}); + // Whatever a bill frees pays its readers' debt, however it shrank, and the debt is forgiven once // the pool is back within its capacity. A closed tail never bills again. Mirrors Rust // `a_debt_is_paid_by_any_shrink_and_forgiven_without_pressure`. diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 2944b802ffc..a0a94912115 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -2081,7 +2081,7 @@ export class Subscriber { } stream.stop(e); } finally { - read(); + read.done(); } } @@ -2202,7 +2202,7 @@ export class Subscriber { if (e instanceof StreamError && e.code === StreamCode.MalformedTrack) track.abort(e); stream.stop(e); } finally { - read?.(); + read?.done(); } } diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 212fd5e044a..99e94242af0 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1395,13 +1395,15 @@ function handStream(version: Version) { write: (chunk: Uint8Array) => controller.enqueue(chunk), finish: () => controller.close(), reset: (code: StreamCode) => controller.error(toTransport(code, "reset")), + error: (err: Error) => controller.error(err), }; } // Lite-07: a stream reset with YIELDED pauses its group, keeping the frame it cut short, and a later // stream continues it from the paused stream's Frame Start, whether it lands after that reset or -// overtakes it. The group's reader gets every frame once, then its end. Mirrors the Rust -// `a_yielded_group_continues_without_loss`. +// overtakes it, which ends the paused stream at once, since a browser can lose its reset. The +// group's reader gets every frame once, then its end. Mirrors the Rust +// `a_yielded_group_continues_without_loss`, which waits for the overtaken reset instead. test.each([false, true])("a yielded group continues without loss (overtakes: %p)", async (overtakes) => { const version = Version.DRAFT_07; const { quic, streams, opened } = fakeSession(); @@ -1435,20 +1437,19 @@ test.each([false, true])("a yielded group continues without loss (overtakes: %p) expect(group?.sequence).toBe(3n); expect((await group?.readFrame())?.payload).toEqual(frames[0].payload); - let continued: Promise | undefined; + let continued: Promise; if (overtakes) { - let held = true; - continued = subscriber.runGroup(header, second.reader).finally(() => { - held = false; - }); + const stop = spyOn(first.reader, "stop"); + continued = subscriber.runGroup(header, second.reader); + await yielded; + expect(stop).toHaveBeenCalled(); + } else { + first.reset(StreamCode.Yielded); + await yielded; await drainReactions(); - expect(held).toBe(true); + expect(group?.closed.peek()).toBeUndefined(); + continued = subscriber.runGroup(header, second.reader); } - first.reset(StreamCode.Yielded); - await yielded; - await drainReactions(); - expect(group?.closed.peek()).toBeUndefined(); - continued ??= subscriber.runGroup(header, second.reader); await continued; for (const { payload } of frames.slice(1)) { @@ -1652,6 +1653,347 @@ test("a yielded group pauses once an update supersedes a stale drop before its h } }); +// Lite-07: Firefox reports a reset that lands before the page read the stream's data as a bare +// TypeError, dropping its code. The publisher owes that group a continuation or a drop, so the +// group pauses as on YIELDED, and whichever comes settles it. +test.each(["continued", "dropped"] as const)("a reset without its code pauses the group (%s)", async (then) => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber + .consume(Path.from("room")) + .track("video") + .subscribe({ delay: { max: Duration.millis(60_000) } }); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + + const first = handStream(version); + first.write(await liteFrame(300, new Uint8Array([1]))); + const running = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 3n }), first.reader); + const group = await track.recvGroup(); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([1])); + first.error(new TypeError("Error in input stream")); + await running; + await drainReactions(); + expect(group?.closed.peek()).toBeUndefined(); + + if (then === "continued") { + const second = handStream(version); + second.write(await liteFrame(300, new Uint8Array([1]))); + second.write(await liteFrame(10, new Uint8Array([2]))); + second.finish(); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence: 3n }), second.reader); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([2])); + expect(await group?.readFrame()).toBeUndefined(); + expect(await group?.closed).toBeNull(); + } else { + await respond(request, version, [ + { drop: new SubscribeDrop({ start: 3n, end: 4n, error: StreamCode.NotFound }) }, + ]); + const err = await group?.readFrame().then( + () => undefined, + (e: unknown) => e, + ); + expect((err as StreamError).code).toBe(StreamCode.NotFound); + } + } finally { + track.close(); + subscriber.close(); + } +}); + +// Lite-07: Chromium can lose the reset of a Group Stream it has not handed to the page yet, which +// leaves the stream open forever. A drop accounts for its group, so the subscription's end waits +// for that stream at most the grace: a stale drop before or after its header, or a final one before +// it, even once a later stale drop of the group expires, whether it came before or after the header. A final drop after it ends the read at once. +test.each([ + ["stale", "before"], + ["stale", "after"], + ["final", "before"], + ["final then stale", "before"], + ["final then stale", "both before"], +] as const)("a drop lets the end give up on a stream that never ends (%s drop %s its header)", async (kind, order) => { + const clock = new Manual(); + const installed = clock.install(); + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + + const error = kind === "stale" ? StreamCode.Old : StreamCode.NotFound; + const drop = { drop: new SubscribeDrop({ start: 0n, end: 1n, error }) }; + const stale = { drop: new SubscribeDrop({ start: 0n, end: 1n, error: StreamCode.Old }) }; + if (order !== "after") { + await respond(request, version, order === "both before" ? [drop, stale] : [drop]); + await drainReactions(); + } + const lost = handStream(version); + lost.write(await liteFrame(0, new Uint8Array([1]))); + void subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), lost.reader); + const group = await track.recvGroup(); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([1])); + if (order === "after") await respond(request, version, [drop]); + if (kind === "final then stale" && order === "before") await respond(request, version, [stale]); + await respond(request, version, [{ finish: new SubscribeFinish(1n) }]); + await drainReactions(); + // The update expires the stale drop, but not the final one beneath it. + if (kind === "final then stale") { + track.update({ priority: 1 }); + await drainReactions(); + } + + let fin = false; + void request.finished.then(() => { + fin = true; + }); + await clock.advance(GRACE.div(2)); + expect(fin).toBe(false); + await clock.advance(GRACE); + expect(fin).toBe(true); + // The publisher's FIN in reply ends the track at once, without another grace. + request.inbound.close(); + expect(await track.recvGroup()).toBeUndefined(); + expect(await track.closed).toBeNull(); + } finally { + track.close(); + subscriber.close(); + installed.close(); + } +}); + +// Lite-07: a header that arrives after the grace gave up on its gap is covered by the tail, but +// no drop accounts for it, so its stream is still delivering and holds the end until it is read. +test("a stream whose gap the grace gave up on still holds the end", async () => { + const clock = new Manual(); + const installed = clock.install(); + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + // Group 2 opens a gap at 1, which the grace gives up on by the time group 3 arrives. + for (const sequence of [0n, 2n, 3n]) { + const stream = handStream(version); + stream.finish(); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence }), stream.reader); + await clock.advance(GRACE.mul(2)); + } + + const late = handStream(version); + late.write(await liteFrame(0, new Uint8Array([1]))); + const running = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 1n }), late.reader); + await drainReactions(); + await respond(request, version, [{ finish: new SubscribeFinish(4n) }]); + await drainReactions(); + + let fin = false; + void request.finished.then(() => { + fin = true; + }); + await clock.advance(GRACE.mul(3)); + expect(fin).toBe(false); + late.finish(); + await running; + await drainReactions(); + expect(fin).toBe(true); + } finally { + track.close(); + subscriber.close(); + installed.close(); + } +}); + +// Lite-07: a recovery holds its FIN for a group a reset without its code paused, as a subscription's +// end does, rather than end the continuation the publisher still owes it. +test("a recovery waits for the continuation of a group a reset without its code paused", async () => { + const clock = new Manual(); + const installed = clock.install(); + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const consumed = subscriber.consume(Path.from("room")); + const live = consumed.track("video").subscribe({ delay: { max: null } }); + let history: ReturnType["subscribe"]> | undefined; + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + // The subscription delivers groups 0 and 1, which age out of the cache below the live edge, + // so it never sends them again. + for (const sequence of [0n, 1n, 10n]) { + const stream = handStream(version); + stream.finish(); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence }), stream.reader); + } + await clock.advance(DEFAULT_EXPIRY.mul(2)); + + // A reader asks for them again, which a recovery of their own answers. + history = consumed.track("video").subscribe({ + delay: { max: null }, + order: "ascending", + ranges: [{ start: { absolute: 0n }, end: { absolute: 2n } }], + }); + const zeroRead = history.recvGroup(); + const recovery = await opened(2); + await recovery.reading; + await respond(recovery, version, [{ start: new SubscribeStart([0n]) }]); + // Over once it FINs, or once the fills cancel it with nothing left to read. + let over = false; + void Promise.race([recovery.finished, recovery.aborted]).then(() => { + over = true; + }); + + const first = handStream(version); + first.write(await liteFrame(0, new Uint8Array([1]))); + void subscriber.runGroup(new Group({ subscribe: 1n, sequence: 0n }), first.reader).catch(() => {}); + const zero = await zeroRead; + expect(zero?.sequence).toBe(0n); + expect((await zero?.readFrame())?.payload).toEqual(new Uint8Array([1])); + const one = handStream(version); + one.finish(); + await subscriber.runGroup(new Group({ subscribe: 1n, sequence: 1n }), one.reader); + await drainReactions(); + expect(over).toBe(false); + // Firefox reports the YIELDED reset as a bare TypeError, which pauses the group. + first.error(new TypeError("Error in input stream")); + await drainReactions(); + await clock.advance(GRACE.div(2)); + expect(over).toBe(false); + + // The continuation completes the group, which lets the recovery end. + const second = handStream(version); + second.write(await liteFrame(0, new Uint8Array([1]))); + second.write(await liteFrame(10, new Uint8Array([2]))); + second.finish(); + await subscriber.runGroup(new Group({ subscribe: 1n, sequence: 0n }), second.reader); + expect((await zero?.readFrame())?.payload).toEqual(new Uint8Array([2])); + expect(await zero?.readFrame()).toBeUndefined(); + await drainReactions(); + expect(over).toBe(true); + } finally { + history?.close(); + live.close(); + subscriber.close(); + installed.close(); + } +}); + +// Lite-07: an update reopens a stale drop, so a stream that drop excused holds the end again +// until it finishes, even once the grace the end started before the update has passed. +test("an update restores the hold a stale drop excused", async () => { + const clock = new Manual(); + const installed = clock.install(); + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + + const slow = handStream(version); + slow.write(await liteFrame(0, new Uint8Array([1]))); + const running = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), slow.reader); + const group = await track.recvGroup(); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([1])); + await respond(request, version, [{ drop: new SubscribeDrop({ start: 0n, end: 1n, error: StreamCode.Old }) }]); + // SUBSCRIBE_FINISH starts the end's grace, and the update then widens the budget. + await respond(request, version, [{ finish: new SubscribeFinish(1n) }]); + await drainReactions(); + track.update({ priority: 0, delay: { max: Duration.millis(60_000) } }); + await drainReactions(); + + let fin = false; + void request.finished.then(() => { + fin = true; + }); + await clock.advance(GRACE.mul(3)); + expect(fin).toBe(false); + // The stream finishes the group, which lets the end go. + slow.write(await liteFrame(10, new Uint8Array([2]))); + slow.finish(); + await running; + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([2])); + await request.finished; + } finally { + track.close(); + subscriber.close(); + installed.close(); + } +}); + +// Lite-07: a continuation judged against a stale drop while it waits out the copy it replaces holds +// the end if an update reopened that drop meanwhile, as the copy it replaces would have. +test("an update while a continuation waits keeps its hold on the end", async () => { + const clock = new Manual(); + const installed = clock.install(); + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + try { + await opened(0); + await answerTrackInfo(streams[0], version, 1000); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([0n]) }]); + // A stale drop overtakes the group's stream. + await respond(request, version, [{ drop: new SubscribeDrop({ start: 0n, end: 1n, error: StreamCode.Old }) }]); + await drainReactions(); + + const first = handStream(version); + first.write(await liteFrame(0, new Uint8Array([1]))); + void subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), first.reader).catch(() => {}); + const group = await track.recvGroup(); + expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([1])); + + // SUBSCRIBE_FINISH starts the end's grace. The continuation arrives under the stale drop, and + // the update lands while it waits out the first copy. + await respond(request, version, [{ finish: new SubscribeFinish(1n) }]); + await drainReactions(); + const second = handStream(version); + second.write(await liteFrame(0, new Uint8Array([1]))); + const running = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), second.reader); + track.update({ priority: 0, delay: { max: Duration.millis(60_000) } }); + await drainReactions(); + + let fin = false; + void request.finished.then(() => { + fin = true; + }); + await clock.advance(GRACE.mul(3)); + expect(fin).toBe(false); + second.write(await liteFrame(10, new Uint8Array([2]))); + second.finish(); + await running; + await request.finished; + } finally { + track.close(); + subscriber.close(); + installed.close(); + } +}); + // Lite-07: a malformed FRAME on a Group Stream ends only that stream and its group, never the // session: a Payload Length of 0, or a continuation that resends the cut frame at another size, ends // before completing it, or starts past the frames held. Mirrors the Rust `UniState::Group`. @@ -1741,10 +2083,10 @@ test("a lite-06 group reset with YIELDED ends", async () => { } }); -// A lite-07 re-offer can overtake the reset of the stream it replaces: it waits, unread, until that -// copy ends. It is refused as a duplicate once the copy finishes, and takes the group once it fails, -// continuing the reads of that copy from their frame. -test.each([false, true])("a duplicate of an arriving copy waits for it to settle (first fails: %p)", async (fails) => { +// A lite-07 re-offer can overtake the reset of the stream it replaces. It proves that reset, which +// a browser can lose, so it ends that copy at once and continues the group from the frames it +// holds: a reader already handed the group reads on, and a later one reads it whole. +test("a re-offer ends the copy still arriving and continues its group", async () => { const version = Version.DRAFT_07; const { quic, streams, opened } = fakeSession(); const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); @@ -1802,30 +2144,9 @@ test.each([false, true])("a duplicate of an arriving copy waits for it to settle undefined, version, ); - const stop = spyOn(duplicate, "stop"); - let held = true; - const receiving = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), duplicate).finally(() => { - held = false; - }); - await drainReactions(); - expect(held).toBe(true); - expect(stop).not.toHaveBeenCalled(); - - if (!fails) { - first.close(); - await original; - await receiving; - expect(stop.mock.calls[0]?.[0]).toBeInstanceOf(Error); - expect(await group?.readFrame()).toBeUndefined(); - return; - } - - // The reset lands: the reader already handed the original waits on the held stream rather - // than failing, and continues on it from its frame once it takes the group. const next = group?.readFrame(); - first.error(new Error("peer reset")); + await subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), duplicate); await original; - await receiving; expect((await next)?.payload).toEqual(new Uint8Array([8])); expect(await group?.readFrame()).toBeUndefined(); expect(await group?.closed).toBeNull(); @@ -3013,7 +3334,7 @@ test("a group wins over its stale drop on lite-07", async () => { const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); type Entry = { track: Producer; - tail: { covers(start: bigint, end: bigint): boolean; open(sequence: bigint): () => void }; + tail: { covers(start: bigint, end: bigint): boolean; open(sequence: bigint): { done(): void } }; }; let entry: Entry | undefined; const internal = subscriber as unknown as { @@ -3056,7 +3377,7 @@ test("a group wins over its stale drop on lite-07", async () => { await send(old(1n)); const one = arrive(1n); one.group.finish(); - one.read(); + one.read.done(); // The group first, still arriving when its OLD lands. const two = arrive(2n); await send(old(2n)); @@ -3075,8 +3396,8 @@ test("a group wins over its stale drop on lite-07", async () => { // Neither the drop nor the update aborts a group still arriving. expect(two.group.closed.peek()).toBeUndefined(); expect(three.group.closed.peek()).toBeUndefined(); - two.read(); - three.read(); + two.read.done(); + three.read.done(); } finally { responses.mockRestore(); track.close(); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 7bede1ea93c..94801db49f2 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1,5 +1,5 @@ import { Duration, sleep, yieldNow } from "@moq/clock"; -import { type Dispose, Effect, type GetPromise, type Getter, race, Signal } from "@moq/signals"; +import { type Dispose, Effect, type GetPromise, type Getter, Once, race, Signal } from "@moq/signals"; import * as announce from "../announced.ts"; import type { Grant } from "../auth.ts"; import type { AuthSession } from "../auth_session.ts"; @@ -50,7 +50,7 @@ import * as Mint from "../mint.ts"; import * as Path from "../path.ts"; import { sendOrder } from "../priority.ts"; import { type OpenOptions, type Reader, Stream, Writer } from "../stream.ts"; -import { Tail } from "../tail.ts"; +import { type Counted, Tail } from "../tail.ts"; import * as Time from "../time.ts"; import type * as track from "../track.ts"; import { untilAborted } from "../util/abort.ts"; @@ -281,16 +281,59 @@ interface PausedGroup { // Whether a YIELDED reset of a stream being read pauses its group. interface Yielding { verdict: YieldVerdict; + // The stream's hold on the subscription's end, which a drop naming its group excuses while + // that drop stands: see `excused`. + counted: Counted; + // End the stream as reset with YIELDED: a later stream for its group proves the publisher + // reset it, even where the browser lost that reset. + supersede: () => void; } -// Stale drops expire on the next update. An already accounted header stays accounted. +// Stale drops expire on the next update. An already accounted header stays accounted, and +// `dropped` says whether a final drop overtook it, rather than a gap the grace gave up on. type YieldVerdict = | { kind: "owed" } - | { kind: "accounted" } + | { kind: "accounted"; dropped: boolean } | { kind: "stale"; under: bigint; error: Error } - | { kind: "accounted-stale"; under: bigint; error: Error } + | { kind: "accounted-stale"; dropped: boolean; under: bigint; error: Error } | { kind: "dropped"; error: Error }; +// Whether a drop accounts for the group under `update`, so the subscription's end waits for its +// stream at most the grace: a browser can lose that stream's reset. Only a drop excuses it: a stream +// whose header came after the grace gave up on its gap is still delivering, and no reset is owed. +// A stale drop expires; one over a final drop falls back to it, as `yieldError` does. +function excused(verdict: YieldVerdict, update: bigint): boolean { + switch (verdict.kind) { + case "owed": + return false; + case "accounted": + return verdict.dropped; + case "stale": + return verdict.under >= update; + case "accounted-stale": + return verdict.dropped || verdict.under >= update; + case "dropped": + return true; + } +} + +// The verdict a header starts with: owed, or accounted by a final drop (`dropped`), a stale drop +// under `staleUnder`, or a gap the grace gave up on. A stale drop over a final one falls back to it. +function headerVerdict(owed: boolean, dropped: boolean, staleUnder: bigint | undefined): YieldVerdict { + if (owed) return { kind: "owed" }; + if (staleUnder === undefined) return { kind: "accounted", dropped }; + const error = new StreamError(StreamCode.Old); + return dropped + ? { kind: "accounted-stale", dropped, under: staleUnder, error } + : { kind: "stale", under: staleUnder, error }; +} + +// Excuse the stream's hold on the end, or restore it, as its verdict now says. +function judge(yielding: Yielding, update: bigint): void { + if (excused(yielding.verdict, update)) yielding.counted.excuse(); + else yielding.counted.restore(); +} + function yieldError(verdict: YieldVerdict, update: bigint): Error | undefined { switch (verdict.kind) { case "owed": @@ -2671,13 +2714,23 @@ export class Subscriber { ended = true; }, ); - while (!ended && !left.signal.aborted) { - if (entry.tail.covers(groups.start, groups.end) && !entry.tail.reading) { - fin = true; - opened.writer.close(); - break; + // An excused stream is waited for at most the grace, as at a subscription's end: a browser + // can lose its reset, while a group it paused waits on the continuation still owed. + const stopped = new Once(); + const covered = () => entry.tail.covers(groups.start, groups.end); + try { + while (!ended && !left.signal.aborted) { + if (covered()) { + await race([entry.tail.settle(covered, stopped), over, leaving]); + if (ended || left.signal.aborted) break; + fin = true; + opened.writer.close(); + break; + } + await race([Signal.race(entry.tail.changed), over, leaving]); } - await race([Signal.race(entry.tail.changed), over, leaving]); + } finally { + stopped.set(null); } // After our FIN, a cancel still stops the wait for the publisher's. if (!ended) await race([over, leaving]); @@ -2758,6 +2811,8 @@ export class Subscriber { const cause = resp.drop.cause(this.version); // A paused group the drop names ends there: no continuation follows. this.#dropPaused(entry, start, end, cause); + // Final, so a stream it overtook is told from one whose gap the grace gave up on. + entry.tail.drop(start, end); giveUp(start, end, dropped(cause)); } else if ("skip" in resp) { // Never produced: final, as the subscription feeding the copy records it. @@ -3046,7 +3101,9 @@ export class Subscriber { // Asked before this drop is accounted: what a fold forgot finishing that the ledger // already accounted may be a stream this drop overtook. const overtaken = stale ? entry.tail.overtaken(start, end) : []; - entry.tail.account(start, end); + // Lite-07 tells a stream a final drop overtook from one whose gap the grace gave up on. + if (!stale && hasYield(this.version)) entry.tail.drop(start, end); + else entry.tail.account(start, end); // A paused group the drop names ends there: no continuation follows. this.#dropPaused(entry, start, end, error, stale ? update : undefined); // Judged under the budget this subscriber sent with that update. @@ -3287,6 +3344,8 @@ export class Subscriber { entry.update = update.update; entry.maxDelay = update.maxDelay; entry.tail.updated(); + // A stream a stale drop excused holds the end again: that verdict no longer stands. + for (const yielding of entry.yielding?.values() ?? []) judge(yielding, entry.update); const held = hooks.supersedeStale(entry.track, judged([...ranges, ...fillRanges])); // A lite-07 publisher re-offers every group it reported stale on any update, and drops // again under the new number what is still stale, so the tail owes every stale run again. @@ -3405,6 +3464,8 @@ export class Subscriber { const yields = hasYield(this.version); // Asked before this stream accounts for the group: a drop that overtook it named it. let owed = !tail.covers(group.sequence, group.sequence + 1n); + // A final drop that overtook this stream, rather than a gap the grace gave up on. + const dropped = !owed && tail.dropped(group.sequence); // A stale drop that overtook this stream: the next update supersedes that verdict, so a // YIELDED reset after it pauses the group. const unowedUnder = @@ -3415,8 +3476,9 @@ export class Subscriber { let arriving: Arriving | undefined; let ingest: Ingest | undefined; let yielding: Yielding | undefined; + const counted = tail.open(group.sequence); // Until the stream is read, or a pause hands it to the paused group. - let read: (() => void) | undefined = tail.open(group.sequence); + let read: (() => void) | undefined = counted.done; this.#reoffered(track, group.sequence, group.sequence + 1n); try { @@ -3455,6 +3517,10 @@ export class Subscriber { arriving = next; // Served again once that copy fails or pauses, so owed again. owed = true; + // A later stream for the group means the publisher reset the earlier one on this + // subscription, so it ends now rather than wait on a reset the browser may have lost: + // Chromium drops a RESET_STREAM that reaches a stream it has not handed to the page. + entry.yielding?.get(group.sequence)?.supersede(); const settled = arriving.settled; const waits: unknown[] = [settled, stream.closed.then(() => settled)]; if (yields) waits.push(Signal.race(pausesOf(entry).changed)); @@ -3467,22 +3533,35 @@ export class Subscriber { }); // Released once this copy took the group, so the failed copy's readers find it. arriving?.release(); + let superseded: Promise | undefined; if (yields) { ingest ??= { index: group.frameStart }; + let supersede = () => {}; + superseded = new Promise((resolve) => { + supersede = resolve; + }); yielding = { - verdict: owed - ? { kind: "owed" } - : unowedUnder !== undefined - ? { kind: "stale", under: unowedUnder, error: new StreamError(StreamCode.Old) } - : { kind: "accounted" }, + verdict: headerVerdict(owed, dropped, unowedUnder), + counted, + supersede, }; entry.yielding ??= new Map(); entry.yielding.set(group.sequence, yielding); + // A stale drop that still stands may already account for the group. + judge(yielding, entry.update); } const format = { scale: timescale, compress: entry.compress, lengthFirst: hasLengthFirst(this.version) }; const reading = readFrames(stream, producer, format, ingest); - await (entry.filling ? untilAbandoned(producer, reading) : reading); + const reads = entry.filling ? untilAbandoned(producer, reading) : reading; + if (!superseded) await reads; + else if ((await race([reads, superseded.then(() => SUPERSEDED)])) === SUPERSEDED) { + // Stopping makes the read end at the next frame boundary as if finished, so wait it out + // and treat the stream as the YIELDED reset it was: the later stream starts no later. + stream.stop(new StreamError(StreamCode.Cancel)); + await reads.catch(() => {}); + throw new StreamError(StreamCode.Yielded, { message: "continued on a later stream" }); + } producer.finish(); // Only a group that arrived whole, not one a drop or the track ended meanwhile, and only @@ -3500,8 +3579,14 @@ export class Subscriber { // The publisher made room for a higher-priority stream: the group pauses, keeping what // arrived and the subscription's end waiting, until a later stream continues it or a drop // names it. Mirrors the Rust `GroupRecv::poll_serve`. - if (producer && ingest && yielding && toStreamCode(e) === StreamCode.Yielded) { + // A reset the browser lost the code of pauses the group too: a lite-07 publisher follows + // every reset with a continuation or a drop, unless a reliable reset kept the group's + // header, after which nothing follows, so the end waits for it at most the grace. Not a + // failure the session's close caused, which sessionCause swapped in. + const uncoded = stream.uncoded && e === err; + if (producer && ingest && yielding && (toStreamCode(e) === StreamCode.Yielded || uncoded)) { this.#endYielding(entry, group.sequence, yielding); + if (uncoded) counted.excuse(); // A drop named the group first: the publisher gave it up, so nothing continues it. One // that overtook the stream's header carried a stale verdict. // A stale drop an update sent since superseded no longer ends the group. @@ -3581,14 +3666,13 @@ export class Subscriber { const verdict = yielding.verdict; if (staleUnder === undefined) yielding.verdict = { kind: "dropped", error }; else if (verdict.kind !== "dropped") - yielding.verdict = { - kind: - verdict.kind === "accounted" || verdict.kind === "accounted-stale" - ? "accounted-stale" - : "stale", - under: staleUnder, - error, - }; + yielding.verdict = + verdict.kind === "accounted" || verdict.kind === "accounted-stale" + ? { kind: "accounted-stale", dropped: verdict.dropped, under: staleUnder, error } + : { kind: "stale", under: staleUnder, error }; + // A drop that stands accounts for the group, so the end no longer waits on its stream + // past the grace: a browser that lost the stream's reset would otherwise hold it forever. + judge(yielding, entry.update); } const paused = entry.paused; if (!paused) return; @@ -3925,6 +4009,9 @@ async function groupUnwanted(group: netGroup.Producer): Promise { } } +// What a read of a Group Stream resolves to once a later stream for its group superseded it. +const SUPERSEDED: unique symbol = Symbol("superseded"); + // Like untilClosed, but also cancels once every reader has left. Demand is level-triggered, so a // caller that coalesces onto the group before the check re-arms it. async function untilAbandoned(group: netGroup.Producer, step: Promise): Promise { diff --git a/js/net/src/lite/tail.test.ts b/js/net/src/lite/tail.test.ts index 5384bfc5895..eb67e1f148f 100644 --- a/js/net/src/lite/tail.test.ts +++ b/js/net/src/lite/tail.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { Duration } from "@moq/clock"; import { Manual } from "@moq/clock/test"; -import { ProtocolViolation, StreamCode, StreamError } from "../error.ts"; +import { ProtocolViolation, StreamCode, StreamError, toTransport } from "../error.ts"; import type { Consumer as GroupConsumer } from "../group.ts"; import { randomHop } from "../hop.ts"; import { hooks } from "../internal.ts"; @@ -51,7 +51,8 @@ function groupStream(subscriber: Subscriber, sequence: bigint) { return { write: (payload: string) => controller.enqueue(frame(payload, subscriber.version)), finish: () => controller.close(), - reset: () => controller.error(new Error("reset")), + // With a code, as a publisher resets: lite-07 pauses a group whose reset lost its code. + reset: () => controller.error(toTransport(StreamCode.Cancel, "reset")), handled, }; } diff --git a/js/net/src/stream.ts b/js/net/src/stream.ts index b7d3925b9b7..6f96afb9e6f 100644 --- a/js/net/src/stream.ts +++ b/js/net/src/stream.ts @@ -264,6 +264,7 @@ export class Reader { #closed?: Promise; // The decode that last ran short and how far, so a retry can wait for those bytes. #short?: { decode: (c: Cursor) => unknown; err: Short }; + #uncoded = false; version: StreamVersion; // Either stream or buffer MUST be provided. @@ -289,7 +290,9 @@ export class Reader { // Every read of this stream funnels through here, so decoding the peer's reset code // once is enough to keep the raw transport error out of every caller (and every app). const result = await reader.read().catch((err: unknown) => { - throw fromTransport(err, { version: asIetf(this.version) }); + const e = fromTransport(err, { version: asIetf(this.version) }); + if (!(e instanceof StreamError)) this.#uncoded = true; + throw e; }); if (result.done) { @@ -502,6 +505,14 @@ export class Reader { return !(await this.#fill()); } + /** + * Whether a read failed underneath without a stream code: Firefox reports a reset that lands + * before the page read the stream's data as a bare `TypeError`, dropping the peer's code. + */ + get uncoded(): boolean { + return this.#uncoded; + } + // The transport error is built only while the stream is still open. After a FIN the // reader is gone, so this allocates nothing and sends nothing. stop(reason: unknown) { diff --git a/js/net/src/tail.test.ts b/js/net/src/tail.test.ts index 5a6b7b9a016..343bf402ee8 100644 --- a/js/net/src/tail.test.ts +++ b/js/net/src/tail.test.ts @@ -185,7 +185,27 @@ describe("Tail", () => { await clock.advance(Duration.millis(50)); expect(settled).toBe(false); - read(); + read.done(); await settling; }); + + test("an excused stream is waited for at most the grace", async () => { + const { clock, tail: t } = tail(Duration.millis(10)); + const stream = t.open(0n); + stream.excuse(); + let settled = false; + const settling = t + .settle(() => true, new Once()) + .then(() => { + settled = true; + }); + + await clock.advance(Duration.millis(5)); + expect(settled).toBe(false); + await clock.advance(Duration.millis(10)); + await settling; + // Read to its end after all, it is let go once. + stream.done(); + stream.done(); + }); }); diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index 528ba918548..519f3d0ac0b 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -1,7 +1,7 @@ import { type Duration, now, sleep } from "@moq/clock"; import { type GetPromise, type Getter, Signal } from "@moq/signals"; import { READER_RUN_OVERHEAD, type Reader } from "./cache.ts"; -import { insertRun, type Run, runGaps } from "./internal.ts"; +import { insertRun, type Run, runGaps, runsFrom } from "./internal.ts"; import { GRACE, Ledger } from "./ledger.ts"; import { max, min } from "./util/sequence.ts"; @@ -13,6 +13,19 @@ export interface TailOptions { bill?: Reader; } +/** A group stream a {@link Tail} counts, from its header until it is read to its end. */ +export interface Counted { + /** Mark it read to its end. Idempotent. */ + done(): void; + /** + * The publisher accounted for its group another way, such as a drop, so {@link Tail.settle} + * waits for it at most the grace. Idempotent. + */ + excuse(): void; + /** Hold the end again: the drop that excused it no longer stands, as an update reopens a stale one. */ + restore(): void; +} + /** * The group streams a subscription has received, so its end can wait for the ones still owed. * @@ -29,9 +42,12 @@ export interface TailOptions { export class Tail { // The sequences accounted for, whose gaps fold once the grace stops waiting for them. #ledger = new Ledger(); - // Group streams whose header arrived, and those still being read. + // Group streams whose header arrived, and those still being read and holding the end. #streams = 0; #active = 0; + // Streams still being read that the end waits for at most the grace, until a settle gives up + // on them, as the ledger does on a gap. + #excused = new Set(); #changed = new Signal(0); #grace: () => Duration; // Groups whose stream finished since the last update was sent. A lite-07 stale drop under that @@ -42,7 +58,11 @@ export class Tail { // stale drop there for a group the ledger already accounted may be one its stream overtook, so // the next update owes none of it again. #folded?: Run; - // What `#finished` bills, `READER_RUN_OVERHEAD` per run, so a publisher fragmenting the + // Sequences a final drop named, so a stream whose header the drop overtook can be told from one + // whose gap the grace gave up on. Under the pool's debt they fold into one span, which excuses + // more streams for at most the grace rather than forget a drop and hold the end forever. + #dropped: Run[] = []; + // What `#finished` and `#dropped` bill, `READER_RUN_OVERHEAD` per run, so a publisher fragmenting the // streams it finishes between updates costs what the cache would hold. #bill?: Reader; @@ -51,12 +71,27 @@ export class Tail { this.#bill = options.bill; } - // Bill what `#finished` keeps, and under the pool's debt fold it. Mirrors the Rust `Tail::rebill`. + // Bill what `#finished` and `#dropped` keep, and under the pool's debt fold them. Mirrors the Rust + // `Tail::rebill`, which keeps no `#dropped`. #rebill(): void { const bill = this.#bill; if (!bill) return; - bill.set((this.#finished.length + (this.#folded ? 1 : 0)) * READER_RUN_OVERHEAD); - if (bill.debt > 0 && this.fold()) bill.set(2 * READER_RUN_OVERHEAD); + bill.set(this.#kept() * READER_RUN_OVERHEAD); + if (bill.debt <= 0) return; + const folded = this.fold(); + if (this.#foldDropped() || folded) bill.set(this.#kept() * READER_RUN_OVERHEAD); + } + + // Fold the dropped runs into the one span they reach, returning whether there was more than one. + #foldDropped(): boolean { + const dropped = this.#dropped; + if (dropped.length < 2) return false; + this.#dropped = [{ start: dropped[0].start, end: dropped[dropped.length - 1].end }]; + return true; + } + + #kept(): number { + return this.#finished.length + (this.#folded ? 1 : 0) + this.#dropped.length; } /** @@ -79,6 +114,7 @@ export class Tail { close(): void { this.#finished = []; this.#folded = undefined; + this.#dropped = []; // A stream still in flight finishes after this: with no bill, it can never bill again. this.#bill?.close(); this.#bill = undefined; @@ -89,21 +125,36 @@ export class Tail { return this.#streams; } - /** - * Record a group stream whose header arrived. Returns the call that marks it read to - * its end, which is idempotent. - */ - open(sequence: bigint): () => void { + /** Record a group stream whose header arrived, held until it is read to its end. */ + open(sequence: bigint): Counted { this.#streams += 1; this.#active += 1; this.#account(sequence, sequence + 1n); - let closed = false; - return () => { - if (closed) return; - closed = true; - this.#active -= 1; - this.#bump(); + let state: "held" | "excused" | "done" = "held"; + const token = {}; + return { + done: () => { + if (state === "done") return; + if (state === "held") this.#active -= 1; + else this.#excused.delete(token); + state = "done"; + this.#bump(); + }, + excuse: () => { + if (state !== "held") return; + state = "excused"; + this.#active -= 1; + this.#excused.add(token); + this.#bump(); + }, + restore: () => { + if (state !== "excused") return; + state = "held"; + this.#excused.delete(token); + this.#active += 1; + this.#bump(); + }, }; } @@ -184,6 +235,19 @@ export class Tail { this.#account(start, end); } + /** Record `[start, end)` as given up by a final drop, accounting for it. */ + drop(start: bigint, end: bigint): void { + insertRun(this.#dropped, start, end); + this.#rebill(); + this.#account(start, end); + } + + /** Whether a final drop named `sequence`, or a fold under the pool's debt spans it. */ + dropped(sequence: bigint): boolean { + const run = this.#dropped[runsFrom(this.#dropped, sequence)]; + return run !== undefined && run.start <= sequence; + } + /** Owe `[start, end)` again: a stale drop that accounted for it was reversed by a widening. */ unaccount(start: bigint, end: bigint): void { this.#ledger.unaccount(start, end, now()); @@ -199,9 +263,9 @@ export class Tail { return runGaps(this.#ledger.gaps(start, end), start, end); } - /** Whether a group stream is still being read. */ + /** Whether a group stream is still being read, an excused one included until a settle gives up on it. */ get reading(): boolean { - return this.#active > 0; + return this.#active > 0 || this.#excused.size > 0; } /** Bumped on every change {@link settle} watches. */ @@ -216,11 +280,12 @@ export class Tail { /** * Wait until every stream is read to its end and `complete()` holds, or the grace passes - * with nothing left being read, or `closed` settles. + * with nothing left being read but excused streams, or `closed` settles. * - * A stream still being read is always waited for: a group ends on its own stream's FIN - * or reset, never because its track ended. The grace only gives up on streams that never - * arrived. + * A stream still being read is waited for: a group ends on its own stream's FIN or reset, + * never because its track ended. The grace gives up on streams that never arrived, and on + * excused ones: a browser can lose a stream's reset, leaving it open forever, and the + * publisher's drop already accounted for its group. */ async settle(complete: () => boolean, closed: GetPromise): Promise { // A gap that aged past the grace since the last insert is no longer waited for either. @@ -237,7 +302,13 @@ export class Tail { ); try { while (closed.peek() === undefined) { - if (this.#active === 0 && (expired || complete())) return; + if (this.#active === 0 && expired) { + // Given up on, so a later settle, such as the one the publisher's FIN starts, does + // not wait another grace for them. + this.#excused.clear(); + return; + } + if (this.#active === 0 && this.#excused.size === 0 && complete()) return; await Signal.race(this.#changed, closed); } } finally { diff --git a/quest/m1/stream-credit/README.md b/quest/m1/stream-credit/README.md deleted file mode 100644 index 2645b26c654..00000000000 --- a/quest/m1/stream-credit/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Stream credit by priority - -## Goal - -When a peer's stream credit runs out, moq gives the streams it can open to -its highest-priority work and makes room for more, instead of refusing or -silently stalling. Every stream a session opens goes through one entry point -that schedules by priority, and a higher-priority group resets a -lower-priority stream (lite-07's YIELDED reset, which pauses that group) when -no credit is left. A JS reader holds hundreds of -live tracks through a browser's ~100-stream WebTransport cap, the most -important ones flowing first. - -## Plan - -Decided 2026-11-04 (maintainer, planning from Kixel's T3 host, which hit -~100-stream caps: `@moq/qmux`'s default, which `@moq/net` and `@moq/p2p` -now raise to 10,000, and browsers' own, -which JS cannot raise; Chrome stalled at 103, Firefox at 100): - -- ✅ Rust and JS, mirrored: the Rust relay is the side that refuses a - subscription with `NoStreamCredit` (sent as `Internal`, code 0) when its - publisher runs out of credit. Rejected: JS only. -- ✅ m1, before the cut, because it changes when `NoStreamCredit` is raised - and what callers see. Rejected: m2. -- ✅ Make room by taking over, else cancelling. Rejected: cancelling only - (each reset frees credit only after the peer's MAX_STREAMS, a round trip - later), and ordering waiting opens without making room. -- ~~Transports report the peer's stream credit and blocked state~~, - reversed the same day (maintainer, on - [moq-dev/moq#7894](https://github.com/moq-dev/moq/pull/7894)): the signal - is the open line's waiting state (the head of the line is pending), with - no transport credit or blocked API. -- Decided 2026-11-05 (maintainer, /quest-plan): "make room" splits into open - streams (ready now), reset fallback, and takeover (after - [moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670)), and the - browser test stays this README's own work. Not chosen: holding the whole - quest until #7670 merges. - -- Decided 2026-11-07 (maintainer, /quest-plan from the API shape audit): - ✅ delete Rust's `StreamCredit` hook too, with no replacement hook: a - transport's open waits only for credit, writing a WebTransport header as - the stream's first data. #7894's - reversal removed the JS half; the Rust trait kept it because the open line - reads it. Not chosen: keeping the hook and recording why. - -Decided 2026-11-13 (maintainer, /quest-run): ✅ takeover moved to -[m3](/quest/m3/group-takeover.md) (rejected: keep it here before lite-07 -finalizes; delete it), so making room is reset only. -[moq-dev/moq#8545](https://github.com/moq-dev/moq/pull/8545)'s YIELDED -reset already gives credit to the highest-priority groups; takeover would -save only the ~1 RTT the reset waits for MAX_STREAMS, at the cost of a new -wire message and multi-group streams. The 2026-11-04 choice of takeover -before cancelling is superseded. ✅ m2 no longer requires this epic -(rejected: keep the gate); it was an ordering rule, not a code dependency. - -The end-to-end test is `just test stream-credit` (`test/stream-credit/`), from -[moq-dev/moq#8652](https://github.com/moq-dev/moq/pull/8652): a Chromium or -Firefox tab reads 300 live tracks of mixed priority through `moq-relay` over -WebTransport, on lite-07 and lite-06. The highest-priority tier receives every -group, the tiers read in priority order, and on lite-07 the relay continues -paused groups. It runs locally only until -[browser-resets](/quest/m1/stream-credit/browser-resets.md) makes it pass -reliably, since about one lite-07 cell in 20 still fails. - -Decided 2026-11-13 (implementer, #8652, unattended): a lite-06 reader may lose a -starved track's final group at the track's end, and that is lite-06's limit, -not a defect: lite-06 has no drops, so a reader cannot wait for a group the -relay may have shed. The harness checks the final group on lite-07 only. -Rejected: holding the relay's lite-06 SUBSCRIBE_FINISH until no group waits for -credit, a heuristic for a version lite-07 replaces. diff --git a/quest/m1/stream-credit/browser-resets.md b/quest/m1/stream-credit/browser-resets.md deleted file mode 100644 index 91692ba4798..00000000000 --- a/quest/m1/stream-credit/browser-resets.md +++ /dev/null @@ -1,46 +0,0 @@ -# [L] A browser group stream reset reaches the reader - -## Goal - -`just test stream-credit` passes reliably on lite-07 in Chromium and Firefox, -and runs in the nightly interop workflow. Each group stream the relay resets -reaches the tab's reader as a reset with its code, so a YIELDED group pauses -and a subscription ends once its groups are accounted for. - -## Plan - -Decided 2026-11-13 (maintainer, the pick round for [moq-dev/moq#8652](https://github.com/moq-dev/moq/pull/8652)): ✅ keep this follow-up (rejected: drop it and leave the harness a manual recipe). - -[moq-dev/moq#8652](https://github.com/moq-dev/moq/pull/8652)'s harness found -two rare failures, both where a relay's reset of a tab's group stream meets the -browser. Heavy logging in the page hides them, so reproduce with the harness as -is and add tracing that buffers in memory. - -- Chromium (about 1 lite-07 cell in 20): one subscription never ends. The page - read the header of a group the relay paused (YIELDED), continued, and then - dropped as stale (`SUBSCRIBE_DROP`, OLD), and that stream never ended or - errored in the page, so the lite-07 tail waits on it forever. No second header - for the group reached the page. Chromium 156 does not negotiate - RESET_STREAM_AT, so the relay sends a plain RESET_STREAM once the 3-byte - WebTransport header is acknowledged (`moq_quic` `take_held_reset`). Next: log - each group stream's ID in the relay and whether Chromium acknowledges its - RESET_STREAM. If Chromium acknowledges it, the loss is Chromium's: report it - upstream, and make the JS reader stop waiting on a stream whose group a drop - or a continuation accounted for. If not, fix the relay's reset. -- Firefox (1 lite-07 cell in about 25): a few starved tracks' final groups fail - with `TypeError: Error in input stream` and no stream error code, which the - reader cannot tell from a YIELDED reset, so the group aborts instead of - pausing. Firefox 157 negotiates RESET_STREAM_AT, and the relay resets with a - Reliable Size of 3. Next: reproduce in a minimal WebTransport page, and report - it upstream if Firefox drops the code. -- Then add `just test stream-credit` to `.github/workflows/interop.yml` after - "Stream limits over WebSocket", and delete the epic README, whose last work - this is. - -To reproduce, loop `just test stream-credit --version moq-lite-07-wip --engine -chromium` until one fails; it keeps the run directory with the relay log, the -page console, and `pending.json`. The page's `console.debug` logs each pause. - -## Related - -- [moq-dev/moq#8643](https://github.com/moq-dev/moq/pull/8643) - a JS copy-fill duplicate group; the harness saw no duplicate group diff --git a/quest/m3/group-takeover.md b/quest/m3/group-takeover.md index 50f4424ae04..7f7bb7e12ad 100644 --- a/quest/m3/group-takeover.md +++ b/quest/m3/group-takeover.md @@ -76,5 +76,5 @@ version. ## Related -- [Stream credit by priority](/quest/m1/stream-credit/README.md) - the reset-based priority scheduling this would shave a round trip off +- [YIELDED reset](https://github.com/moq-dev/moq/pull/8545) - the reset-based priority scheduling this would shave a round trip off - [Finalize lite-07](/quest/m2/lite07-finalize.md) - freezes lite-07 without takeover diff --git a/quest/m4/browser-reset-reports.md b/quest/m4/browser-reset-reports.md new file mode 100644 index 00000000000..9fa5ced9783 --- /dev/null +++ b/quest/m4/browser-reset-reports.md @@ -0,0 +1,46 @@ +# [S] Browsers deliver a reset that lands right after a stream's data + +## Goal + +Chromium and Firefox each have an upstream bug report, with a minimal +WebTransport repro, for a RESET_STREAM that reaches an incoming unidirectional +stream before the page took its data: Chromium loses the reset, leaving the +stream open forever, and Firefox errors it with a bare `TypeError` that drops +its code. `@moq/net` already copes on lite-07 (see Plan); the reports let a +browser fix make those paths dead code. + +## Plan + +Found by [moq-dev/moq#8673](https://github.com/moq-dev/moq/pull/8673) with +`just test stream-credit`, whose YIELDED resets of just-opened streams hit +both bugs within a few milliseconds of the stream's data. + +- Chromium 156: the network service accepts the stream and replies to Blink's + `AcceptUnidirectionalStream` on the `WebTransport` mojo pipe, while + `OnReceivedResetStream` travels on the `WebTransportClient` pipe, so the reset + can reach `blink::WebTransport::OnReceivedResetStream` before the stream is in + `incoming_stream_map_`, where it is dropped. The FIN path keeps such a close in + `closed_potentially_pending_streams_`; the reset path has no counterpart. The + network service then closes the data pipe, so the page's stream neither ends + nor errors. The relay logged that Chromium acknowledged every such + RESET_STREAM. +- Firefox 157: in one run, 339 of the ~1100 YIELDED resets sent within 5 ms of + the stream's data reached the page as `TypeError: Error in input stream` with + no `streamErrorCode`, and none of the 30 sent 50 ms or later did, with + RESET_STREAM and RESET_STREAM_AT alike. Where Gecko or neqo drops the code is + not yet known. +- A minimal repro is a WebTransport server that opens a unidirectional stream, + writes a few bytes, and resets it with a code about a millisecond later, and a + page that reads it: Chromium's read never settles, Firefox's rejects without + the code. `moq-quic` can serve it. +- Once a browser fix ships, the matching workaround in `js/net/src/lite/subscriber.ts` + stays: the draft allows it, and older browsers keep the bug. + +Decided 2026-11-13 (maintainer, `/quest-complete`): ✅ keep this follow-up +(rejected: drop it and live with the workaround alone). + +## Waiting + +- Maintainer: approve filing at crbug.com and bugzilla.mozilla.org, outside the + moq orgs, which needs your go-ahead; check each tracker's policy on + LLM-written reports first. Recommendation: file both, with the repro. diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 27e36737d82..a963fb15517 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -36,8 +36,5 @@ ignore moq-video::frame::tests::d3d11_resize_matches_cpu /quest/m5/video-hardwar ignore moq-video::render::export::dx12::tests::exports_an_nv12_texture /quest/m5/video-hardware.md ignore moq-video::render::export::dx12::tests::media_foundation_encodes_the_export_in_place /quest/m5/video-hardware.md -# Flaked under load; disabled until fixed at its cause. -recipe test::stream-credit /quest/m1/stream-credit/browser-resets.md - # Too slow for the merge gate; the quest runs it nightly once every scenario settles. ignore moq-net::route::tests::cache::sweep_cache_routing_long /quest/m2/cache-routing-proof.md diff --git a/test/stream-credit/README.md b/test/stream-credit/README.md index c56f8d500ac..4dbe2fd8d5f 100644 --- a/test/stream-credit/README.md +++ b/test/stream-credit/README.md @@ -33,7 +33,9 @@ The driver plays every other part: limit is far above 300, so the browser's cap is the only one in play. Each track keeps one group open at a time, as a live video track keeps its GOP open: a frame at its start, one halfway through, and its end as the next - group starts, 40 groups at 200 ms. + group starts, 40 groups at 200 ms. The high tier starts its groups half an + interval after the others, while their streams hold every credit the browser + gave, so the relay must pause a lower group (YIELDED) to make room. - **The reader** is a page built from `src/` with Vite, in Playwright's Chromium or Firefox. It dials the relay over WebTransport only and subscribes to every track in three tiers of subscriber priority: @@ -59,3 +61,11 @@ A cell passes when: With every tier at one priority, the high tier misses groups past its budget and the relay continues no paused group, so the run fails. + +Both browsers mishandle a reset that lands just after its stream's data, +before the page read it, which a YIELDED reset of a freshly opened stream +does: Chromium loses the reset, leaving the stream open forever, and Firefox +reports it as a bare `TypeError` without its code. The `@moq/net` reader +copes, since a lite-07 publisher follows every reset with a continuation or a +drop, so a failure here that names a stream left open or an uncoded +`TypeError` is worth checking against those first. diff --git a/test/stream-credit/driver.ts b/test/stream-credit/driver.ts index 2b52d031069..4ef7263682f 100644 --- a/test/stream-credit/driver.ts +++ b/test/stream-credit/driver.ts @@ -153,21 +153,41 @@ try { const reading = page.evaluate(() => window.report()); // Left pending if the drain times out, and rejected once the browser closes. reading.catch(() => {}); - // Every track keeps one group open: the previous one finishes as the next one starts. - let open: Moq.Group.Producer[] = []; - const write = (sequence: number, frame: number) => { - for (const [i, group] of open.entries()) - group.writeFrame({ payload: new TextEncoder().encode(payload(plan.tracks[i], sequence, frame)) }); + // Every track keeps one group open: the previous one finishes as the next one starts. The top + // tier starts its groups half an interval after the rest, while their streams hold every + // credit, so the relay must pause a lower group to make room rather than hand out credit in + // priority order as all the groups start at once. + const open = new Map(); + const step = (offset: boolean, half: number) => { + for (const [i, track] of tracks.entries()) { + if (tierOf.get(plan.tracks[i].name)?.every !== offset) continue; + const current = open.get(i); + // The first half of the interval starts a group, the second writes its second frame. + if (half === 0) { + current?.group.finish(); + const sequence = current ? current.sequence + 1 : 0; + if (sequence === GROUPS) { + open.delete(i); + continue; + } + const group = track.appendGroup(); + open.set(i, { group, sequence }); + group.writeFrame({ payload: new TextEncoder().encode(payload(plan.tracks[i], sequence, 0)) }); + } else if (current) { + current.group.writeFrame({ + payload: new TextEncoder().encode(payload(plan.tracks[i], current.sequence, 1)), + }); + } + } }; - for (let sequence = 0; sequence < GROUPS; sequence++) { - for (const group of open) group.finish(); - open = tracks.map((track) => track.appendGroup()); - write(sequence, 0); + for (let sequence = 0; sequence <= GROUPS; sequence++) { + step(false, 0); + if (sequence > 0) step(true, 1); await sleep(INTERVAL_MS / 2); - write(sequence, 1); + if (sequence < GROUPS) step(false, 1); + step(true, 0); await sleep(INTERVAL_MS / 2); } - for (const group of open) group.finish(); for (const track of tracks) track.finish(); log(`published ${GROUPS} groups on each track`);