From 86f0ef17ffe7f0d8b0f26ba4901dfc4a3ada45f0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 8 Oct 2026 16:29:59 -0700 Subject: [PATCH 1/3] quest: claim m1/flate-stream-budget Co-Authored-By: Claude Opus 5.5 From 884ee76bfb68961855a295a1ce45c825ef135c98 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 8 Oct 2026 17:07:12 -0700 Subject: [PATCH 2/3] fix(flate): refuse a stream payload past the group budget without ending the log A flate stream rides one group, like a json stream, so it now checks the payload (plus DEFLATE's worst case when compressing) against moq-net's group budget before touching the window and refuses with GroupTooLarge, leaving the log intact. The budget is below the decoder's 64 MiB cap, so the separate terminal decoder-size check is gone. The DEFLATE bound moves into flate as Encoder::bound and json reuses it. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/flate.md | 8 ++ doc/lib/rs/moq-flate.md | 7 ++ js/flate/src/codec.test.ts | 24 ++++++ js/flate/src/codec.ts | 13 +++ js/flate/src/index.ts | 4 +- js/flate/src/stream/index.ts | 14 ++-- js/flate/src/stream/producer.ts | 40 +++++---- js/flate/src/stream/stream.test.ts | 78 +++++++++++++++-- js/json/src/stream/codec.test.ts | 26 +----- js/json/src/stream/encoder.ts | 10 +-- quest/m1/README.md | 1 - quest/m1/flate-stream-budget.md | 29 ------- rs/moq-flate/src/lib.rs | 37 ++++++++- rs/moq-flate/src/stream/mod.rs | 124 ++++++++++++++++++++-------- rs/moq-flate/src/stream/producer.rs | 45 +++++++--- rs/moq-json/src/stream/encoder.rs | 49 ++--------- rs/moq-mux/src/binary.rs | 37 ++++++++- 17 files changed, 356 insertions(+), 190 deletions(-) delete mode 100644 quest/m1/flate-stream-budget.md diff --git a/doc/lib/js/flate.md b/doc/lib/js/flate.md index 7ddf9e5b80a..865e2eebcc3 100644 --- a/doc/lib/js/flate.md +++ b/doc/lib/js/flate.md @@ -28,6 +28,14 @@ A payload is stamped when written, unless you pass its capture time: `producer.update(payload, at)`. Every frame carries a timestamp, so the track must declare a timescale, such as `createTrack(name, { timescale: Time.Timescale.MILLI })`. +A Stream rides one group, so the whole log shares `@moq/net`'s group budget: +32 MiB of payload and 8192 payloads. An `append` that might not fit throws +`GroupTooLarge` before it is encoded and leaves the log intact, compressed or +not. With `"deflate"` the check counts the raw payload plus DEFLATE's +worst-case overhead. Once the budget is spent every `append` throws; start a +new track to keep going. Any other failed append aborts the track, so readers +see the error rather than a log that looks complete. + The codec underneath is exported as `Encoder`/`Decoder`. Create one pair per group and feed frames in order. diff --git a/doc/lib/rs/moq-flate.md b/doc/lib/rs/moq-flate.md index 5996d62539c..524a6bb725d 100644 --- a/doc/lib/rs/moq-flate.md +++ b/doc/lib/rs/moq-flate.md @@ -30,6 +30,13 @@ A payload is stamped when written, unless it carries its capture time: `moq_net::Timed::from(bytes).at(captured)`. Writes return the encoded frame size. +A stream rides one group, so the whole log shares moq-net's group budget: +32 MiB of payload and 8192 payloads. An `append` that might not fit returns +`moq_net::Error::GroupTooLarge` before it is encoded and leaves the log intact, +compressed or not. Once the budget is spent every `append` is refused; start a +new track to keep going. Any other failed append aborts the track, so readers +see the error rather than a log that looks complete. + The group-scoped codec underneath is exported as `Encoder`/`Decoder`, which [`moq-json`](/lib/rs/moq-json) reuses for its merge-patch deltas. Create one pair per group and feed frames in order. diff --git a/js/flate/src/codec.test.ts b/js/flate/src/codec.test.ts index 61458493977..add575702b4 100644 --- a/js/flate/src/codec.test.ts +++ b/js/flate/src/codec.test.ts @@ -99,6 +99,30 @@ test("a frame larger than pako's chunk size round-trips", () => { expect(new Decoder().frame(slice)).toEqual(payload); }); +// The bound holds for incompressible input, the worst case, at sizes straddling the 16 KiB block +// boundary, and on a window already primed by earlier frames. Twin of the Rust test, since pako is a +// separate implementation from the zlib the Rust side checks. +test("the bound covers incompressible frames", () => { + // xorshift: incompressible enough to force stored blocks, and deterministic. Each frame is fresh + // noise; a repeat would match the window instead. + let state = 0x9e3779b9; + const noise = (len: number) => { + const out = new Uint8Array(len); + for (let i = 0; i < len; i++) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + out[i] = state & 0xff; + } + return out; + }; + + const encoder = new Encoder(); + for (const len of [1, 2, 100, 16_383, 16_384, 16_385, 65_535, 65_536, 1 << 20, 3 << 20]) { + expect(encoder.frame(noise(len)).byteLength).toBeLessThanOrEqual(Encoder.bound(len)); + } +}); + test("cross-frame context shrinks a repeated frame", () => { // A later frame identical to an earlier one compresses far smaller once the window holds it. const encoder = new Encoder(); diff --git a/js/flate/src/codec.ts b/js/flate/src/codec.ts index 15c10ab6ee8..e0a4fb8a3d6 100644 --- a/js/flate/src/codec.ts +++ b/js/flate/src/codec.ts @@ -54,6 +54,19 @@ export class Encoder { }; } + /** + * The most bytes {@link frame} can return for a payload of `len` bytes, at any level and whatever + * the window already holds. + * + * zlib's `deflateBound` for its default window and memory level, which both pako and the Rust + * `moq-flate` use: incompressible input falls back to stored blocks, 5 bytes per 16 KiB. The + * constant covers the block headers and the flush, whose fixed 4-byte marker is stripped anyway. + */ + static bound(len: number): number { + // Division rather than shifts, which wrap past 2^31. + return len + Math.floor(len / 4096) + Math.floor(len / 16384) + Math.floor(len / 33554432) + 13; + } + /** * Compress the next frame's `payload`, returning its slice of the stream: the DEFLATE bytes minus * the fixed sync-flush marker. Empty in yields empty out. Slices must be produced in frame order. diff --git a/js/flate/src/index.ts b/js/flate/src/index.ts index aee9cb3298b..057ebe02ff6 100644 --- a/js/flate/src/index.ts +++ b/js/flate/src/index.ts @@ -4,8 +4,8 @@ * - {@link Snapshot}: **lossy**. One value updated over time; a consumer only gets the most recent * one. Older values are superseded and dropped. * - {@link Stream}: **lossless**. An ordered append-log of self-contained payloads, delivered in - * order with nothing superseded. Bounded by the group cache: see {@link Stream} for what that - * costs a consumer that falls behind. + * order with nothing superseded. Bounded by the group budget: see {@link Stream} for what happens + * once it is spent. * * Pick {@link Snapshot} when consumers care about "what is the value now" (a poster image, a * serialized state blob) and {@link Stream} when they care about every payload (an event log, a diff --git a/js/flate/src/stream/index.ts b/js/flate/src/stream/index.ts index 508439578c8..aa04c2d590a 100644 --- a/js/flate/src/stream/index.ts +++ b/js/flate/src/stream/index.ts @@ -5,18 +5,20 @@ * sequence of samples). Nothing is ever superseded: a consumer yields each payload in the order it * was appended. For a latest-value document, use the `Snapshot` module instead. * - * Retention is bounded, which is the limit of "lossless" here. The group's cache is finite, so a - * log longer than it holds evicts its earliest frames, and a consumer that falls behind or - * subscribes late then fails its read with `Lagged` rather than silently resuming partway through: - * a partial log presented as a whole one is what this mode exists to prevent. Keep a log inside - * what the group retains, and split anything unbounded across successive tracks. - * * On the wire the log rides a **single group** that is never rolled, one payload per frame. A * payload that cannot be written ends the track rather than opening a second group: a log missing a * record is not lossless, and a gap dressed up as a complete log is worse than a visible failure. * With {@link Config.compression} `"deflate"`, that one group is one DEFLATE window, so each payload * compresses against the earlier ones and a run of similar payloads shrinks sharply. * + * That one group bounds the log at `@moq/net`'s group budget: 32 MiB of payload and 8192 frames. A + * consumer always starts at frame 0, so the cap is never met by dropping a prefix some readers + * missed. Instead an append that might not fit throws `GroupTooLarge` before it is encoded, and the + * log stays intact and writable. With compression the check counts the payload's raw size plus + * DEFLATE's worst-case overhead (`Encoder.bound`), since the compressed size is only known once the + * window has moved. The budget covers the whole log, so once it is spent every append throws and a + * publisher with more to say opens a new track. + * * @module */ diff --git a/js/flate/src/stream/producer.ts b/js/flate/src/stream/producer.ts index c63a14bd327..908ea60dcbe 100644 --- a/js/flate/src/stream/producer.ts +++ b/js/flate/src/stream/producer.ts @@ -1,6 +1,6 @@ import type * as Moq from "@moq/net"; -import { Time } from "@moq/net"; -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "../codec.ts"; +import { Group, Error as NetError, Time } from "@moq/net"; +import { Encoder as Flate } from "../codec.ts"; import { type Compression, isDeflate } from "../compression.ts"; @@ -25,6 +25,10 @@ export class Producer { #flate?: Flate; // The single group carrying the whole log, opened on the first append and never rolled. #group?: Moq.Group.Producer; + // Frames and payload bytes written to the group, checked against the group budget before each + // payload is encoded. + #frames = 0; + #bytes = 0; /** Wrap a track to publish a payload log into it. */ constructor(config: Producer.Config) { @@ -36,25 +40,26 @@ export class Producer { /** * Append one payload to the log. * - * A payload that cannot be written ends the track: a log missing a record is not the lossless - * log this mode promises, so the failure is surfaced rather than papered over with a second - * group. The group is aborted rather than closed cleanly, so a consumer sees the failure + * A payload that might not fit in what is left of the group's budget throws `GroupTooLarge` + * before anything is written, leaving the log intact. The budget covers the whole log, so once + * it is spent every append throws; a publisher with more to say opens a new track. + * + * Any other payload that cannot be written ends the track: a log missing a record is not the + * lossless log this mode promises, so the failure is surfaced rather than papered over with a + * second group. The group is aborted rather than closed cleanly, so a consumer sees the failure * instead of a log that merely looks complete. Every later append fails on the closed track. * * `at` is when the payload was captured, written as its frame timestamp. Defaults to now. */ append(payload: Uint8Array, at: Time.Timestamp = Time.Timestamp.now()): void { - // A payload no consumer could decode is as terminal as one the track rejects: consumers all - // decode with `@moq/flate`'s default cap, so this would publish a record none of them could - // read. Ends the track like any other lost record, and aborts the group the same way the - // write path below does: an earlier append may already have opened one, and leaving the two - // paths to tear down differently only invites a reader to wonder which is authoritative. - if (this.#flate && payload.byteLength > DEFAULT_MAX_FRAME_SIZE) { - const err = new Error(`payload larger than the decoder's ${DEFAULT_MAX_FRAME_SIZE} byte limit`); - this.#group?.close(err); - this.#group = undefined; - this.#track.close(err); - throw err; + // Check before compressing: encoding advances the window, so a payload refused afterwards + // would leave the encoder ahead of every reader. The worst case is checked rather than the + // actual size for the same reason. Also checked before the group is opened, so a refused + // first payload publishes nothing. The budget is below the decoder's cap, so any payload that + // fits is one every consumer can inflate. + const bound = this.#flate ? Flate.bound(payload.byteLength) : payload.byteLength; + if (this.#frames >= Group.MAX_GROUP_FRAMES || this.#bytes + bound > Group.MAX_GROUP_CACHE_BYTES) { + throw new NetError.GroupTooLarge(); } // Open the group before compressing: a failure here must not leave the window ahead of a @@ -82,6 +87,9 @@ export class Producer { this.#track.close(abort); throw err; } + + this.#frames++; + this.#bytes += encoded.byteLength; } /** Finish the track, closing the group. */ diff --git a/js/flate/src/stream/stream.test.ts b/js/flate/src/stream/stream.test.ts index 5cf435db8cc..1ca1a1be954 100644 --- a/js/flate/src/stream/stream.test.ts +++ b/js/flate/src/stream/stream.test.ts @@ -1,5 +1,5 @@ import { expect, spyOn, test } from "bun:test"; -import { Time, Track } from "@moq/net"; +import { Group, Error as NetError, Time, Track } from "@moq/net"; import { DEFAULT_MAX_FRAME_SIZE } from "../codec.ts"; import { Consumer, Producer, Rolled } from "./index.ts"; @@ -129,10 +129,9 @@ test("a second group is reported while the first is still open", async () => { await expect(consumer.next()).rejects.toThrow(Rolled); }); -test("an undecodable payload ends the log for a reader already inside the group", async () => { - // The decoder-limit check ends the track like any other lost record, and by then an earlier - // append may already have opened the group. A reader that pulled that group has to see the - // failure rather than park on a group nothing will ever finish. +test("a failed write ends the log for a reader already inside the group", async () => { + // A reader that pulled the group holds its own handle, so it has to see the failure rather than + // park on a group nothing will ever finish. const track = new Track.Producer("test"); const producer = new Producer({ track, compression: "deflate" }); producer.append(payloads(1)[0]); @@ -140,13 +139,76 @@ test("an undecodable payload ends the log for a reader already inside the group" const consumer = new Consumer({ track: track.subscribe(), compression: "deflate" }); expect(await consumer.next()).toBeDefined(); - const oversized = new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1); - expect(() => producer.append(oversized)).toThrow("limit"); + // The budget check refuses anything the group would, so stub the write to stand in for any + // rejection past it. + const failure = new Error("write rejected"); + const write = spyOn(Group.Producer.prototype, "writeFrame").mockImplementation(() => { + throw failure; + }); + try { + expect(() => producer.append(payloads(2)[1])).toThrow(failure); + } finally { + write.mockRestore(); + } // Surfaces the terminal error rather than hanging on the still-open group. - await expect(consumer.next()).rejects.toThrow("limit"); + await expect(consumer.next()).rejects.toThrow(failure); + expect(() => producer.append(payloads(3)[2])).toThrow(); +}); + +// A payload past the group budget is refused before anything is written, so the log carries on: the +// next payload lands and decodes, which with compression proves the window never moved. One past the +// decoder's cap is refused the same way rather than ending the track. +for (const compression of [false, true]) { + test(`an oversized payload is refused and the log continues (compression=${compression})`, async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: compression ? "deflate" : "none" }); + producer.append(payloads(1)[0]); + + const huge = new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1); + expect(() => producer.append(huge.subarray(0, Group.MAX_GROUP_CACHE_BYTES))).toThrow(NetError.GroupTooLarge); + expect(() => producer.append(huge)).toThrow(NetError.GroupTooLarge); + + producer.append(payloads(2)[1]); + producer.finish(); + expect(await drain(track.subscribe(), compression)).toEqual(payloads(2)); + }); +} + +// The budget check stands in for the decoder's cap: a payload that fits is one every consumer can +// inflate. +test("the group budget is within the decoder's cap", () => { + expect(Group.MAX_GROUP_CACHE_BYTES).toBeLessThanOrEqual(DEFAULT_MAX_FRAME_SIZE); }); +test("a refused first payload publishes nothing", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: "deflate" }); + expect(() => producer.append(new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1))).toThrow(NetError.GroupTooLarge); + producer.finish(); + + expect(await track.subscribe().ordered().nextGroup()).toBeUndefined(); +}); + +// The budget covers the whole log, so once its frames are spent every append is refused, and the log +// written so far still finishes cleanly and reads back whole. +for (const compression of [false, true]) { + test(`a spent budget refuses every append (compression=${compression})`, async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: compression ? "deflate" : "none" }); + const frame = (n: number) => new Uint8Array(new Uint32Array([n]).buffer); + for (let n = 0; n < Group.MAX_GROUP_FRAMES; n++) producer.append(frame(n)); + + expect(() => producer.append(frame(-1))).toThrow(NetError.GroupTooLarge); + expect(() => producer.append(frame(-2))).toThrow(NetError.GroupTooLarge); + producer.finish(); + + const out = await drain(track.subscribe(), compression); + expect(out.length).toBe(Group.MAX_GROUP_FRAMES); + expect(out.at(-1)).toEqual(frame(Group.MAX_GROUP_FRAMES - 1)); + }); +} + // Counts the reactions `run` attaches to promises still pending once it returns. A promise holds each // reaction until it settles, so one left per iteration on a promise that outlives the loop is a leak. // Recorded by hand: Bun's `mock.contexts` misses the engine's own calls from `Promise.race`. diff --git a/js/json/src/stream/codec.test.ts b/js/json/src/stream/codec.test.ts index b3304271bc5..07c4a2b3fc7 100644 --- a/js/json/src/stream/codec.test.ts +++ b/js/json/src/stream/codec.test.ts @@ -1,10 +1,9 @@ import { expect, spyOn, test } from "bun:test"; -import { Encoder as Flate } from "@moq/flate"; import { Group, Track } from "@moq/net"; import * as z from "@zod/mini"; import { Desync } from "../error.ts"; import { Decoder } from "./decoder.ts"; -import { deflateBound, Encoder } from "./encoder.ts"; +import { Encoder } from "./encoder.ts"; import { Producer } from "./producer.ts"; type Rec = { n: number }; @@ -209,26 +208,3 @@ test("a lost compressed stream frame has a typed error", () => { encoder.encode({ n: 1 }); expect(() => encoder.encode({ n: 2 })).toThrow(Desync); }); - -// The bound holds for incompressible input, the worst case, at sizes straddling the 16 KiB block -// boundary, and on a window already primed by earlier frames. Twin of the Rust test, since pako is a -// separate implementation from the zlib the Rust side checks. -test("deflate bound covers incompressible frames", () => { - // xorshift: incompressible enough to force stored blocks, and deterministic. - let state = 0x9e3779b9; - const noise = (len: number) => { - const out = new Uint8Array(len); - for (let i = 0; i < len; i++) { - state ^= state << 13; - state ^= state >>> 17; - state ^= state << 5; - out[i] = state & 0xff; - } - return out; - }; - - const flate = new Flate(); - for (const len of [1, 2, 100, 16_383, 16_384, 16_385, 65_535, 65_536, 1 << 20, 3 << 20]) { - expect(flate.frame(noise(len)).byteLength).toBeLessThanOrEqual(deflateBound(len)); - } -}); diff --git a/js/json/src/stream/encoder.ts b/js/json/src/stream/encoder.ts index a921f09e464..58bcd352323 100644 --- a/js/json/src/stream/encoder.ts +++ b/js/json/src/stream/encoder.ts @@ -124,7 +124,7 @@ export class Encoder { // actual size for the same reason. The budget is also below `@moq/flate`'s per-frame decode // cap, so any record that fits is one every consumer can inflate. const budget = this.#budget; - const bound = this.#compress ? deflateBound(bytes.byteLength) : bytes.byteLength; + const bound = this.#compress ? Flate.bound(bytes.byteLength) : bytes.byteLength; if (budget.frames >= Group.MAX_GROUP_FRAMES || budget.bytes + bound > Group.MAX_GROUP_CACHE_BYTES) { throw new NetError.GroupTooLarge(); } @@ -152,11 +152,3 @@ export class Encoder { }; } } - -// The largest a sync-flushed DEFLATE frame of `len` raw bytes can grow to: zlib's `deflateBound` for -// its default window and memory level, which both pako and moq-flate use. Incompressible input falls -// back to stored blocks, 5 bytes per 16 KiB; the constant covers the block headers and the flush, -// whose fixed 4-byte marker is stripped anyway. Division rather than shifts, which wrap past 2^31. -export function deflateBound(len: number): number { - return len + Math.floor(len / 4096) + Math.floor(len / 16384) + Math.floor(len / 33554432) + 13; -} diff --git a/quest/m1/README.md b/quest/m1/README.md index d258ffe349f..b4080b444c9 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -60,7 +60,6 @@ were done or not worth their cost. - [Cross-relay FETCH on any peer link](/quest/m1/ietf-peer-fetch-old.md) - a lite or moq-transport peer link serves held groups instead of refusing them as old, and the drill covers both - [Untimed lite-07](/quest/m1/lite-untimed.md) - lite-07 carries an untimed track in both languages; lite-05/06 write send time - [Late lower groups](/quest/m1/lite-late-lower-group.md) - a moq-lite subscriber with a floor receives a group created below the first served one, as moq-transport does -- [Flate stream budget](/quest/m1/flate-stream-budget.md) - a flate stream refuses an oversized append without ending, sharing one DEFLATE bound with json - [A watch and publish release ships assets()](/quest/m1/assets-release.md) - the release that lets the sites host the worklets - [Dogfood hosted worklets](/quest/m1/dogfood-assets.md) - moq.dev and the moq.pro dashboard host the worklets and call `assets()` after the release - [Media audio-tone check](/quest/m1/media-audio-tone.md) - the media lane's audio-tone check passes under load, fixed at its cause diff --git a/quest/m1/flate-stream-budget.md b/quest/m1/flate-stream-budget.md deleted file mode 100644 index 84b23dec60b..00000000000 --- a/quest/m1/flate-stream-budget.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] A flate stream refuses an oversized append without ending - -## Goal - -`moq-flate` and `@moq/flate` `stream` mode refuse an append that cannot fit -the group budget with `GroupTooLarge` and leave the log intact, compressed or -not, as JSON streams do after #4911. In deflate mode, a payload beyond the -decoder's frame size limit is also refused before encoding without ending the -log. - -## Plan - -Flate's stream mode rides one group, like JSON's, so it has the same hole. -Move the DEFLATE worst-case bound (`deflateBound` / `deflate_bound`) from json -into flate and have json reuse it, so the bound and its tests live in one -place per language. Rust's helper is private; JS's is exported from its -encoder module, so expose only what json needs from flate. - -The decoder-size check only runs in deflate mode (`self.flate.is_some()` in -`rs/moq-flate/src/stream/producer.rs`, `this.#flate &&` in -`js/flate/src/stream/producer.ts`). Refusing it without ending the track -deliberately reverses both producers' current behavior and their comments, -which treat it as terminal like any other lost record; update those comments. - -Check both limits before mutating the compression window. A refusal leaves -prior records readable and allows a later fitting append; a failed write -after encoding still aborts, as JSON does. Mirror #4911's tests in both -compression modes, including the decoder-size refusal, and document the -budget and refusal behavior in the flate library docs. diff --git a/rs/moq-flate/src/lib.rs b/rs/moq-flate/src/lib.rs index 717ae878c80..479cd8e02b2 100644 --- a/rs/moq-flate/src/lib.rs +++ b/rs/moq-flate/src/lib.rs @@ -5,8 +5,8 @@ //! - [`snapshot`]: **lossy**. One value updated over time; a consumer only gets the most recent //! one. Older values are superseded and dropped. //! - [`stream`]: **lossless**. An ordered append-log of self-contained payloads, delivered in order -//! with nothing superseded. Bounded by the group cache: see [`stream`] for what that costs a -//! consumer that falls behind. +//! with nothing superseded. Bounded by the group budget: see [`stream`] for what happens once +//! it is spent. //! //! Pick [`snapshot`] when consumers care about "what is the value now" (a poster image, a //! serialized state blob) and [`stream`] when they care about every payload (an event log, a @@ -140,6 +140,16 @@ impl Encoder { Self(Box::new(Compress::new(flate2::Compression::new(level.min(9)), false))) } + /// The most bytes [`frame`](Self::frame) can return for a payload of `len` bytes, at any level + /// and whatever the window already holds. + /// + /// zlib's `deflateBound` for its default window and memory level, which both this crate and the + /// browser's pako use: incompressible input falls back to stored blocks, 5 bytes per 16 KiB. The + /// constant covers the block headers and the flush, whose fixed 4-byte marker is stripped anyway. + pub const fn bound(len: u64) -> u64 { + len + (len >> 12) + (len >> 14) + (len >> 25) + 13 + } + /// Compress the next frame's `payload`, returning its slice of the stream: the DEFLATE bytes minus /// the fixed sync-flush marker. Empty in yields empty out. Later frames reuse earlier ones as /// context, so slices must be produced (and later decoded) in frame order. @@ -381,6 +391,29 @@ mod test { } } + /// The bound holds for incompressible input, the worst case, at sizes straddling the 16 KiB + /// block boundary, and on a window already primed by earlier frames. + #[test] + fn bound_covers_incompressible_frames() { + let lens = [1, 2, 100, 16_383, 16_384, 16_385, 65_535, 65_536, 1 << 20, 3 << 20]; + // Each frame is fresh noise; a repeat would match the window instead. + let noise = noise(lens.iter().sum()); + let mut rest = noise.as_slice(); + + let mut enc = Encoder::new(); + for len in lens { + let (frame, next) = rest.split_at(len); + rest = next; + let slice = enc.frame(frame); + let bound = Encoder::bound(len as u64); + assert!( + slice.len() as u64 <= bound, + "{len} raw bytes deflated to {}, past the bound {bound}", + slice.len() + ); + } + } + #[test] fn decompress_rejects_garbage() { let mut dec = Decoder::new(); diff --git a/rs/moq-flate/src/stream/mod.rs b/rs/moq-flate/src/stream/mod.rs index 03e5e86d4c5..9ddb1b2b323 100644 --- a/rs/moq-flate/src/stream/mod.rs +++ b/rs/moq-flate/src/stream/mod.rs @@ -4,18 +4,29 @@ //! sequence of samples). Nothing is ever superseded: a consumer yields each payload in the order it //! was appended. For a latest-value document, use [`snapshot`](crate::snapshot) instead. //! -//! Retention is bounded, which is the limit of "lossless" here. The group's cache is finite, so a -//! write that would outgrow it aborts the group with [`moq_net::Error::GroupTooLarge`] rather than -//! dropping a prefix some readers missed: a partial log presented as a whole one is what this -//! mode exists to prevent. Keep a log inside what the group retains, and split anything unbounded -//! across successive tracks. -//! //! On the wire the log is a single group that is never rolled, one payload per frame. A payload //! that cannot be written ends the track rather than opening a second group: a log missing a record //! is not lossless, and a gap dressed up as a complete log is worse than a visible failure. With //! [`Config::compression`] set to [`Compression::Deflate`], that one //! group is one sync-flushed DEFLATE stream, so each //! payload compresses against the earlier ones and a run of similar payloads shrinks sharply. +//! +//! # Budget +//! +//! That single group is what bounds the log. moq-net caps a group at +//! [`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES) (32 MiB) of payload and +//! [`MAX_GROUP_FRAMES`](moq_net::group::MAX_GROUP_FRAMES) (8192) frames, and a consumer always +//! starts at frame 0, so the cap is never met by dropping a prefix some readers missed: a partial +//! log presented as a whole one is what this mode exists to prevent. +//! +//! Instead an append that might not fit is refused with [`moq_net::Error::GroupTooLarge`] before +//! it is encoded, and the log stays intact and writable. With compression the check counts the +//! payload's raw size plus DEFLATE's worst-case overhead ([`Encoder::bound`](crate::Encoder::bound)), +//! since the compressed size is only known once the window has moved. The budget is below the +//! decoder's [`DEFAULT_MAX_FRAME_SIZE`](crate::DEFAULT_MAX_FRAME_SIZE), so any payload that fits is +//! one every consumer can inflate. The budget covers the whole log rather than a single payload, so +//! once it is spent every append is refused: the live stream is bounded history by design, and a +//! publisher with more to say opens a new track. use crate::Compression; @@ -218,28 +229,68 @@ mod test { ); } - /// A record the consumer could never decode is as lost as one the track rejects, so it takes the - /// track with it rather than leaving a live log missing a record. Guards the guard: an early - /// return here would bypass the terminal path and let a later append continue the gap. + fn refused(result: crate::Result) -> bool { + matches!(result, Err(crate::Error::Net(moq_net::Error::GroupTooLarge))) + } + + /// A payload past the group budget is refused before anything is written, so the log carries + /// on: the next payload lands and decodes, which with compression proves the window never + /// moved. One past the decoder's cap is refused the same way rather than ending the track. #[test] - fn an_undecodable_record_ends_the_track() { - let track = moq_net::broadcast::Info::new() - .produce() - .create_track("test", None) - .unwrap(); - let mut subscriber = track.subscribe(None); - let mut producer = Producer::new(track, cfg(true)); + fn an_oversized_payload_is_refused_and_the_log_continues() { + let huge = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); + let past_budget = huge.slice(..moq_net::group::MAX_CACHE_BYTES as usize); + + for compression in [false, true] { + let (mut producer, track) = producer(compression); + producer.append(&b"first"[..]).unwrap(); + + assert!( + refused(producer.append(past_budget.clone())), + "compression={compression}" + ); + assert!(refused(producer.append(huge.clone())), "compression={compression}"); + assert!(producer.demand().is_used(), "a refusal must not end the track"); + + producer.append(&b"second"[..]).unwrap(); + producer.finish().unwrap(); + + assert_eq!( + drain(track, compression), + vec![Bytes::from_static(b"first"), Bytes::from_static(b"second")], + "compression={compression}" + ); + } + } - assert!(producer.demand().is_used()); - let oversized = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); - assert!(matches!(producer.append(oversized), Err(crate::Error::TooLarge(_)))); + /// A refused first payload opens no group, so a subscriber is not handed an empty one to wait on. + #[test] + fn a_refused_first_payload_publishes_nothing() { + let (mut producer, track) = producer(true); + let huge = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); + assert!(refused(producer.append(huge))); + assert_eq!(track.latest(), None); + } - assert!(!producer.demand().is_used()); + /// The budget covers the whole log, so once its frames are spent every append is refused, and + /// the log written so far still finishes cleanly and reads back whole. + #[test] + fn a_spent_budget_refuses_every_append() { + let frames = moq_net::group::MAX_GROUP_FRAMES as u32; + for compression in [false, true] { + let (mut producer, track) = producer(compression); + for n in 0..frames { + producer.append(Bytes::copy_from_slice(&n.to_be_bytes())).unwrap(); + } - // Nothing was published, and the track is terminal rather than merely skipping the record. - let waiter = kio::Waiter::noop(); - assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Err(_)))); - assert!(producer.append(&b"after"[..]).is_err()); + assert!(refused(producer.append(&b"late"[..])), "compression={compression}"); + assert!(refused(producer.append(&b"later"[..])), "compression={compression}"); + producer.finish().unwrap(); + + let payloads = drain(track, compression); + assert_eq!(payloads.len(), frames as usize, "compression={compression}"); + assert_eq!(payloads.last().unwrap()[..], (frames - 1).to_be_bytes()); + } } /// A reader already inside the group keeps its own handle, which `track::Producer::abort` @@ -247,14 +298,15 @@ mod test { /// told the producer went away rather than why the log stopped. #[test] fn a_reader_inside_the_group_sees_the_real_error() { - let track = moq_net::broadcast::Info::new() - .produce() - .create_track("test", None) - .unwrap(); + let track = rejecting_track(); let mut subscriber = track.subscribe(None); let mut producer = Producer::new(track, Config::default()); - producer.append(&b"first"[..]).unwrap(); + // The epoch still fits the extreme timescale, so this write lands and opens the group. + let epoch = moq_net::Timestamp::from_millis(0).unwrap(); + producer + .append(moq_net::Timed::from(Bytes::from_static(b"first")).at(epoch)) + .unwrap(); // Pull the group, the way a live reader would, so we hold a handle of our own. let waiter = kio::Waiter::noop(); @@ -263,14 +315,16 @@ mod test { }; assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); - // A payload the group cannot hold, so the write fails and ends the log. - let oversized = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); - assert!(producer.append(oversized).is_err()); + // Stamped now, which overflows the timescale, so the write fails and ends the log. + let Err(crate::Error::Net(written)) = producer.append(&b"rejected"[..]) else { + panic!("the write should fail"); + }; match group.poll_read_frame(&waiter) { - Poll::Ready(Err(err)) => assert!( - matches!(err, moq_net::Error::FrameTooLarge), - "the reader should see the write's own error, got {err:?}" + Poll::Ready(Err(err)) => assert_eq!( + err.to_string(), + written.to_string(), + "the reader should see the write's own error" ), other => panic!("expected the write error, got {other:?}"), } diff --git a/rs/moq-flate/src/stream/producer.rs b/rs/moq-flate/src/stream/producer.rs index bbf56702899..5ca72948e6a 100644 --- a/rs/moq-flate/src/stream/producer.rs +++ b/rs/moq-flate/src/stream/producer.rs @@ -27,6 +27,8 @@ impl Producer { track, group: None, flate: config.compression.is_deflate().then(crate::Encoder::new), + frames: 0, + bytes: 0, })), } } @@ -48,9 +50,14 @@ impl Producer { /// Append one payload to the log. /// - /// A payload that cannot be written ends the track: a log missing a record is not the lossless - /// log this mode promises, so the failure is surfaced rather than papered over with a second - /// group. The group is aborted rather than closed cleanly, so a consumer sees the failure + /// A payload that might not fit in what is left of the group's budget is refused with + /// [`moq_net::Error::GroupTooLarge`] before anything is written, leaving the log intact. The + /// budget covers the whole log, so once it is spent every append is refused; a publisher with + /// more to say opens a new track. + /// + /// Any other payload that cannot be written ends the track: a log missing a record is not the + /// lossless log this mode promises, so the failure is surfaced rather than papered over with a + /// second group. The group is aborted rather than closed cleanly, so a consumer sees the failure /// instead of a log that merely looks complete. Every later append fails on the closed track. /// /// Returns the frame's encoded size. @@ -73,20 +80,36 @@ struct Inner { /// The DEFLATE encoder, one window for the whole group, `Some` while compressing. flate: Option, + + /// Frames and payload bytes written to the group, checked against moq-net's group budget before + /// each payload is encoded. + frames: usize, + bytes: u64, } +// The budget check also stands in for the decoder's cap: a payload that fits the group is one every +// consumer can inflate. +const _: () = assert!(moq_net::group::MAX_CACHE_BYTES <= crate::DEFAULT_MAX_FRAME_SIZE); + impl Inner { fn append(&mut self, payload: Timed) -> Result { let timestamp = payload.at.unwrap_or_else(moq_net::Timestamp::now); let payload = payload.value; - // A payload no consumer could decode is as terminal as one the track rejects: the log is - // missing a record either way, and carrying on would present that gap as a complete log. - // Checked before the group is opened, so nothing is published, and routed through the same - // abort so a reader sees the failure rather than a clean end. - if self.flate.is_some() && payload.len() as u64 > crate::DEFAULT_MAX_FRAME_SIZE { - self.abort(moq_net::Error::FrameTooLarge); - return Err(crate::Error::TooLarge(crate::DEFAULT_MAX_FRAME_SIZE)); + // Check before compressing: encoding advances the window, so a payload refused afterwards + // would leave the encoder ahead of every reader. The worst case is checked rather than the + // actual size for the same reason. Also checked before the group is opened, so a refused + // first payload publishes nothing. + let size = payload.len() as u64; + let bound = if self.flate.is_some() { + crate::Encoder::bound(size) + } else { + size + }; + if self.frames >= moq_net::group::MAX_GROUP_FRAMES + || self.bytes.saturating_add(bound) > moq_net::group::MAX_CACHE_BYTES + { + return Err(moq_net::Error::GroupTooLarge.into()); } // Open the group before compressing: a failure here must not leave the window ahead of a @@ -103,6 +126,8 @@ impl Inner { let size = payload.len(); let group = self.group.as_mut().expect("a group is open"); let Err(err) = group.write_frame(timestamp, payload) else { + self.frames += 1; + self.bytes += size as u64; return Ok(size); }; diff --git a/rs/moq-json/src/stream/encoder.rs b/rs/moq-json/src/stream/encoder.rs index faab346d3d8..d7ecea8d69e 100644 --- a/rs/moq-json/src/stream/encoder.rs +++ b/rs/moq-json/src/stream/encoder.rs @@ -146,7 +146,11 @@ impl Encoder { // actual size for the same reason. The budget is also below moq-flate's per-frame decode cap, // so any record that fits is one every consumer can inflate. let size = bytes.len() as u64; - let bound = if self.compression { deflate_bound(size) } else { size }; + let bound = if self.compression { + moq_flate::Encoder::bound(size) + } else { + size + }; if self.frames >= moq_net::group::MAX_GROUP_FRAMES || self.bytes.saturating_add(bound) > moq_net::group::MAX_CACHE_BYTES { @@ -165,46 +169,3 @@ impl Encoder { }) } } - -/// The largest a sync-flushed DEFLATE frame of `len` raw bytes can grow to. -/// -/// zlib's `deflateBound` for its default window and memory level, which both moq-flate and the -/// browser's pako use: incompressible input falls back to stored blocks, 5 bytes per 16 KiB. The -/// constant covers the block headers and the flush, whose fixed 4-byte marker is stripped anyway. -fn deflate_bound(len: u64) -> u64 { - len + (len >> 12) + (len >> 14) + (len >> 25) + 13 -} - -#[cfg(test)] -mod test { - use super::*; - - /// The bound holds for incompressible input, the worst case, at sizes straddling the 16 KiB - /// block boundary, and on a window already primed by earlier frames. - #[test] - fn deflate_bound_covers_incompressible_frames() { - // xorshift: incompressible enough to force stored blocks, and deterministic. - let mut state = 0x9e37_79b9_7f4a_7c15u64; - let mut noise = |len: usize| { - (0..len) - .map(|_| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - state as u8 - }) - .collect::>() - }; - - let mut flate = moq_flate::Encoder::new(); - for len in [1, 2, 100, 16_383, 16_384, 16_385, 65_535, 65_536, 1 << 20, 3 << 20] { - let payload = flate.frame(&noise(len)); - assert!( - payload.len() as u64 <= deflate_bound(len as u64), - "{len} raw bytes deflated to {}, past the bound {}", - payload.len(), - deflate_bound(len as u64) - ); - } - } -} diff --git a/rs/moq-mux/src/binary.rs b/rs/moq-mux/src/binary.rs index 49ba04b3d34..07518ddf041 100644 --- a/rs/moq-mux/src/binary.rs +++ b/rs/moq-mux/src/binary.rs @@ -250,9 +250,10 @@ impl Stream { /// Append one payload to the log. /// /// A payload that cannot be written ends the track (see - /// [`moq_flate::stream::Producer::append`]) and retires the catalog entry with it. A catalog - /// error publishing the measured bitrate is returned after the payload was written, so the track - /// stays open and a retry would duplicate it. + /// [`moq_flate::stream::Producer::append`]) and retires the catalog entry with it. A payload + /// refused with [`moq_net::Error::GroupTooLarge`] leaves both intact, since nothing was written. + /// A catalog error publishing the measured bitrate is returned after the payload was written, so + /// the track stays open and a retry would duplicate it. pub fn append(&mut self, payload: impl Into>) -> crate::Result<()> { let (payload, captured) = match &mut self.listing { Some(listing) => listing.stamp(payload.into()), @@ -261,6 +262,8 @@ impl Stream { }; let size = match self.inner.append(payload) { Ok(size) => size, + // Refused before anything was written, so the log and its entry carry on. + Err(err @ moq_flate::Error::Net(moq_net::Error::GroupTooLarge)) => return Err(err.into()), Err(err) => { // The inner producer has already closed the track. Dropping the listing retires the // catalog entry too: waiting for the handle to drop would keep advertising a track @@ -658,6 +661,34 @@ mod test { assert_eq!(drain(consumer), expected); } + /// A payload past the group budget is refused before anything is written, so the log stays + /// advertised and the next payload still lands. + #[test] + fn an_oversized_payload_keeps_the_track_and_the_entry() { + let (mut broadcast, catalog) = catalog(); + let mut samples = catalog + .binary_stream(track(&mut broadcast, "samples"), Config::default()) + .unwrap(); + let track = samples.consume(); + + let oversized = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); + assert!(matches!( + samples.append(oversized), + Err(crate::Error::Flate(moq_flate::Error::Net( + moq_net::Error::GroupTooLarge + ))) + )); + assert!(catalog.snapshot().binary.tracks.contains_key("samples")); + + samples.append(&b"after"[..]).unwrap(); + let entry = entry(&catalog, "samples"); + samples.finish().unwrap(); + assert_eq!( + drain(Consumer::from_track(track, &entry).unwrap()), + vec![Bytes::from_static(b"after")] + ); + } + #[test] fn a_snapshot_track_roundtrips() { let (mut broadcast, catalog) = catalog(); From aa28803957d6b7be6a4630b6e2f624b74e83c33b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 Oct 2026 11:35:43 -0700 Subject: [PATCH 3/3] fix(flate): report an ended stream log before the group budget An over-budget append on an aborted track returned GroupTooLarge, which promises the log is still writable. Check the track's closed state first in both producers. Co-Authored-By: Claude Opus 5.5 --- js/flate/src/stream/producer.ts | 5 +++++ js/flate/src/stream/stream.test.ts | 19 +++++++++++++++++++ rs/moq-flate/src/stream/mod.rs | 18 ++++++++++++++++++ rs/moq-flate/src/stream/producer.rs | 7 +++++++ 4 files changed, 49 insertions(+) diff --git a/js/flate/src/stream/producer.ts b/js/flate/src/stream/producer.ts index 86ef3c1b9d8..fabc92e2a2b 100644 --- a/js/flate/src/stream/producer.ts +++ b/js/flate/src/stream/producer.ts @@ -53,6 +53,11 @@ export class Producer { * untimed track: it is never filled in. */ append({ value: payload, at }: Timed): void { + // A closed track refuses every append, so say so before the budget: `GroupTooLarge` promises + // the log is still writable. + const closed = this.#track.closed.peek(); + if (closed !== undefined) throw closed ?? new Error("track is closed"); + // Check before compressing: encoding advances the window, so a payload refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the // actual size for the same reason. Also checked before the group is opened, so a refused diff --git a/js/flate/src/stream/stream.test.ts b/js/flate/src/stream/stream.test.ts index 4019152bd35..6232dbcaed7 100644 --- a/js/flate/src/stream/stream.test.ts +++ b/js/flate/src/stream/stream.test.ts @@ -156,6 +156,25 @@ test("a failed write ends the log for a reader already inside the group", async expect(() => producer.append({ value: payloads(3)[2] })).toThrow(); }); +// An ended log reports why it ended, even for a payload past the budget: `GroupTooLarge` would tell +// the caller the log is still writable. +test("an ended log refuses with its own error", () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: "deflate" }); + + const failure = new Error("write rejected"); + const write = spyOn(Group.Producer.prototype, "writeFrame").mockImplementation(() => { + throw failure; + }); + try { + expect(() => producer.append({ value: payloads(1)[0] })).toThrow(failure); + } finally { + write.mockRestore(); + } + + expect(() => producer.append({ value: new Uint8Array(Group.MAX_GROUP_CACHE_BYTES + 1) })).toThrow(failure); +}); + // A payload past the group budget is refused before anything is written, so the log carries on: the // next payload lands and decodes, which with compression proves the window never moved. One past the // decoder's cap is refused the same way rather than ending the track. diff --git a/rs/moq-flate/src/stream/mod.rs b/rs/moq-flate/src/stream/mod.rs index 9ddb1b2b323..4656e2bb976 100644 --- a/rs/moq-flate/src/stream/mod.rs +++ b/rs/moq-flate/src/stream/mod.rs @@ -350,6 +350,24 @@ mod test { ); } + /// An ended log reports why it ended, even for a payload past the budget: `GroupTooLarge` would + /// tell the caller the log is still writable. + #[test] + fn an_ended_log_refuses_with_its_own_error() { + let track = rejecting_track(); + let mut producer = Producer::new(track, Config::default()); + + let Err(written) = producer.append(&b"rejected"[..]) else { + panic!("the write should fail"); + }; + + let huge = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); + let Err(err) = producer.append(huge) else { + panic!("an append on an ended log should fail"); + }; + assert_eq!(err.to_string(), written.to_string()); + } + /// A stream is one group. A publisher that rolls to a second one lost whatever would have /// completed the first, so the read reports that rather than handing back the remainder as a /// continuous log. Written by hand because this producer never rolls. diff --git a/rs/moq-flate/src/stream/producer.rs b/rs/moq-flate/src/stream/producer.rs index 5ca72948e6a..487675028e8 100644 --- a/rs/moq-flate/src/stream/producer.rs +++ b/rs/moq-flate/src/stream/producer.rs @@ -1,6 +1,7 @@ //! Publishing an ordered log of opaque payloads over a track. use std::sync::{Arc, Mutex}; +use std::task::Poll; use bytes::Bytes; use moq_net::Timed; @@ -96,6 +97,12 @@ impl Inner { let timestamp = payload.at.unwrap_or_else(moq_net::Timestamp::now); let payload = payload.value; + // A closed track refuses every append, so say so before the budget: `GroupTooLarge` promises + // the log is still writable. + if let Poll::Ready(err) = self.track.poll_closed(&kio::Waiter::noop()) { + return Err(err.into()); + } + // Check before compressing: encoding advances the window, so a payload refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the // actual size for the same reason. Also checked before the group is opened, so a refused