diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 1cea657e414..768e8bea138 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -186,8 +186,8 @@ writes only when a track's frame changed, so silence means nothing moved, not that the producer is gone. The track ends when the producer does, or at depth 1 or more when its group's broadcast is unannounced; the group may return later as a new broadcast. A frame too large to cache (32 MB) ends its -track with a `FrameTooLarge` error, while the `.z` flavor, which compresses -under the limit, keeps updating. +track with a `FrameTooLarge` error. The `.z` flavor meets the same limit by its +uncompressed size, since a reader inflates the whole group. ### Totals @@ -260,7 +260,7 @@ real sums. Started minus ended stays exact. A child appears while a broadcast under it has counters, and leaves after the frame carrying its last one; its tail stays in the rollup, and a child that returns starts over. Children are never capped, so roughly ten thousand across four tiers outgrow the 32 MB frame -limit: that encoding's track then ends with `FRAME_TOO_LARGE` instead of going +limit: both encodings' tracks then end with `FRAME_TOO_LARGE` instead of going stale. ## Entries diff --git a/js/flate/MIGRATE.md b/js/flate/MIGRATE.md index 3c9ea9653a1..8a0f7b2c80e 100644 --- a/js/flate/MIGRATE.md +++ b/js/flate/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +`Decoder` holds its whole window, across every frame, to `Group.MAX_GROUP_CACHE_BYTES` (32 MiB) instead of capping each frame at 64 MiB. +`DEFAULT_MAX_FRAME_SIZE`, `DecoderOptions`, and `Decoder.maxFrameSize` are removed; construct with `new Decoder()`. +The producers charge a compressed payload its inflated size: a snapshot value over 32 MiB throws `FrameTooLarge` however small it deflates, and a stream log ends its budget at 32 MiB of inflated payload. + ### @moq/binary is @moq/flate The opaque `Snapshot` and `Stream` tracks moved beside the codec; the wire and the catalog's `binary` section are unchanged. diff --git a/js/flate/README.md b/js/flate/README.md index 4625243096f..ac7d000875a 100644 --- a/js/flate/README.md +++ b/js/flate/README.md @@ -50,4 +50,4 @@ new TextDecoder().decode(decoder.frame(a)); // "the quick brown fox" new TextDecoder().decode(decoder.frame(b)); // "the quick brown dog" ``` -Create a fresh `Encoder`/`Decoder` pair per independent stream (in moq-net terms, per group). `new Encoder({ level })` sets the DEFLATE level (`0..=9`, default `6`); `new Decoder({ maxFrameSize })` caps how far a single frame may inflate (default 64 MiB), rejecting zip bombs. +Create a fresh `Encoder`/`Decoder` pair per independent stream (in moq-net terms, per group). `new Encoder({ level })` sets the DEFLATE level (`0..=9`, default `6`); a `Decoder` holds its whole window to the group budget (32 MiB), rejecting zip bombs: a compressed group never costs a reader more than a plain one. diff --git a/js/flate/src/codec.test.ts b/js/flate/src/codec.test.ts index ff4dad69d48..2930f3c5853 100644 --- a/js/flate/src/codec.test.ts +++ b/js/flate/src/codec.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { Group } from "@moq/net"; import { Deflate, Inflate } from "fflate"; import { Decoder, Encoder } from "./codec.ts"; @@ -64,19 +65,18 @@ test("codec rejects garbage", () => { expect(() => decoder.frame(new Uint8Array(64).fill(0xff))).toThrow(); }); -test("codec rejects a frame that inflates past the cap", () => { - // A tiny slice can inflate enormously, so the decoder bounds the output as it is produced. The - // cap's value doesn't change that path, so use a small one instead of paying for a real bomb. - const slice = new Encoder().frame(new Uint8Array(1024)); - const decoder = new Decoder({ maxFrameSize: 512 }); - expect(() => decoder.frame(slice)).toThrow(/exceeded 512/); -}); +test("the decoder holds its whole window to the group budget", () => { + // Each frame fits on its own and deflates to almost nothing; together they inflate past what a + // plain group can carry. + const half = new Uint8Array(Group.MAX_GROUP_CACHE_BYTES / 2 + 1); + const encoder = new Encoder(); + const first = encoder.frame(half); + const second = encoder.frame(half); + expect(second.length).toBeLessThan(64 * 1024); -test("a decoder's frame cap cannot be lifted after construction", () => { const decoder = new Decoder(); - expect(() => { - (decoder as { maxFrameSize: number }).maxFrameSize = Infinity; - }).toThrow(TypeError); + expect(decoder.frame(first).length).toBe(half.length); + expect(() => decoder.frame(second)).toThrow(/decompressed group exceeded/); }); test("a frame larger than pako's chunk size round-trips", () => { diff --git a/js/flate/src/codec.ts b/js/flate/src/codec.ts index e0a4fb8a3d6..31709e62650 100644 --- a/js/flate/src/codec.ts +++ b/js/flate/src/codec.ts @@ -1,13 +1,11 @@ // The group-scoped DEFLATE codec: see the module docs in index.ts. +import { Group } from "@moq/net"; import * as pako from "pako"; /** The default DEFLATE level ({@link Encoder}): a good size/speed balance for small, repetitive payloads. */ export const DEFAULT_LEVEL = 6; -/** The default per-frame decompressed-size cap ({@link Decoder}): 64 MiB. */ -export const DEFAULT_MAX_FRAME_SIZE = 64 * 1024 * 1024; - // The trailing bytes of a DEFLATE sync flush, stripped on the wire and re-appended to decode. const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); @@ -92,42 +90,33 @@ export class Encoder { } } -/** Options for a {@link Decoder}. */ -export interface DecoderOptions { - /** - * Per-frame decompressed-size cap. A frame that inflates past this is rejected (zip-bomb guard). - * Defaults to {@link DEFAULT_MAX_FRAME_SIZE}. - */ - maxFrameSize?: number; -} - /** * Decodes a stream's frame slices back into the original payloads. Hold one per stream; feed slices * in frame order (each frame builds on the earlier ones). * + * A peer could send tiny slices that inflate hugely, so the window's total output is held to the + * group budget, `Group.MAX_GROUP_CACHE_BYTES`: the most a plain group can carry. A compressed group + * then costs a reader no more than a plain one, however small its slices. + * * @public */ export class Decoder { #inflate = new pako.Inflate({ raw: true }); #chunks: Uint8Array[] = []; #total = 0; + // Bytes the whole window has inflated to, across every frame. + #inflated = 0; #tooLarge = false; - #maxFrameSize: number; - - /** The per-frame decompressed-size cap in bytes, in effect for this decoder. */ - get maxFrameSize(): number { - return this.#maxFrameSize; - } /** Start a fresh decoder with a cold window. */ - constructor(options: DecoderOptions = {}) { - this.#maxFrameSize = options.maxFrameSize ?? DEFAULT_MAX_FRAME_SIZE; + constructor() { this.#inflate.onData = (chunk) => { const bytes = chunk as Uint8Array; this.#total += bytes.length; + this.#inflated += bytes.length; // Bound the inflated output as it is produced; a tiny slice can expand enormously. Stop - // retaining past the cap, then reject once the push returns. - if (this.#total > this.#maxFrameSize) { + // retaining past the budget, then reject once the push returns. + if (this.#inflated > Group.MAX_GROUP_CACHE_BYTES) { this.#tooLarge = true; return; } @@ -137,21 +126,22 @@ export class Decoder { /** * Decompress the next frame's `slice` back into its payload. Empty in yields empty out. Throws if - * the input is malformed or inflates past the per-frame size cap. + * the input is malformed, or once the window's output across every frame so far passes the group + * budget. */ frame(slice: Uint8Array): Uint8Array { if (slice.length === 0) return slice; + if (this.#tooLarge) throw new Error(`decompressed group exceeded ${Group.MAX_GROUP_CACHE_BYTES} bytes`); this.#chunks = []; this.#total = 0; - this.#tooLarge = false; // Feed the slice then the re-appended sync-flush marker as two pushes, so no combined buffer is // allocated. The marker delimits the frame and flushes its last bytes out of the inflate buffer. this.#inflate.push(slice, false); this.#inflate.push(SYNC_FLUSH_TAIL, pako.Z_SYNC_FLUSH); if (this.#inflate.err) throw new Error(`decompression failed: ${this.#inflate.msg}`); - if (this.#tooLarge) throw new Error(`decompressed frame exceeded ${this.#maxFrameSize} bytes`); + if (this.#tooLarge) throw new Error(`decompressed group exceeded ${Group.MAX_GROUP_CACHE_BYTES} bytes`); return concat(this.#chunks, this.#total); } diff --git a/js/flate/src/index.ts b/js/flate/src/index.ts index bb5cc8cb1ad..518f6f77f35 100644 --- a/js/flate/src/index.ts +++ b/js/flate/src/index.ts @@ -34,20 +34,14 @@ * A sync flush always ends in the fixed 4-byte marker `00 00 ff ff`. {@link Encoder.frame} drops it * and {@link Decoder.frame} re-appends it, saving 4 bytes per frame, the same trick * [RFC 7692](https://www.rfc-editor.org/rfc/rfc7692.html#section-7.2.1) (permessage-deflate) uses. A - * small slice can still inflate enormously, so {@link Decoder.frame} caps the inflated output as it - * is produced. pako is synchronous, so the whole codec is synchronous. + * small slice can still inflate enormously, so a {@link Decoder} holds its whole window to the group + * budget as output is produced: a compressed group never costs a reader more than a plain one, and + * the producers refuse before a group would. pako is synchronous, so the whole codec is synchronous. * * @module */ -export { - DEFAULT_LEVEL, - DEFAULT_MAX_FRAME_SIZE, - Decoder, - type DecoderOptions, - Encoder, - type EncoderOptions, -} from "./codec.ts"; +export { DEFAULT_LEVEL, Decoder, Encoder, type EncoderOptions } from "./codec.ts"; export type { Compression } from "./compression.ts"; /** Lossy latest-value opaque publishing. */ export * as Snapshot from "./snapshot/index.ts"; diff --git a/js/flate/src/snapshot/producer.ts b/js/flate/src/snapshot/producer.ts index e4ff4b9fb8e..c9c8e6250c5 100644 --- a/js/flate/src/snapshot/producer.ts +++ b/js/flate/src/snapshot/producer.ts @@ -1,6 +1,6 @@ import type { Timed } from "@moq/net"; import * as Moq from "@moq/net"; -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "../codec.ts"; +import { Encoder as Flate } from "../codec.ts"; import { type Compression, isDeflate } from "../compression.ts"; @@ -43,19 +43,16 @@ export class Producer { * untimed track: it is never filled in. */ update({ value: payload, at }: Timed): void { - // Consumers all decode with `@moq/flate`'s default cap, so publishing past it would advertise - // a value that always fails to read. Rejected before anything is published; unlike a stream - // this is not terminal, since the previous value still stands and the next update supersedes. - if (this.#compress && payload.byteLength > DEFAULT_MAX_FRAME_SIZE) { - throw new Error(`payload larger than the decoder's ${DEFAULT_MAX_FRAME_SIZE} byte limit`); - } + // Check before opening a group. `appendGroup` publishes immediately, so letting `writeFrame` + // reject the frame would leave an empty newest group behind: a snapshot consumer jumps to the + // newest, so the previous value would be lost even though this update threw. A compressed value + // is held to the same budget by its inflated size, since that is what every consumer's decoder + // allows: compression saves bandwidth, never what a reader holds. + if (payload.byteLength > Moq.Group.MAX_GROUP_CACHE_BYTES) throw new Moq.Error.FrameTooLarge(); // One frame per group, so the window spans a single value and starts cold every time. const encoded = this.#compress ? new Flate().frame(payload) : payload; - // Check before opening a group. `appendGroup` publishes immediately, so letting `writeFrame` - // reject the frame would leave an empty newest group behind: a snapshot consumer jumps to the - // newest, so the previous value would be lost even though this update threw. if (encoded.byteLength > Moq.Group.MAX_GROUP_CACHE_BYTES) throw new Moq.Error.FrameTooLarge(); const group = this.#track.appendGroup(); diff --git a/js/flate/src/snapshot/snapshot.test.ts b/js/flate/src/snapshot/snapshot.test.ts index 2ea8ba6170e..842c459a511 100644 --- a/js/flate/src/snapshot/snapshot.test.ts +++ b/js/flate/src/snapshot/snapshot.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; -import { Time, Track } from "@moq/net"; +import { Group, Error as NetError, Time, Track } from "@moq/net"; import { Consumer, Producer } from "./index.ts"; const bytes = (...values: number[]) => new Uint8Array(values); @@ -85,6 +85,20 @@ test("compression shrinks the frame on the wire", async () => { expect(frame?.payload.byteLength).toBeLessThan(payload.byteLength / 4); }); +// Compression doesn't lift the group limit: a reader inflates the whole value, so one past the budget +// is refused however small it deflates, and the previous value stays readable. +test("a compressed value past the group budget is refused", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: "deflate" }); + producer.update({ value: bytes(1) }); + expect(() => producer.update({ value: new Uint8Array(Group.MAX_GROUP_CACHE_BYTES + 1) })).toThrow( + NetError.FrameTooLarge, + ); + producer.finish(); + + expect(await drain(track.subscribe(), true)).toEqual([bytes(1)]); +}); + test("a finished track ends the consumer", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); diff --git a/js/flate/src/stream/producer.ts b/js/flate/src/stream/producer.ts index fabc92e2a2b..fb023d26d98 100644 --- a/js/flate/src/stream/producer.ts +++ b/js/flate/src/stream/producer.ts @@ -25,8 +25,9 @@ export class Producer { #flate?: Flate; // The single group carrying the whole log, opened on the first append and never rolled. #group?: Moq.Group.Producer; - // Frames and payload bytes written to the group, checked against the group budget before each - // payload is encoded. + // Frames and bytes charged to the group, checked against the group budget before each payload is + // encoded. A compressed payload is charged its inflated size (or its slice, if larger), the budget + // every consumer's `Decoder` holds its window to. #frames = 0; #bytes = 0; @@ -60,9 +61,8 @@ export class Producer { // Check before compressing: encoding advances the window, so a payload refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. Also checked before the group is opened, so a refused - // first payload publishes nothing. The budget is below the decoder's cap, so any payload that - // fits is one every consumer can inflate. + // actual size for the same reason, and it covers the inflated size too. Also checked before + // the group is opened, so a refused first payload publishes nothing. const bound = this.#flate ? Flate.bound(payload.byteLength) : payload.byteLength; if (this.#frames >= Group.MAX_GROUP_FRAMES || this.#bytes + bound > Group.MAX_GROUP_CACHE_BYTES) { throw new NetError.GroupTooLarge(); @@ -95,7 +95,7 @@ export class Producer { } this.#frames++; - this.#bytes += encoded.byteLength; + this.#bytes += Math.max(payload.byteLength, encoded.byteLength); } /** Finish the track, closing the group. */ diff --git a/js/flate/src/stream/stream.test.ts b/js/flate/src/stream/stream.test.ts index 22d3d438edc..e3983a7e07e 100644 --- a/js/flate/src/stream/stream.test.ts +++ b/js/flate/src/stream/stream.test.ts @@ -1,7 +1,6 @@ import { expect, spyOn, test } from "bun:test"; import { Milli } from "@moq/clock"; import { Group, Error as NetError, Time, Track } from "@moq/net"; -import { DEFAULT_MAX_FRAME_SIZE } from "../codec.ts"; import { Consumer, Producer, Rolled } from "./index.ts"; // Ask for a replay window, so the superseded first group is delivered rather than skipped by the @@ -173,18 +172,14 @@ test("an ended log refuses with its own error", () => { }); // A payload past the group budget is refused before anything is written, so the log carries on: the -// next payload lands and decodes, which with compression proves the window never moved. One past the -// decoder's cap is refused the same way rather than ending the track. +// next payload lands and decodes, which with compression proves the window never moved. for (const compression of [false, true]) { test(`an oversized payload is refused and the log continues (compression=${compression})`, async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, compression: compression ? "deflate" : "none" }); producer.append({ value: payloads(1)[0] }); - const huge = new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1); - expect(() => producer.append({ value: huge.subarray(0, Group.MAX_GROUP_CACHE_BYTES) })).toThrow( - NetError.GroupTooLarge, - ); + const huge = new Uint8Array(Group.MAX_GROUP_CACHE_BYTES); expect(() => producer.append({ value: huge })).toThrow(NetError.GroupTooLarge); producer.append({ value: payloads(2)[1] }); @@ -193,16 +188,22 @@ for (const compression of [false, true]) { }); } -// The budget check stands in for the decoder's cap: a payload that fits is one every consumer can -// inflate. -test("the group budget is within the decoder's cap", () => { - expect(Group.MAX_GROUP_CACHE_BYTES).toBeLessThanOrEqual(DEFAULT_MAX_FRAME_SIZE); +// A compressed payload is charged its inflated size, since that is what every reader's decoder holds +// the group to: two payloads that each fit, and deflate to almost nothing, still overflow together. +test("a compressed log is charged its inflated size", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: "deflate" }); + const half = new Uint8Array(Group.MAX_GROUP_CACHE_BYTES / 2 + 1); + producer.append({ value: half }); + expect(() => producer.append({ value: half })).toThrow(NetError.GroupTooLarge); + producer.finish(); + expect(await drain(track.subscribe(), true)).toEqual([half]); }); test("a refused first payload publishes nothing", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, compression: "deflate" }); - expect(() => producer.append({ value: new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1) })).toThrow( + expect(() => producer.append({ value: new Uint8Array(Group.MAX_GROUP_CACHE_BYTES + 1) })).toThrow( NetError.GroupTooLarge, ); producer.finish(); diff --git a/js/json/MIGRATE.md b/js/json/MIGRATE.md index 0de64bf2179..e97738e3cda 100644 --- a/js/json/MIGRATE.md +++ b/js/json/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +A compressed group may inflate to at most `Group.MAX_GROUP_CACHE_BYTES` (32 MiB), the most a plain group carries. +The encoders charge each frame its plaintext: a compressed snapshot over 32 MiB throws `FrameTooLarge` (it used to pass up to 64 MiB), a window header over it throws, and deltas and ops roll a new group once a group's plaintext would pass the budget. +A compressed stream's budget is 32 MiB of record plaintext. + ### Groups roll on joiner demand `Snapshot` `deltaRatio` and `Window` `opRatio` are replaced by `deltas` and `ops` booleans (both default `true`). diff --git a/js/json/src/snapshot/encoder.ts b/js/json/src/snapshot/encoder.ts index f62688666c4..bfd41a977d3 100644 --- a/js/json/src/snapshot/encoder.ts +++ b/js/json/src/snapshot/encoder.ts @@ -1,5 +1,5 @@ -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "@moq/flate"; -import { Group } from "@moq/net"; +import { Encoder as Flate } from "@moq/flate"; +import { Group, Error as NetError } from "@moq/net"; import type * as z from "@zod/mini"; import { type Compression, isDeflate } from "../compression.ts"; @@ -112,8 +112,8 @@ export class Encoder { // baseline every delta is diffed against, and `undefined` until the first snapshot commits. #last?: unknown; - // Bytes emitted into the current group, snapshot included: compressed frame sizes when - // compressing, raw otherwise, matching what the group holds on the wire. + // Bytes charged to the current group, snapshot included: each frame's slice, or its plaintext + // when that is larger, since a reader's DEFLATE window is held to the same budget. #groupBytes = 0; // Frames emitted into the current group, snapshot included. #groupFrames = 0; @@ -210,18 +210,20 @@ export class Encoder { // A delta is only readable while the group still holds the snapshot it applies to. // Admitting a patch that pushes the group past that budget would abort it - // (`GroupTooLarge`), leaving a late subscriber with no value, so fall through to a - // fresh snapshot instead. + // (`GroupTooLarge`), or past a reader's inflate budget, leaving a late subscriber with no + // value, so fall through to a fresh snapshot instead. // - // Measured on the encoded payload rather than the plaintext: a sync-flushed DEFLATE frame - // can come out slightly larger than its input, so the plaintext is not an upper bound. - // Compressing first advances the window, but `#snapshot` opens a fresh one, so an - // over-budget delta costs only the wasted compression. - if (this.#groupBytes + payload.length <= this.#maxGroupBytes) { - this.#groupBytes += payload.length; + // Charged the larger of the encoded payload and the plaintext: a sync-flushed DEFLATE frame + // can come out slightly larger than its input, while the reader's window counts what it + // inflates. Compressing first advanced the window past what any reader holds, so drop the + // group before rolling: if the snapshot is refused too, the next update must not be a delta. + const charge = Math.max(payload.length, delta.length); + if (this.#groupBytes + charge <= this.#maxGroupBytes) { + this.#groupBytes += charge; this.#groupFrames += 1; return this.#pend({ payload, keyframe: false }, json); } + this.reset(); } const payload = this.#snapshot(new TextEncoder().encode(text)); @@ -277,19 +279,13 @@ export class Encoder { return new TextEncoder().encode(JSON.stringify(result.patch)); } - // Reject plaintext the consumer's decoder could never reproduce. Every consumer decodes with - // `@moq/flate`'s default output cap, so a frame past it would be unreadable however small it - // compresses to. Checked before anything is published or the window advances. - #checkDecodable(frame: Uint8Array): void { - if (this.#compress && frame.byteLength > DEFAULT_MAX_FRAME_SIZE) { - throw new Error(`value larger than the decoder's ${DEFAULT_MAX_FRAME_SIZE} byte limit`); - } - } - // Encode a group's snapshot (frame 0), returning its payload. On the compressed path this opens a // fresh stream (cold window), so the snapshot and its deltas share one DEFLATE window. #snapshot(frame: Uint8Array): Uint8Array { - this.#checkDecodable(frame); + // Every consumer holds a compressed group to the group budget by its inflated size, so a value + // past it would be unreadable however small it compresses to: the same `FrameTooLarge` a plain + // one meets. Checked before anything is published or the window advances. + if (this.#compress && frame.byteLength > this.#maxGroupBytes) throw new NetError.FrameTooLarge(); // Build the window locally and install it only once framing succeeds. Assigning it first would // leave a throw with a cold window in place, no resync pending, and the old group counters @@ -299,7 +295,7 @@ export class Encoder { const payload = flate ? flate.frame(frame) : frame; this.#flate = flate; - this.#groupBytes = payload.length; + this.#groupBytes = Math.max(payload.length, frame.length); this.#groupFrames = 1; this.#resync = false; @@ -308,9 +304,6 @@ export class Encoder { // Compress a frame into the current group's window, or pass it through when uncompressed. #frame(frame: Uint8Array): Uint8Array { - // A delta is worse than a snapshot here: there is no keyframe after it to resynchronize on, - // so one the consumer cannot decode makes the rest of the group unreadable. - this.#checkDecodable(frame); return this.#flate ? this.#flate.frame(frame) : frame; } } diff --git a/js/json/src/snapshot/snapshot.test.ts b/js/json/src/snapshot/snapshot.test.ts index b6357f525f6..8de63fd99a8 100644 --- a/js/json/src/snapshot/snapshot.test.ts +++ b/js/json/src/snapshot/snapshot.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; +import { Decoder as Flate } from "@moq/flate"; import { Group, Error as NetError, StreamCode, Time, Track } from "@moq/net"; import { Consumer } from "./consumer.ts"; import { Encoder } from "./encoder.ts"; @@ -538,6 +539,43 @@ test("a delta that would overflow the snapshot rolls a new one instead", async ( expect(values[values.length - 1]).toEqual({ v: "y".repeat(size) }); }); +// A compressed delta is charged its plaintext too: the reader's window counts what it inflates, so a +// patch that deflates to almost nothing still rolls the group once its plaintext would overflow the +// budget. Matches the Rust `a_compressed_delta_is_gated_on_its_inflated_size`. +test("a compressed delta is gated on its inflated size", () => { + const value = { v: "x".repeat(1000) }; + const patched = { v: "y".repeat(1000) }; + const encoder = new Encoder({ + compression: "deflate", + deltas: true, + maxGroupBytes: JSON.stringify(value).length + 500, + }); + encoder.update(value)?.commit(); + const rolled = encoder.update(patched); + expect(rolled?.keyframe).toBe(true); + expect(rolled?.payload.length).toBeLessThan(500); +}); + +// A delta that overflows the group advances the window before the roll is attempted, so a refused roll +// must not leave that window behind: the next update is a snapshot a reader can decode. +test("a refused roll resynchronizes", () => { + const encoder = new Encoder({ compression: "deflate", deltas: true, maxGroupBytes: 200 }); + encoder.update({ a: 1 })?.commit(); + expect(() => encoder.update({ a: 1, b: "x".repeat(300) })).toThrow(NetError.FrameTooLarge); + + const next = encoder.update({ a: 2 }); + expect(next?.keyframe).toBe(true); + const decoded = new TextDecoder().decode(new Flate().frame(next?.payload ?? new Uint8Array())); + expect(JSON.parse(decoded)).toEqual({ a: 2 }); +}); + +// A compressed snapshot is refused by its plaintext, the way a plain one is, however small it deflates: +// no reader could inflate it. +test("a compressed snapshot past the budget is refused", () => { + const encoder = new Encoder({ compression: "deflate", deltas: false, maxGroupBytes: 100 }); + expect(() => encoder.update({ v: "x".repeat(1000) })).toThrow(NetError.FrameTooLarge); +}); + test("a compressed delta is gated on its encoded size, not its plaintext", async () => { // A sync-flushed DEFLATE frame can come out larger than its input, so the plaintext is not an // upper bound on what lands in the group. A patch that fits the budget by its plaintext but not @@ -554,8 +592,9 @@ test("a compressed delta is gated on its encoded size, not its plaintext", async expect(delta?.keyframe).toBe(false); expect(delta?.payload.length).toBeGreaterThan(plaintext); - // A budget with room for the plaintext patch but not the encoded one. - const maxGroupBytes = (snapshot?.payload.length ?? 0) + plaintext; + // A budget with room for the plaintext patch but not the encoded one. The snapshot is charged its + // plaintext, which outweighs its slice. + const maxGroupBytes = JSON.stringify(value).length + plaintext; const track = new Track.Producer("test"); const producer = new Producer({ track, compression: "deflate", maxGroupBytes }); producer.update({ value: value }); diff --git a/js/json/src/stream/encoder.ts b/js/json/src/stream/encoder.ts index 58bcd352323..034eac3ded7 100644 --- a/js/json/src/stream/encoder.ts +++ b/js/json/src/stream/encoder.ts @@ -66,9 +66,10 @@ export class Encoder { // Bumped for each record handed out, so a commit that arrives after the encoder has moved on can // tell that it is acknowledging a record that is no longer the outstanding one. #generation = 0; - // Frames and payload bytes committed to the current group, checked against the group budget - // before each record is encoded. Replaced on reset, so a commit that lands after one charges the - // old group rather than the new. + // Frames and bytes committed to the current group, checked against the group budget before each + // record is encoded. A compressed record is charged its plaintext (or its slice, if larger), since + // a reader's DEFLATE window is held to the same budget. Replaced on reset, so a commit that lands + // after one charges the old group rather than the new. #budget = { frames: 0, bytes: 0 }; constructor(config: Config = {}) { @@ -121,8 +122,7 @@ export class Encoder { // Check before compressing: encoding advances the window, so a record refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. The budget is also below `@moq/flate`'s per-frame decode - // cap, so any record that fits is one every consumer can inflate. + // actual size for the same reason; it also covers the plaintext a reader inflates. const budget = this.#budget; const bound = this.#compress ? Flate.bound(bytes.byteLength) : bytes.byteLength; if (budget.frames >= Group.MAX_GROUP_FRAMES || budget.bytes + bound > Group.MAX_GROUP_CACHE_BYTES) { @@ -142,7 +142,7 @@ export class Encoder { if (!charged) { charged = true; budget.frames++; - budget.bytes += payload.byteLength; + budget.bytes += Math.max(payload.byteLength, bytes.byteLength); } // A caller that starts the next encode before this record settles has already made the // encoder account for it. Acknowledging it now would clear the flag belonging to the newer diff --git a/js/json/src/stream/stream.test.ts b/js/json/src/stream/stream.test.ts index cbcaf38182a..8e2d208e7a5 100644 --- a/js/json/src/stream/stream.test.ts +++ b/js/json/src/stream/stream.test.ts @@ -226,6 +226,18 @@ for (const compression of [false, true]) { }); } +// A compressed record is charged its plaintext, since that is what a reader's window inflates: two +// records that each fit, and deflate to almost nothing, still overflow the budget together. +test("a compressed log is charged its inflated size", async () => { + const track = new Track.Producer("test"); + const producer = new Producer<{ n: number; pad: string }>({ track, compression: "deflate" }); + const pad = "x".repeat(Group.MAX_GROUP_CACHE_BYTES / 2); + producer.append({ value: { n: 0, pad } }); + expect(() => producer.append({ value: { n: 1, pad } })).toThrow(NetError.GroupTooLarge); + producer.finish(); + expect(await drain(track.subscribe(), true)).toEqual([0]); +}); + // The budget covers the whole log, so once its frames are spent every append is refused, and the log // written so far still finishes cleanly and reads back whole. for (const compression of [false, true]) { diff --git a/js/json/src/window/encoder.ts b/js/json/src/window/encoder.ts index 471535e75ce..44c7f333cbf 100644 --- a/js/json/src/window/encoder.ts +++ b/js/json/src/window/encoder.ts @@ -1,4 +1,4 @@ -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "@moq/flate"; +import { Encoder as Flate } from "@moq/flate"; import { Group } from "@moq/net"; /** Frames (header included) in one group before a new group is forced, matching the Snapshot cap. Kept well below the net per-group cap so a roll always precedes GroupTooLarge. */ @@ -83,7 +83,8 @@ export class Encoder { #start = 0; #flate?: Flate; - // Bytes in this group, header included. + // Bytes charged to this group, header included: each frame's slice, or its plaintext when that + // is larger, since a reader's DEFLATE window is held to the same budget. #groupBytes = 0; #groupFrames = 0; // Whether the next frame must be a header after a lost frame. @@ -204,16 +205,17 @@ export class Encoder { return this.#ops && this.#groupFrames > 0 && this.#groupFrames < MAX_GROUP_FRAMES; } - /** Compress an already-serialized op into the open group, if its header remains cached. */ + /** + * Compress an already-serialized op into the open group, if its header remains cached and + * inflatable. Charged the larger of the slice and the plaintext: a sync-flushed DEFLATE frame can + * come out slightly larger than its input, while a reader's window counts what it inflates. + */ #emitOp(text: string): Uint8Array | undefined { const bytes = new TextEncoder().encode(text); - if (bytes.length > DEFAULT_MAX_FRAME_SIZE) { - throw new Error("window frame exceeds the decoder's decompressed size limit"); - } const payload = this.#flate ? this.#flate.frame(bytes) : bytes; - this.#groupBytes += payload.length; this.#groupFrames += 1; + this.#groupBytes += Math.max(payload.length, bytes.length); if (this.#groupBytes > Group.MAX_GROUP_CACHE_BYTES) { this.#resyncGroup(); return undefined; @@ -240,8 +242,9 @@ export class Encoder { const checkpoint = start === offset ? "" : `,"start":${start}`; const text = `{"offset":${offset}${checkpoint},"records":[${window.join(",")}]}`; const bytes = new TextEncoder().encode(text); - if (bytes.length > DEFAULT_MAX_FRAME_SIZE) { - throw new Error("window header exceeds the decoder's decompressed size limit"); + // A reader inflates the header whole, so its plaintext meets the group budget too. + if (bytes.length > Group.MAX_GROUP_CACHE_BYTES) { + throw new Error("window header exceeds the group cache limit"); } // A fresh per-group encoder (cold window), with the header as frame 0. @@ -252,7 +255,7 @@ export class Encoder { } this.#flate = flate; - this.#groupBytes = payload.length; + this.#groupBytes = Math.max(payload.length, bytes.length); this.#groupFrames = 1; this.#resync = false; diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index df40978a6da..d7ce680bb3e 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -517,15 +517,19 @@ test("an op that would evict the header rolls first", () => { frame.commit(); }); -test("a header larger than the group cache is rejected", () => { - const encoder = new Encoder(); - - expect(() => encoder.push("x".repeat(Group.MAX_GROUP_CACHE_BYTES))).toThrow("group cache limit"); - expect(encoder.window).toEqual([]); - - const frame = encoder.push("ok"); - expect(frame.keyframe).toBeTrue(); -}); +// Compression doesn't lift the limit: a reader inflates the header whole, so one past the group budget +// is refused however small it deflates. +for (const compression of [false, true]) { + test(`a header larger than the group cache is rejected (compression=${compression})`, () => { + const encoder = new Encoder({ compression }); + + expect(() => encoder.push("x".repeat(Group.MAX_GROUP_CACHE_BYTES))).toThrow("group cache limit"); + expect(encoder.window).toEqual([]); + + const frame = encoder.push("ok"); + expect(frame.keyframe).toBeTrue(); + }); +} test("an uncommitted edit leaves the window unchanged", () => { const encoder = new Encoder(); diff --git a/rs/moq-flate/MIGRATE.md b/rs/moq-flate/MIGRATE.md index 97a02c1bc80..d9ec3809e96 100644 --- a/rs/moq-flate/MIGRATE.md +++ b/rs/moq-flate/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +`Decoder` holds its whole window, across every frame, to `moq_net::group::MAX_CACHE_BYTES` (32 MiB), the most a plain group carries, instead of capping each frame at 64 MiB. +`DEFAULT_MAX_FRAME_SIZE` and `Decoder::with_max_frame_size` are removed; there is no other cap to set. +The producers charge a compressed payload its inflated size: a snapshot value over 32 MiB fails with `moq_net::Error::FrameTooLarge` however small it deflates, and a stream log ends its budget at 32 MiB of inflated payload. +Split a larger value across tracks. + ### Consumers return `Timed` (#5677) Snapshot and stream `next` / `poll_next` return `moq_net::Timed`: read `.value` for the payload and `.at` for the frame's exact timestamp, `None` when untimed. diff --git a/rs/moq-flate/src/lib.rs b/rs/moq-flate/src/lib.rs index 1c17293eef9..bf96de72da8 100644 --- a/rs/moq-flate/src/lib.rs +++ b/rs/moq-flate/src/lib.rs @@ -31,7 +31,9 @@ //! This is plain raw DEFLATE with a `Z_SYNC_FLUSH` after each frame, so any peer using the same //! primitive (zlib's sync flush, the browser's `deflate-raw`) interoperates on the wire. There is no //! length prefix: the caller is expected to frame each slice (moq-net already does). A small slice -//! can still inflate to far more than its own size, so [`Decoder::frame`] bounds each frame's output. +//! can still inflate to far more than its own size, so a [`Decoder`] holds its whole window to the +//! group budget ([`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES)): a compressed group never +//! costs a reader more than a plain one, and the encoders refuse or roll before a group would. //! //! A sync flush always ends in the 4-byte empty-block marker `00 00 ff ff`. That marker is fixed, so //! [`Encoder::frame`] drops it from each slice and [`Decoder::frame`] re-appends it before inflating, @@ -67,9 +69,6 @@ use flate2::{Compress, Decompress, FlushCompress, FlushDecompress, Status}; /// the small, repetitive payloads this targets. pub const DEFAULT_LEVEL: u32 = 6; -/// The default per-frame decompressed-size cap ([`Decoder::new`]): 64 MiB. -pub const DEFAULT_MAX_FRAME_SIZE: u64 = 64 * 1024 * 1024; - /// The trailing bytes of a DEFLATE sync flush, stripped on the wire and re-appended to decode. const SYNC_FLUSH_TAIL: [u8; 4] = [0x00, 0x00, 0xff, 0xff]; @@ -101,8 +100,8 @@ pub enum Error { #[error("decompression failed")] Decompress, - /// A frame's decompressed size exceeded the configured limit (zip-bomb guard). - #[error("decompressed frame exceeded {0} bytes")] + /// A group inflated past the group budget, which no conforming encoder produces (zip-bomb guard). + #[error("decompressed group exceeded {0} bytes")] TooLarge(u64), /// An error from the underlying track. @@ -198,34 +197,31 @@ impl Default for Encoder { /// Decodes a stream's frame slices back into the original payloads. Hold one per stream; feed slices /// in frame order (each frame builds on the earlier ones). +/// +/// A peer could send tiny slices that inflate hugely, so the window's total output is held to the +/// group budget, [`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES): the most a plain group can +/// carry. A compressed group then costs a reader no more than a plain one, however small its slices. pub struct Decoder { inner: Box, - max_frame_size: u64, + /// The most bytes the whole window may inflate to. A field so a test can shrink it. + budget: u64, } impl Decoder { - /// Start a fresh decoder with a cold window and the [`DEFAULT_MAX_FRAME_SIZE`] cap. + /// Start a fresh decoder with a cold window. pub fn new() -> Self { - Self::with_max_frame_size(DEFAULT_MAX_FRAME_SIZE) - } - - /// Start a fresh decoder with a cold window and a custom per-frame decompressed-size cap. - /// - /// A malicious or buggy peer could send a tiny slice that inflates hugely, so [`frame`](Self::frame) - /// stops and returns [`Error::TooLarge`] once a single frame's output would exceed `max_frame_size`. - pub fn with_max_frame_size(max_frame_size: u64) -> Self { // `false`: raw DEFLATE, matching the encoder. Self { inner: Box::new(Decompress::new(false)), - max_frame_size, + budget: moq_net::group::MAX_CACHE_BYTES, } } /// Decompress the next frame's `slice` back into its payload. /// - /// An empty slice yields an empty payload. Returns [`Error::TooLarge`] if the frame inflates past - /// the configured cap (checked as output is produced, not from any declared size), and - /// [`Error::Decompress`] on malformed input. + /// An empty slice yields an empty payload. Returns [`Error::TooLarge`] once the window's output, + /// counted across every frame so far, passes the group budget (checked as output is produced, not + /// from any declared size), and [`Error::Decompress`] on malformed input. pub fn frame(&mut self, slice: &[u8]) -> Result { let mut out = Vec::new(); self.frame_into(slice, &mut out)?; @@ -255,9 +251,9 @@ impl Decoder { .map_err(|_| Error::Decompress)?; let consumed = (self.inner.total_in() - before_in) as usize; let produced = (self.inner.total_out() - before_out) as usize; - // Bound the inflated output as it is produced; a tiny slice can expand enormously. - if out.len() as u64 + produced as u64 > self.max_frame_size { - return Err(Error::TooLarge(self.max_frame_size)); + // Bound the window's inflated output as it is produced; a tiny slice can expand enormously. + if self.inner.total_out() > self.budget { + return Err(Error::TooLarge(self.budget)); } out.extend_from_slice(&tmp[..produced]); input = &input[consumed..]; @@ -425,13 +421,16 @@ mod test { } #[test] - fn enforces_max_frame_size() { - // A tiny slice of a highly compressible payload inflates past a small cap. - let payload = vec![0u8; 1024]; - let slice = Encoder::new().frame(&payload); + fn the_budget_spans_the_whole_window() { + // Each frame fits on its own; together they inflate past the budget. + let mut enc = Encoder::new(); + let first = enc.frame(&[0u8; 1024]); + let second = enc.frame(&[0u8; 1024]); - let mut dec = Decoder::with_max_frame_size(512); - assert!(matches!(dec.frame(&slice), Err(Error::TooLarge(512)))); + let mut dec = Decoder::new(); + dec.budget = 1536; + dec.frame(&first).unwrap(); + assert!(matches!(dec.frame(&second), Err(Error::TooLarge(1536)))); } #[test] diff --git a/rs/moq-flate/src/snapshot/mod.rs b/rs/moq-flate/src/snapshot/mod.rs index 055ea9eb446..39ed7583a47 100644 --- a/rs/moq-flate/src/snapshot/mod.rs +++ b/rs/moq-flate/src/snapshot/mod.rs @@ -267,18 +267,24 @@ mod test { /// `append_group` publishes immediately, so rejecting the frame inside `write_frame` would leave /// an empty newest group behind. A snapshot consumer jumps to the newest, so the previous value - /// would vanish even though the update reported an error. + /// would vanish even though the update reported an error. Compression doesn't lift the limit: a + /// value that inflates past the group budget is refused even though it deflates to a few KiB. #[test] fn a_rejected_update_leaves_the_previous_value_readable() { - let (mut producer, track) = producer(false); - producer.update(&b"keep"[..]).unwrap(); - - let oversized = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); - assert!(producer.update(oversized).is_err()); - producer.finish().unwrap(); - - // A reader arriving now still finds the last good value, not an empty superseding group. - assert_eq!(drain(consume(track, false)), vec![Bytes::from_static(b"keep")]); + for compression in [false, true] { + let (mut producer, track) = producer(compression); + producer.update(&b"keep"[..]).unwrap(); + + let oversized = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); + assert!(matches!( + producer.update(oversized), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + )); + producer.finish().unwrap(); + + // A reader arriving now still finds the last good value, not an empty superseding group. + assert_eq!(drain(consume(track, compression)), vec![Bytes::from_static(b"keep")]); + } } /// `finish` closes the underlying track, so a later update fails rather than being silently diff --git a/rs/moq-flate/src/snapshot/producer.rs b/rs/moq-flate/src/snapshot/producer.rs index b2dcd80f075..5515d4df39a 100644 --- a/rs/moq-flate/src/snapshot/producer.rs +++ b/rs/moq-flate/src/snapshot/producer.rs @@ -71,23 +71,20 @@ impl Inner { let timestamp = payload.at; let payload = payload.value; + // Check before opening a group. `append_group` publishes immediately, so discovering the limit + // inside `write_frame` would leave an empty newest group behind: a snapshot consumer jumps to + // the newest, so the previous value would be lost even though this update reported an error. + // A compressed value is held to the same budget by its inflated size, since that is what every + // consumer's decoder allows: compression saves bandwidth, never what a reader holds. + if payload.len() as u64 > moq_net::group::MAX_CACHE_BYTES { + return Err(moq_net::Error::FrameTooLarge.into()); + } + // One frame per group, so the window spans a single value and starts cold every time. let payload = match self.compression { - true => { - // Compression can take a large value under the group's frame limit, but every consumer - // decodes with the default output cap, so publishing past it would advertise a - // value that always fails to read. Reject it here instead. - if payload.len() as u64 > crate::DEFAULT_MAX_FRAME_SIZE { - return Err(crate::Error::TooLarge(crate::DEFAULT_MAX_FRAME_SIZE)); - } - crate::Encoder::new().frame(&payload) - } + true => crate::Encoder::new().frame(&payload), false => payload, }; - - // Check before opening a group. `append_group` publishes immediately, so discovering the limit - // inside `write_frame` would leave an empty newest group behind: a snapshot consumer jumps to - // the newest, so the previous value would be lost even though this update reported an error. if payload.len() as u64 > moq_net::group::MAX_CACHE_BYTES { return Err(moq_net::Error::FrameTooLarge.into()); } diff --git a/rs/moq-flate/src/stream/mod.rs b/rs/moq-flate/src/stream/mod.rs index 96fbea32c69..01ad81bec98 100644 --- a/rs/moq-flate/src/stream/mod.rs +++ b/rs/moq-flate/src/stream/mod.rs @@ -22,9 +22,9 @@ //! Instead an append that might not fit is refused with [`moq_net::Error::GroupTooLarge`] before //! it is encoded, and the log stays intact and writable. With compression the check counts the //! payload's raw size plus DEFLATE's worst-case overhead ([`Encoder::bound`](crate::Encoder::bound)), -//! since the compressed size is only known once the window has moved. The budget is below the -//! decoder's [`DEFAULT_MAX_FRAME_SIZE`](crate::DEFAULT_MAX_FRAME_SIZE), so any payload that fits is -//! one every consumer can inflate. The budget covers the whole log rather than a single payload, so +//! since the compressed size is only known once the window has moved, and each written payload is +//! charged its inflated size: the [`Decoder`](crate::Decoder) holds its window to the same budget, so +//! compression shrinks the wire but not how much a log can hold. The budget covers the whole log rather than a single payload, so //! once it is spent every append is refused: the live stream is bounded history by design, and a //! publisher with more to say opens a new track. @@ -290,11 +290,10 @@ mod test { /// A payload past the group budget is refused before anything is written, so the log carries /// on: the next payload lands and decodes, which with compression proves the window never - /// moved. One past the decoder's cap is refused the same way rather than ending the track. + /// moved. #[test] fn an_oversized_payload_is_refused_and_the_log_continues() { - let huge = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); - let past_budget = huge.slice(..moq_net::group::MAX_CACHE_BYTES as usize); + let past_budget = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize]); for compression in [false, true] { let (mut producer, track) = producer(compression); @@ -304,7 +303,6 @@ mod test { refused(producer.append(past_budget.clone())), "compression={compression}" ); - assert!(refused(producer.append(huge.clone())), "compression={compression}"); assert!(producer.demand().wanted().is_some(), "a refusal must not end the track"); producer.append(&b"second"[..]).unwrap(); @@ -322,7 +320,7 @@ mod test { #[test] fn a_refused_first_payload_publishes_nothing() { let (mut producer, mut track) = producer(true); - let huge = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); + let huge = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); assert!(refused(producer.append(huge))); assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), @@ -330,6 +328,19 @@ mod test { ); } + /// A compressed payload is charged its inflated size, since that is what every reader's decoder + /// holds the group to: two payloads that each fit, and deflate to almost nothing, still overflow + /// the budget together. + #[test] + fn a_compressed_log_is_charged_its_inflated_size() { + let half = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize / 2 + 1]); + let (mut producer, track) = producer(true); + assert!(producer.append(half.clone()).unwrap() < 64 * 1024); + assert!(refused(producer.append(half.clone()))); + producer.finish().unwrap(); + assert_eq!(drain(track, true), vec![half]); + } + /// The budget covers the whole log, so once its frames are spent every append is refused, and /// the log written so far still finishes cleanly and reads back whole. #[test] diff --git a/rs/moq-flate/src/stream/producer.rs b/rs/moq-flate/src/stream/producer.rs index 37fefe79c8e..678a0035c5a 100644 --- a/rs/moq-flate/src/stream/producer.rs +++ b/rs/moq-flate/src/stream/producer.rs @@ -84,16 +84,13 @@ struct Inner { /// The DEFLATE encoder, one window for the whole group, `Some` while compressing. flate: Option, - /// Frames and payload bytes written to the group, checked against moq-net's group budget before - /// each payload is encoded. + /// Frames and bytes charged to the group, checked against moq-net's group budget before each + /// payload is encoded. A compressed payload is charged its inflated size (or its slice, if + /// larger), the budget every consumer's [`Decoder`](crate::Decoder) holds its window to. frames: usize, bytes: u64, } -// The budget check also stands in for the decoder's cap: a payload that fits the group is one every -// consumer can inflate. -const _: () = assert!(moq_net::group::MAX_CACHE_BYTES <= crate::DEFAULT_MAX_FRAME_SIZE); - impl Inner { fn append(&mut self, payload: Timed) -> Result { let timestamp = payload.at; @@ -113,13 +110,13 @@ impl Inner { // Check before compressing: encoding advances the window, so a payload refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. Also checked before the group is opened, so a refused - // first payload publishes nothing. - let size = payload.len() as u64; + // actual size for the same reason, and it covers the inflated size too. Also checked before + // the group is opened, so a refused first payload publishes nothing. + let plain = payload.len() as u64; let bound = if self.flate.is_some() { - crate::Encoder::bound(size) + crate::Encoder::bound(plain) } else { - size + plain }; if self.frames >= moq_net::group::MAX_GROUP_FRAMES || self.bytes.saturating_add(bound) > moq_net::group::MAX_CACHE_BYTES @@ -142,7 +139,7 @@ impl Inner { let group = self.group.as_mut().expect("a group is open"); let Err(err) = group.write_frame(timestamp, payload) else { self.frames += 1; - self.bytes += size as u64; + self.bytes += plain.max(size as u64); return Ok(size); }; diff --git a/rs/moq-json/MIGRATE.md b/rs/moq-json/MIGRATE.md index b991973441f..507f510cf72 100644 --- a/rs/moq-json/MIGRATE.md +++ b/rs/moq-json/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +A compressed group may inflate to at most `moq_net::group::MAX_CACHE_BYTES` (32 MiB), the most a plain group carries, so a peer's small frames cannot make a reader hold more than an uncompressed track would. +The encoders charge each frame its plaintext: a compressed snapshot or window header over 32 MiB fails with `moq_net::Error::FrameTooLarge` (snapshot) or the window's group cache error, where it used to pass up to 64 MiB, and deltas and ops roll a new group once a group's plaintext would pass the budget. +A compressed stream's budget is 32 MiB of record plaintext. +`snapshot::consumer::Config::max_size` still sets a tighter limit, but no longer stops inflation early; a frame inflates within the group budget and is then refused. + ### Groups roll on joiner demand `snapshot::Config::delta_ratio` and `window::ProducerConfig::op_ratio`, with `with_delta_ratio` and `with_op_ratio`, are replaced by `deltas: bool` and `ops: bool` (both default `true`). diff --git a/rs/moq-json/src/snapshot/consumer.rs b/rs/moq-json/src/snapshot/consumer.rs index a21692f2d16..defea94d42c 100644 --- a/rs/moq-json/src/snapshot/consumer.rs +++ b/rs/moq-json/src/snapshot/consumer.rs @@ -19,7 +19,11 @@ pub struct Config { pub compression: Compression, /// Maximum bytes in an inflated frame or the compact JSON of the reconstructed value. - /// Unset preserves the DEFLATE decoder's default frame cap and leaves state size unlimited. + /// + /// Unset, the group budget is the only bound: a compressed group inflates to at most + /// [`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES), the most a plain one carries, so the + /// reconstructed value costs no more than an uncompressed track's. Set it to hold a smaller + /// document, at the cost of re-measuring the value after every delta. pub max_size: Option, } diff --git a/rs/moq-json/src/snapshot/decoder.rs b/rs/moq-json/src/snapshot/decoder.rs index 835e9441c7b..fbc9d179b4f 100644 --- a/rs/moq-json/src/snapshot/decoder.rs +++ b/rs/moq-json/src/snapshot/decoder.rs @@ -68,12 +68,8 @@ impl Decoder { /// only the first. pub fn snapshot(&mut self, payload: &[u8]) -> Result<()> { self.current = None; - // Every group starts a new DEFLATE window; enforce the budget while inflating. - self.flate = self.compression.then(|| { - moq_flate::Decoder::with_max_frame_size(self.max_size.map_or(moq_flate::DEFAULT_MAX_FRAME_SIZE, |size| { - (size as u64).min(moq_flate::DEFAULT_MAX_FRAME_SIZE) - })) - }); + // Every group starts a new DEFLATE window, which holds the whole group to the group budget. + self.flate = self.compression.then(moq_flate::Decoder::new); let inflated = self.flate.as_mut().map(|flate| flate.frame(payload)).transpose()?; let plain = inflated.as_deref().unwrap_or(payload); if let Some(limit) = self.max_size @@ -272,7 +268,7 @@ mod test { } else { decoder.delta(&payload) }; - assert!(matches!(result, Err(Error::Flate(moq_flate::Error::TooLarge(1024))))); + assert!(matches!(result, Err(Error::TooLarge(1024)))); assert!(decoder.value().is_none()); } } diff --git a/rs/moq-json/src/snapshot/encoder.rs b/rs/moq-json/src/snapshot/encoder.rs index 493e3dc49c2..fcb79a6569b 100644 --- a/rs/moq-json/src/snapshot/encoder.rs +++ b/rs/moq-json/src/snapshot/encoder.rs @@ -215,7 +215,8 @@ pub struct Encoder { /// The current group's DEFLATE encoder (one window per group), `Some` while compressing. flate: Option, - /// Payload bytes emitted into the current group, snapshot included. + /// Bytes charged to the current group, snapshot included: each frame's slice, or its plaintext + /// when that is larger, since a reader's DEFLATE window is held to the same budget. group_bytes: u64, /// Frames emitted into the current group, snapshot included. @@ -352,12 +353,14 @@ impl Encoder { /// Encode an already-serialized value as a full snapshot, opening a new group. fn snapshot_encoded(&mut self, snapshot: Bytes) -> Result { - // Every consumer decodes with moq-flate's default output cap, so a value past it would be - // unreadable however small it compresses to. Reject it before anything is published, so the - // previously published value stands rather than being superseded by one nothing can read. - if self.config.compression.is_deflate() && snapshot.len() as u64 > moq_flate::DEFAULT_MAX_FRAME_SIZE { - return Err(moq_flate::Error::TooLarge(moq_flate::DEFAULT_MAX_FRAME_SIZE).into()); + // Every consumer holds a compressed group to the group budget by its inflated size, so a value + // past it would be unreadable however small it compresses to: the same `FrameTooLarge` a plain + // one meets. Reject it before anything is published, so the previously published value stands + // rather than being superseded by one nothing can read. + if self.config.compression.is_deflate() && snapshot.len() as u64 > self.max_group_bytes { + return Err(moq_net::Error::FrameTooLarge.into()); } + let plain = snapshot.len() as u64; // Open a fresh per-group encoder (cold window) and compress the snapshot as frame 0. let (payload, flate) = match self.config.compression { @@ -369,7 +372,7 @@ impl Encoder { Compression::None => (snapshot, None), }; - self.group_bytes = payload.len() as u64; + self.group_bytes = plain.max(payload.len() as u64); self.group_frames = 1; self.flate = flate; self.resync = false; @@ -443,13 +446,6 @@ impl Encoder { // Compress into the per-group window only now, for a frame we are committed to emitting. let bytes = Bytes::from(patch); - - // Same cap as a snapshot, on the patch's plaintext: a delta that decompresses past the - // consumer's limit makes the whole group unreadable, since there is no keyframe after it to - // resynchronize on. Rejecting here leaves the encoder to reset and the group as it was. - if self.config.compression.is_deflate() && bytes.len() as u64 > moq_flate::DEFAULT_MAX_FRAME_SIZE { - return Err(moq_flate::Error::TooLarge(moq_flate::DEFAULT_MAX_FRAME_SIZE).into()); - } let payload = match self.flate.as_mut() { Some(flate) => flate.frame(&bytes), None => bytes.clone(), @@ -457,18 +453,20 @@ impl Encoder { // A delta is only readable while the group still holds the snapshot it applies to. // Admitting a patch that pushes the group past that budget would abort it - // (`GroupTooLarge`), leaving a late subscriber with no value. Roll a fresh snapshot - // instead, which is cheap next to losing the value. + // (`GroupTooLarge`), or past a reader's inflate budget, leaving a late subscriber with no + // value. Roll a fresh snapshot instead, which is cheap next to losing the value. // - // Measured on the encoded payload rather than the plaintext: a sync-flushed DEFLATE frame can - // come out slightly larger than its input, so the plaintext is not an upper bound. Compressing - // first advances the window, but a snapshot opens a fresh one, so an over-budget delta - // costs only the wasted compression. - if self.group_bytes + payload.len() as u64 > self.max_group_bytes { + // Charged the larger of the encoded payload and the plaintext: a sync-flushed DEFLATE frame + // can come out slightly larger than its input, while the reader's window counts what it + // inflates. Compressing first advanced the window past what any reader holds, so drop the + // group before rolling: if the snapshot is refused too, the next update must not be a delta. + let charge = (payload.len() as u64).max(bytes.len() as u64); + if self.group_bytes + charge > self.max_group_bytes { + self.reset(); return self.encode_snapshot(value).map(Some); } - self.group_bytes += payload.len() as u64; + self.group_bytes += charge; self.group_frames += 1; // The patch folds into the baseline on commit, so a frame that never reaches the wire leaves @@ -950,7 +948,7 @@ mod test { // Measure the frames under the default budget, which admits the delta. let mut probe = Encoder::::new(config.clone()); - let snapshot = commit(&mut probe, &value).expect("a snapshot"); + assert!(commit(&mut probe, &value).expect("a snapshot").keyframe); let delta = commit(&mut probe, &patched).expect("a delta"); assert!(!delta.keyframe); assert!( @@ -959,9 +957,10 @@ mod test { delta.payload.len() ); - // A budget with room for the plaintext patch but not the encoded one. + // A budget with room for the plaintext patch but not the encoded one. The snapshot is charged + // its plaintext, which outweighs its slice. let mut encoder = Encoder::::new(config); - encoder.max_group_bytes = (snapshot.payload.len() + plaintext) as u64; + encoder.max_group_bytes = (serde_json::to_vec(&value).unwrap().len() + plaintext) as u64; assert!(commit(&mut encoder, &value).unwrap().keyframe); // The patch rolls a fresh snapshot instead of joining the first group. @@ -975,6 +974,77 @@ mod test { ); } + /// A compressed delta is charged its plaintext too: the reader's window counts what it inflates, + /// so a patch that deflates to almost nothing still rolls the group once its plaintext would + /// overflow the budget. Matches the JS `a compressed delta is gated on its inflated size`. + #[test] + fn a_compressed_delta_is_gated_on_its_inflated_size() { + let value = json!({ "v": "x".repeat(1000) }); + let patched = json!({ "v": "y".repeat(1000) }); + let config = Config { + deltas: true, + compression: Compression::Deflate, + }; + + let mut encoder = Encoder::::new(config); + encoder.max_group_bytes = (serde_json::to_vec(&value).unwrap().len() + 500) as u64; + assert!(commit(&mut encoder, &value).unwrap().keyframe); + let rolled = commit(&mut encoder, &patched).expect("a rolled snapshot"); + assert!( + rolled.keyframe, + "a 1 KB patch must not ride in a group with 500 bytes left" + ); + assert!( + rolled.payload.len() < 500, + "the patch deflates small, so only its plaintext overflows" + ); + } + + /// A delta that overflows the group advances the window before the roll is attempted, so a refused + /// roll must not leave that window behind: the next update is a snapshot a reader can decode. + #[test] + fn a_refused_roll_resynchronizes() { + let config = Config { + deltas: true, + compression: Compression::Deflate, + }; + let mut encoder = Encoder::::new(config); + encoder.max_group_bytes = 200; + assert!(commit(&mut encoder, &json!({ "a": 1 })).unwrap().keyframe); + assert!(matches!( + encoder.update(&json!({ "a": 1, "b": "x".repeat(300) })), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + )); + + let next = commit(&mut encoder, &json!({ "a": 2 })).expect("a snapshot"); + assert!(next.keyframe); + let mut decoder = moq_flate::Decoder::new(); + assert_eq!( + serde_json::from_slice::(&decoder.frame(&next.payload).unwrap()).unwrap(), + json!({ "a": 2 }) + ); + } + + /// A compressed snapshot is refused by its plaintext, the way a plain one is, however small it + /// deflates: no reader could inflate it. + #[test] + fn a_compressed_snapshot_past_the_budget_is_refused() { + let config = Config { + deltas: false, + compression: Compression::Deflate, + }; + let mut encoder = Encoder::::new(config); + encoder.max_group_bytes = 100; + assert!(matches!( + encoder.update(&json!({ "v": "x".repeat(1000) })), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + )); + assert!( + encoder.value().is_none(), + "a refused snapshot leaves the encoder as it was" + ); + } + #[test] fn compressed_deltas_reuse_the_group_window() { let phrase = "Media over QUIC delivers real-time latency at massive scale"; diff --git a/rs/moq-json/src/stream/encoder.rs b/rs/moq-json/src/stream/encoder.rs index d7ecea8d69e..e48a048d8c0 100644 --- a/rs/moq-json/src/stream/encoder.rs +++ b/rs/moq-json/src/stream/encoder.rs @@ -37,6 +37,8 @@ pub struct Config { pub struct Pending<'a, T> { encoder: &'a mut Encoder, payload: Bytes, + /// What the record costs the group's budget: its plaintext or its slice, whichever is larger. + charge: u64, committed: bool, } @@ -52,7 +54,7 @@ impl Pending<'_, T> { pub fn commit(mut self) { self.committed = true; self.encoder.frames += 1; - self.encoder.bytes += self.payload.len() as u64; + self.encoder.bytes += self.charge; } } @@ -82,8 +84,9 @@ pub struct Encoder { /// consumer for the rest of the group, so encoding stops until the caller rolls a new one. desynced: bool, - /// Frames and payload bytes committed to the current group, checked against moq-net's group - /// budget before each record is encoded. + /// Frames and bytes committed to the current group, checked against moq-net's group budget + /// before each record is encoded. A compressed record is charged its plaintext (or its slice, if + /// larger), since a reader's DEFLATE window is held to the same budget. frames: usize, bytes: u64, @@ -143,8 +146,7 @@ impl Encoder { // Check before compressing: encoding advances the window, so a record refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. The budget is also below moq-flate's per-frame decode cap, - // so any record that fits is one every consumer can inflate. + // actual size for the same reason; it also covers the plaintext a reader inflates. let size = bytes.len() as u64; let bound = if self.compression { moq_flate::Encoder::bound(size) @@ -164,6 +166,7 @@ impl Encoder { Ok(Pending { encoder: self, + charge: size.max(payload.len() as u64), payload, committed: false, }) diff --git a/rs/moq-json/src/stream/mod.rs b/rs/moq-json/src/stream/mod.rs index 7f6a2b3d8f9..4aa09493f90 100644 --- a/rs/moq-json/src/stream/mod.rs +++ b/rs/moq-json/src/stream/mod.rs @@ -29,7 +29,9 @@ //! Instead an append that might not fit is refused with [`moq_net::Error::GroupTooLarge`] before //! it is encoded, and the log stays intact and writable. With compression the check counts the //! record's raw size plus DEFLATE's worst-case overhead, since the compressed size is only known -//! once the window has moved. The budget covers the whole log rather than a single record, so once +//! once the window has moved, and each written record is charged its raw size: a reader's DEFLATE +//! window is held to the same budget, so compression shrinks the wire but not how much a log can +//! hold. The budget covers the whole log rather than a single record, so once //! it is spent every append is refused: the live stream is bounded history by design, a publisher //! with more to say opens a new track, and deep history is served from a recording. //! @@ -453,6 +455,18 @@ mod test { } } + /// A compressed record is charged its plaintext, since that is what a reader's window inflates: + /// two records that each fit, and deflate to almost nothing, still overflow the budget together. + #[test] + fn a_compressed_log_is_charged_its_inflated_size() { + let (mut producer, track) = producer(compressed()); + let half = Value::String("x".repeat(moq_net::group::MAX_CACHE_BYTES as usize / 2)); + producer.append(&half).unwrap(); + assert!(refused(producer.append(&half))); + producer.finish().unwrap(); + assert_eq!(drain(consume(track, true)), vec![half]); + } + /// The budget covers the whole log, so once its frames are spent every append is refused, and /// the log written so far still finishes cleanly and reads back whole. #[test] diff --git a/rs/moq-json/src/window/encoder.rs b/rs/moq-json/src/window/encoder.rs index 8f348944305..3364532499f 100644 --- a/rs/moq-json/src/window/encoder.rs +++ b/rs/moq-json/src/window/encoder.rs @@ -178,7 +178,8 @@ pub struct Encoder { /// The current group's DEFLATE encoder (one window per group), `Some` while compressing. flate: Option, - /// Payload bytes emitted into the current group, header included. + /// Bytes charged to the current group, header included: each frame's slice, or its plaintext + /// when that is larger, since a reader's DEFLATE window is held to the same budget. group_bytes: u64, /// Frames emitted into the current group, header included. @@ -280,42 +281,27 @@ impl Encoder { self.config.ops && self.group_frames > 0 && self.group_frames < MAX_GROUP_FRAMES } - /// Reject plaintext that the paired DEFLATE decoder could not produce. - fn validate_plaintext(len: usize, kind: &str) -> Result<()> { - if u64::try_from(len).unwrap_or(u64::MAX) > moq_flate::DEFAULT_MAX_FRAME_SIZE { - return Err(Error::Json(format!( - "window {kind} exceeds the decoder's decompressed size limit" - ))); - } - Ok(()) - } - - /// Compress an already-serialized op into the open group, counting it toward the caps. - fn frame(&mut self, bytes: Vec) -> Result { - Self::validate_plaintext(bytes.len(), "frame")?; + /// Emit an op when the header will remain cached and inflatable, otherwise restate the window + /// in a new group. + /// + /// Charged the larger of the slice and the plaintext: a sync-flushed DEFLATE frame can come out + /// slightly larger than its input, while a reader's window counts what it inflates. + fn emit_op(&mut self, bytes: Vec) -> Result> { + let plain = bytes.len() as u64; let payload = match self.flate.as_mut() { Some(flate) => flate.frame(&bytes), None => Bytes::from(bytes), }; - - self.group_bytes += payload.len() as u64; self.group_frames += 1; - - Ok(Encoded { - payload, - keyframe: false, - }) - } - - /// Emit an op when the header will remain cached, otherwise restate the window in a new group. - fn emit_op(&mut self, bytes: Vec) -> Result> { - let encoded = self.frame(bytes)?; + self.group_bytes = self.group_bytes.saturating_add(plain.max(payload.len() as u64)); if self.group_bytes > moq_net::group::MAX_CACHE_BYTES { self.reset(); - Ok(None) - } else { - Ok(Some(encoded)) + return Ok(None); } + Ok(Some(Encoded { + payload, + keyframe: false, + })) } /// Serialize a bounded suffix before mutably borrowing the compression state. @@ -343,7 +329,11 @@ impl Encoder { /// Encode the header restating the whole window and opening a new group. fn emit_header(&mut self, bytes: Vec) -> Result { - Self::validate_plaintext(bytes.len(), "header")?; + // A reader inflates the header whole, so its plaintext meets the group budget too. + let plain = bytes.len() as u64; + if plain > moq_net::group::MAX_CACHE_BYTES { + return Err(Error::Json("window header exceeds the group cache limit".into())); + } // Open a fresh per-group encoder (cold window) and compress the header as frame 0. let (payload, flate) = match self.config.compression { @@ -358,7 +348,7 @@ impl Encoder { return Err(Error::Json("window header exceeds the group cache limit".into())); } - self.group_bytes = payload.len() as u64; + self.group_bytes = plain.max(payload.len() as u64); self.group_frames = 1; self.flate = flate; self.resync = false; @@ -540,22 +530,24 @@ mod test { assert_eq!(encoder.window(), vec![Value::from(2)]); } + /// Compression doesn't lift the limit: a reader inflates the header whole, so one past the group + /// budget is refused however small it deflates. #[test] fn a_header_larger_than_the_group_cache_is_rejected() { - let mut encoder = Encoder::::new(ProducerConfig::default()); - let record = "x".repeat(moq_net::group::MAX_CACHE_BYTES as usize); - - let err = encoder.push(&record).err().expect("oversized header should fail"); - assert!(err.to_string().contains("group cache limit")); - assert!(encoder.window().is_empty()); - - let frame = encoder.push(&"ok".to_string()).unwrap(); - assert!(frame.keyframe); - } - - #[test] - fn plaintext_is_bounded_by_the_decoder_limit() { - let len = usize::try_from(moq_flate::DEFAULT_MAX_FRAME_SIZE + 1).unwrap(); - assert!(Encoder::<()>::validate_plaintext(len, "frame").is_err()); + for compression in [false, true] { + let config = ProducerConfig { + compression, + ..Default::default() + }; + let mut encoder = Encoder::::new(config); + let record = "x".repeat(moq_net::group::MAX_CACHE_BYTES as usize); + + let err = encoder.push(&record).err().expect("oversized header should fail"); + assert!(err.to_string().contains("group cache limit")); + assert!(encoder.window().is_empty()); + + let frame = encoder.push(&"ok".to_string()).unwrap(); + assert!(frame.keyframe); + } } } diff --git a/rs/moq-stats/MIGRATE.md b/rs/moq-stats/MIGRATE.md index a14972577b2..a7c8ddf77f9 100644 --- a/rs/moq-stats/MIGRATE.md +++ b/rs/moq-stats/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### The `.z` flavor ends with the plain one past 32 MB + +A frame whose uncompressed JSON passes the 32 MB group budget now ends both the plain track and its `.z` sibling with `FrameTooLarge`: a reader inflates the whole group, so compression no longer carries a larger frame. +A reader that fell back to `.z` for very large frames sees the same end the plain reader does. + ### A returning group restarts under a new epoch (#4904) A stats group that returns announces a new epoch and starts its group numbers at zero, instead of continuing the old sequence. diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index b4c3342622c..4f8d113c3c2 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -485,7 +485,8 @@ impl Snapshot { /// /// A frame over the cache limit aborts the track with /// [`moq_net::Error::FrameTooLarge`]: skipping it would leave readers on the - /// previous snapshot with no sign it stopped tracking. + /// previous snapshot with no sign it stopped tracking. A compressed frame + /// meets the same limit by its uncompressed size, which the encoder refuses. fn update(&mut self, value: &Frame) -> moq_json::Result> { if self.track.is_closed() { return Ok(None); @@ -498,18 +499,20 @@ impl Snapshot { { self.encoder.reset(); } - let Some(frame) = self.encoder.update(value)? else { - return Ok(None); + let frame = match self.encoder.update(value) { + Ok(Some(frame)) if frame.payload.len() as u64 <= moq_net::group::MAX_CACHE_BYTES => frame, + Ok(None) => return Ok(None), + Ok(Some(_)) | Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) => { + let err = moq_net::Error::FrameTooLarge; + if let Some(group) = self.group.take() { + let _ = group.abort(err.clone()); + } + let _ = self.track.clone().abort(err.clone()); + return Err(err.into()); + } + Err(err) => return Err(err), }; let len = frame.payload.len(); - if len as u64 > moq_net::group::MAX_CACHE_BYTES { - let err = moq_net::Error::FrameTooLarge; - if let Some(group) = self.group.take() { - let _ = group.abort(err.clone()); - } - let _ = self.track.clone().abort(err.clone()); - return Err(err.into()); - } if frame.keyframe { if let Some(mut group) = self.group.take() { group.finish()?; @@ -1854,11 +1857,12 @@ mod tests { } } - /// A flavor whose frame outgrows the cache ends with `FrameTooLarge` instead - /// of leaving readers on its last snapshot, while its sibling, which DEFLATE - /// shrinks under the limit, keeps updating. + /// A frame that outgrows the cache ends both flavors with `FrameTooLarge` + /// instead of leaving readers on their last snapshot. DEFLATE shrinks the + /// `.z` frame under the limit, but a reader inflates it whole, so the + /// encoder holds it to the same budget by its uncompressed size. #[tokio::test] - async fn oversized_frame_aborts_only_its_flavor() { + async fn oversized_frame_ends_both_flavors() { use futures::FutureExt; type Value = BTreeMap; @@ -1898,17 +1902,12 @@ mod tests { let huge = "a".repeat(moq_net::group::MAX_CACHE_BYTES as usize); publish(&huge, 2); - assert!(matches!( - next(&mut plain), - Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) - )); - assert_eq!(next(&mut compressed).unwrap(), Some(Value::from([(huge, 2)]))); - - publish("small", 3); - assert_eq!( - next(&mut compressed).unwrap(), - Some(Value::from([("small".to_string(), 3)])) - ); + for reader in [&mut plain, &mut compressed] { + assert!(matches!( + next(reader), + Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) + )); + } } /// Build an origin producer, spawning its driver on the ambient runtime.