From 3e84f8fc44fbe70a2dbd7855e8ddbe185867efdc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 10:02:00 -0700 Subject: [PATCH 1/3] fix(catalog): bound catalog decoding from a peer A compressed catalog frame is bounded only by the 64 MiB DEFLATE frame cap, and the reconstructed JSON value allocates several times the inflated size. Default snapshot consumers to 4 MiB, which every catalog and metadata track fits far below, and refuse an MSF catalog frame over the same limit before parsing it. Co-Authored-By: DeepSeek V4.1 Flash --- rs/moq-json/MIGRATE.md | 4 +++ rs/moq-json/src/snapshot/consumer.rs | 28 +++++++++++++-- rs/moq-mux/src/catalog/msf/consumer.rs | 47 ++++++++++++++++++++------ rs/moq-mux/src/catalog/msf/mod.rs | 9 +++++ 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/rs/moq-json/MIGRATE.md b/rs/moq-json/MIGRATE.md index d888087cec3..a0f4a7c71df 100644 --- a/rs/moq-json/MIGRATE.md +++ b/rs/moq-json/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Snapshot consumers are bounded by default + +`snapshot::consumer::Config::default()` now sets `max_size` to `DEFAULT_MAX_SIZE` (4 MiB), so a compressed frame from a peer cannot reconstruct an unbounded value. A consumer that needs a larger state sets `max_size` itself, or `None` to keep only the DEFLATE frame cap. + ### Consumers return `Timed` (#5677) Snapshot, stream, and window `next` / `poll_next` return `moq_net::Timed`: read `.value` for the value and `.at` for the frame's exact timestamp, `None` when untimed. diff --git a/rs/moq-json/src/snapshot/consumer.rs b/rs/moq-json/src/snapshot/consumer.rs index 688d4d3ea96..fd1530630df 100644 --- a/rs/moq-json/src/snapshot/consumer.rs +++ b/rs/moq-json/src/snapshot/consumer.rs @@ -7,11 +7,18 @@ use serde::de::DeserializeOwned; use super::Decoder; use crate::{Compression, Result}; +/// The default [`Config::max_size`]. +/// +/// A peer supplies the compressed frame, so an unset limit lets a small DEFLATE frame +/// reconstruct an unbounded value. Every catalog and metadata track this crate reads +/// fits far below this, and a consumer that needs more sets its own. +pub const DEFAULT_MAX_SIZE: usize = 4 * 1024 * 1024; + /// Track-owning options for a [`Consumer`]. /// /// Build from [`Default`] and override fields (the struct is `#[non_exhaustive]`, so new options /// stay additive). -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] #[non_exhaustive] pub struct Config { /// How the frames are compressed. Must match the encoder's @@ -19,10 +26,20 @@ pub struct Config { pub compression: Compression, /// Maximum bytes in an inflated frame or the compact JSON of the reconstructed value. - /// Unset preserves the DEFLATE decoder's default frame cap and leaves state size unlimited. + /// Defaults to [`DEFAULT_MAX_SIZE`]; `None` keeps only the DEFLATE decoder's frame + /// cap and leaves state size unlimited. pub max_size: Option, } +impl Default for Config { + fn default() -> Self { + Self { + compression: Compression::None, + max_size: Some(DEFAULT_MAX_SIZE), + } + } +} + /// Consumes a JSON value from a track, reconstructing it from snapshots and deltas. /// /// A [`Decoder`] that owns its track: it reads groups, routes each frame by its position, and @@ -168,6 +185,13 @@ impl Consumer { mod timestamps { use super::*; + #[test] + fn default_config_bounds_the_reconstructed_value() { + // A peer picks the compressed frame, so the default must not leave the + // reconstructed value unlimited. + assert_eq!(Config::default().max_size, Some(DEFAULT_MAX_SIZE)); + } + #[test] fn exact_timestamps_survive_reads() { for compression in [crate::Compression::None, crate::Compression::Deflate] { diff --git a/rs/moq-mux/src/catalog/msf/consumer.rs b/rs/moq-mux/src/catalog/msf/consumer.rs index 2b45d756c5c..8479e394ea8 100644 --- a/rs/moq-mux/src/catalog/msf/consumer.rs +++ b/rs/moq-mux/src/catalog/msf/consumer.rs @@ -50,16 +50,7 @@ impl Consumer { match ready!(group.poll_read_frame(waiter)?) { Some(frame) => { self.group = None; - let json = std::str::from_utf8(&frame.payload).map_err(|_| Error::InvalidUtf8)?; - let msf = match moq_msf::Catalog::::from_str(json) { - Ok(msf) => msf, - Err(err) => { - tracing::warn!(error = %err, "failed to parse MSF catalog frame"); - return Poll::Ready(Err(Error::ParseFrame.into())); - } - }; - let catalog = from_msf(&msf, &self.track.broadcast().path)?; - catalog.check_renditions()?; + let catalog = parse::(&frame.payload, &self.track.broadcast().path)?; return Poll::Ready(Ok(Some(catalog))); } None => self.group = None, @@ -83,6 +74,31 @@ impl Consumer { } } +/// Decode one catalog frame, bounding it before parsing. +/// +/// The frame comes from a peer, and JSON parsing allocates several times its bytes, +/// so a catalog must fit [`moq_json::snapshot::consumer::DEFAULT_MAX_SIZE`], the same +/// limit every compressed catalog track uses. +fn parse(payload: &[u8], base: &moq_net::Path<'_>) -> Result> { + const MAX: usize = moq_json::snapshot::consumer::DEFAULT_MAX_SIZE; + if payload.len() > MAX { + return Err(Error::FrameTooLarge { + size: payload.len(), + max: MAX, + } + .into()); + } + + let json = std::str::from_utf8(payload).map_err(|_| Error::InvalidUtf8)?; + let msf = moq_msf::Catalog::::from_str(json).map_err(|err| { + tracing::warn!(error = %err, "failed to parse MSF catalog frame"); + Error::ParseFrame + })?; + let catalog = from_msf(&msf, base)?; + catalog.check_renditions()?; + Ok(catalog) +} + impl From for Consumer { fn from(inner: moq_net::track::Subscriber) -> Self { Self::new(inner) @@ -1094,6 +1110,17 @@ mod test { ); } + #[test] + fn oversized_catalog_frame_is_refused_before_parsing() { + let payload = vec![b'{'; moq_json::snapshot::consumer::DEFAULT_MAX_SIZE + 1]; + let err = parse::<()>(&payload, &moq_net::Path::empty()).expect_err("oversized frame must be refused"); + assert!(matches!( + err, + crate::Error::Msf(Error::FrameTooLarge { size, max }) + if size == payload.len() && max == moq_json::snapshot::consumer::DEFAULT_MAX_SIZE + )); + } + #[test] fn invalid_video_codec_includes_codec_in_error() { // avc1 with a too-short profile string is a malformed structured codec. diff --git a/rs/moq-mux/src/catalog/msf/mod.rs b/rs/moq-mux/src/catalog/msf/mod.rs index de2539cc912..12ed707847f 100644 --- a/rs/moq-mux/src/catalog/msf/mod.rs +++ b/rs/moq-mux/src/catalog/msf/mod.rs @@ -48,6 +48,15 @@ pub enum Error { #[error("MSF catalog frame is not valid UTF-8")] InvalidUtf8, + /// The catalog frame is larger than a catalog may be. + #[error("MSF catalog frame is {size} bytes, over the {max}-byte limit")] + FrameTooLarge { + /// The frame's size in bytes. + size: usize, + /// The limit it exceeded. + max: usize, + }, + /// The catalog frame cannot be decoded as an MSF catalog. #[error("failed to parse MSF catalog frame")] ParseFrame, From 01388853d12c6877912ffce8820109147484f053 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 11:41:28 -0700 Subject: [PATCH 2/3] fix(flate): hold a compressed group to the group budget A compressed JSON or opaque track capped each frame at 64 MiB of inflated output and left the reconstructed snapshot unbounded, so a peer's tiny frames could make every reader inflate, parse, and hold far more than a plain group carries. The DEFLATE decoder now holds its whole window to moq-net's group budget (MAX_CACHE_BYTES), the rule moq-net's compressed tracks already follow, and every encoder charges a compressed frame its plaintext so it rolls or refuses before a reader would. This replaces the 4 MiB snapshot default and the MSF frame check: the budget comes from the transport, so no catalog-specific constant and no binding override is needed. A compressed value past 32 MiB now fails with FrameTooLarge like a plain one, so moq-stats' .z flavor ends with its plain sibling. Co-Authored-By: Claude Opus 5.5 --- doc/concept/stats.md | 6 +- js/flate/MIGRATE.md | 6 ++ js/flate/README.md | 2 +- js/flate/src/codec.test.ts | 22 +++--- js/flate/src/codec.ts | 40 ++++------- js/flate/src/index.ts | 14 ++-- js/flate/src/snapshot/producer.ts | 17 ++--- js/flate/src/snapshot/snapshot.test.ts | 16 ++++- js/flate/src/stream/producer.ts | 12 ++-- js/flate/src/stream/stream.test.ts | 25 +++---- js/json/MIGRATE.md | 6 ++ js/json/src/snapshot/encoder.ts | 40 ++++++----- js/json/src/snapshot/snapshot.test.ts | 29 +++++++- js/json/src/stream/encoder.ts | 12 ++-- js/json/src/stream/stream.test.ts | 12 ++++ js/json/src/window/encoder.ts | 24 ++++--- js/json/src/window/window.test.ts | 22 +++--- rs/moq-flate/MIGRATE.md | 7 ++ rs/moq-flate/src/lib.rs | 57 ++++++++-------- rs/moq-flate/src/snapshot/mod.rs | 26 ++++--- rs/moq-flate/src/snapshot/producer.rs | 23 +++---- rs/moq-flate/src/stream/mod.rs | 27 +++++--- rs/moq-flate/src/stream/producer.rs | 21 +++--- rs/moq-json/MIGRATE.md | 7 +- rs/moq-json/src/snapshot/consumer.rs | 32 ++------- rs/moq-json/src/snapshot/decoder.rs | 10 +-- rs/moq-json/src/snapshot/encoder.rs | 95 ++++++++++++++++++++------ rs/moq-json/src/stream/encoder.rs | 13 ++-- rs/moq-json/src/stream/mod.rs | 16 ++++- rs/moq-json/src/window/encoder.rs | 88 ++++++++++++------------ rs/moq-mux/src/catalog/msf/consumer.rs | 47 +++---------- rs/moq-mux/src/catalog/msf/mod.rs | 9 --- rs/moq-stats/MIGRATE.md | 5 ++ rs/moq-stats/src/produce.rs | 51 +++++++------- 34 files changed, 462 insertions(+), 377 deletions(-) diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 1cea657e414..768e8bea138 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -186,8 +186,8 @@ writes only when a track's frame changed, so silence means nothing moved, not that the producer is gone. The track ends when the producer does, or at depth 1 or more when its group's broadcast is unannounced; the group may return later as a new broadcast. A frame too large to cache (32 MB) ends its -track with a `FrameTooLarge` error, while the `.z` flavor, which compresses -under the limit, keeps updating. +track with a `FrameTooLarge` error. The `.z` flavor meets the same limit by its +uncompressed size, since a reader inflates the whole group. ### Totals @@ -260,7 +260,7 @@ real sums. Started minus ended stays exact. A child appears while a broadcast under it has counters, and leaves after the frame carrying its last one; its tail stays in the rollup, and a child that returns starts over. Children are never capped, so roughly ten thousand across four tiers outgrow the 32 MB frame -limit: that encoding's track then ends with `FRAME_TOO_LARGE` instead of going +limit: both encodings' tracks then end with `FRAME_TOO_LARGE` instead of going stale. ## Entries diff --git a/js/flate/MIGRATE.md b/js/flate/MIGRATE.md index 3c9ea9653a1..8a0f7b2c80e 100644 --- a/js/flate/MIGRATE.md +++ b/js/flate/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +`Decoder` holds its whole window, across every frame, to `Group.MAX_GROUP_CACHE_BYTES` (32 MiB) instead of capping each frame at 64 MiB. +`DEFAULT_MAX_FRAME_SIZE`, `DecoderOptions`, and `Decoder.maxFrameSize` are removed; construct with `new Decoder()`. +The producers charge a compressed payload its inflated size: a snapshot value over 32 MiB throws `FrameTooLarge` however small it deflates, and a stream log ends its budget at 32 MiB of inflated payload. + ### @moq/binary is @moq/flate The opaque `Snapshot` and `Stream` tracks moved beside the codec; the wire and the catalog's `binary` section are unchanged. diff --git a/js/flate/README.md b/js/flate/README.md index 4625243096f..ac7d000875a 100644 --- a/js/flate/README.md +++ b/js/flate/README.md @@ -50,4 +50,4 @@ new TextDecoder().decode(decoder.frame(a)); // "the quick brown fox" new TextDecoder().decode(decoder.frame(b)); // "the quick brown dog" ``` -Create a fresh `Encoder`/`Decoder` pair per independent stream (in moq-net terms, per group). `new Encoder({ level })` sets the DEFLATE level (`0..=9`, default `6`); `new Decoder({ maxFrameSize })` caps how far a single frame may inflate (default 64 MiB), rejecting zip bombs. +Create a fresh `Encoder`/`Decoder` pair per independent stream (in moq-net terms, per group). `new Encoder({ level })` sets the DEFLATE level (`0..=9`, default `6`); a `Decoder` holds its whole window to the group budget (32 MiB), rejecting zip bombs: a compressed group never costs a reader more than a plain one. diff --git a/js/flate/src/codec.test.ts b/js/flate/src/codec.test.ts index ff4dad69d48..2930f3c5853 100644 --- a/js/flate/src/codec.test.ts +++ b/js/flate/src/codec.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { Group } from "@moq/net"; import { Deflate, Inflate } from "fflate"; import { Decoder, Encoder } from "./codec.ts"; @@ -64,19 +65,18 @@ test("codec rejects garbage", () => { expect(() => decoder.frame(new Uint8Array(64).fill(0xff))).toThrow(); }); -test("codec rejects a frame that inflates past the cap", () => { - // A tiny slice can inflate enormously, so the decoder bounds the output as it is produced. The - // cap's value doesn't change that path, so use a small one instead of paying for a real bomb. - const slice = new Encoder().frame(new Uint8Array(1024)); - const decoder = new Decoder({ maxFrameSize: 512 }); - expect(() => decoder.frame(slice)).toThrow(/exceeded 512/); -}); +test("the decoder holds its whole window to the group budget", () => { + // Each frame fits on its own and deflates to almost nothing; together they inflate past what a + // plain group can carry. + const half = new Uint8Array(Group.MAX_GROUP_CACHE_BYTES / 2 + 1); + const encoder = new Encoder(); + const first = encoder.frame(half); + const second = encoder.frame(half); + expect(second.length).toBeLessThan(64 * 1024); -test("a decoder's frame cap cannot be lifted after construction", () => { const decoder = new Decoder(); - expect(() => { - (decoder as { maxFrameSize: number }).maxFrameSize = Infinity; - }).toThrow(TypeError); + expect(decoder.frame(first).length).toBe(half.length); + expect(() => decoder.frame(second)).toThrow(/decompressed group exceeded/); }); test("a frame larger than pako's chunk size round-trips", () => { diff --git a/js/flate/src/codec.ts b/js/flate/src/codec.ts index e0a4fb8a3d6..31709e62650 100644 --- a/js/flate/src/codec.ts +++ b/js/flate/src/codec.ts @@ -1,13 +1,11 @@ // The group-scoped DEFLATE codec: see the module docs in index.ts. +import { Group } from "@moq/net"; import * as pako from "pako"; /** The default DEFLATE level ({@link Encoder}): a good size/speed balance for small, repetitive payloads. */ export const DEFAULT_LEVEL = 6; -/** The default per-frame decompressed-size cap ({@link Decoder}): 64 MiB. */ -export const DEFAULT_MAX_FRAME_SIZE = 64 * 1024 * 1024; - // The trailing bytes of a DEFLATE sync flush, stripped on the wire and re-appended to decode. const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); @@ -92,42 +90,33 @@ export class Encoder { } } -/** Options for a {@link Decoder}. */ -export interface DecoderOptions { - /** - * Per-frame decompressed-size cap. A frame that inflates past this is rejected (zip-bomb guard). - * Defaults to {@link DEFAULT_MAX_FRAME_SIZE}. - */ - maxFrameSize?: number; -} - /** * Decodes a stream's frame slices back into the original payloads. Hold one per stream; feed slices * in frame order (each frame builds on the earlier ones). * + * A peer could send tiny slices that inflate hugely, so the window's total output is held to the + * group budget, `Group.MAX_GROUP_CACHE_BYTES`: the most a plain group can carry. A compressed group + * then costs a reader no more than a plain one, however small its slices. + * * @public */ export class Decoder { #inflate = new pako.Inflate({ raw: true }); #chunks: Uint8Array[] = []; #total = 0; + // Bytes the whole window has inflated to, across every frame. + #inflated = 0; #tooLarge = false; - #maxFrameSize: number; - - /** The per-frame decompressed-size cap in bytes, in effect for this decoder. */ - get maxFrameSize(): number { - return this.#maxFrameSize; - } /** Start a fresh decoder with a cold window. */ - constructor(options: DecoderOptions = {}) { - this.#maxFrameSize = options.maxFrameSize ?? DEFAULT_MAX_FRAME_SIZE; + constructor() { this.#inflate.onData = (chunk) => { const bytes = chunk as Uint8Array; this.#total += bytes.length; + this.#inflated += bytes.length; // Bound the inflated output as it is produced; a tiny slice can expand enormously. Stop - // retaining past the cap, then reject once the push returns. - if (this.#total > this.#maxFrameSize) { + // retaining past the budget, then reject once the push returns. + if (this.#inflated > Group.MAX_GROUP_CACHE_BYTES) { this.#tooLarge = true; return; } @@ -137,21 +126,22 @@ export class Decoder { /** * Decompress the next frame's `slice` back into its payload. Empty in yields empty out. Throws if - * the input is malformed or inflates past the per-frame size cap. + * the input is malformed, or once the window's output across every frame so far passes the group + * budget. */ frame(slice: Uint8Array): Uint8Array { if (slice.length === 0) return slice; + if (this.#tooLarge) throw new Error(`decompressed group exceeded ${Group.MAX_GROUP_CACHE_BYTES} bytes`); this.#chunks = []; this.#total = 0; - this.#tooLarge = false; // Feed the slice then the re-appended sync-flush marker as two pushes, so no combined buffer is // allocated. The marker delimits the frame and flushes its last bytes out of the inflate buffer. this.#inflate.push(slice, false); this.#inflate.push(SYNC_FLUSH_TAIL, pako.Z_SYNC_FLUSH); if (this.#inflate.err) throw new Error(`decompression failed: ${this.#inflate.msg}`); - if (this.#tooLarge) throw new Error(`decompressed frame exceeded ${this.#maxFrameSize} bytes`); + if (this.#tooLarge) throw new Error(`decompressed group exceeded ${Group.MAX_GROUP_CACHE_BYTES} bytes`); return concat(this.#chunks, this.#total); } diff --git a/js/flate/src/index.ts b/js/flate/src/index.ts index bb5cc8cb1ad..518f6f77f35 100644 --- a/js/flate/src/index.ts +++ b/js/flate/src/index.ts @@ -34,20 +34,14 @@ * A sync flush always ends in the fixed 4-byte marker `00 00 ff ff`. {@link Encoder.frame} drops it * and {@link Decoder.frame} re-appends it, saving 4 bytes per frame, the same trick * [RFC 7692](https://www.rfc-editor.org/rfc/rfc7692.html#section-7.2.1) (permessage-deflate) uses. A - * small slice can still inflate enormously, so {@link Decoder.frame} caps the inflated output as it - * is produced. pako is synchronous, so the whole codec is synchronous. + * small slice can still inflate enormously, so a {@link Decoder} holds its whole window to the group + * budget as output is produced: a compressed group never costs a reader more than a plain one, and + * the producers refuse before a group would. pako is synchronous, so the whole codec is synchronous. * * @module */ -export { - DEFAULT_LEVEL, - DEFAULT_MAX_FRAME_SIZE, - Decoder, - type DecoderOptions, - Encoder, - type EncoderOptions, -} from "./codec.ts"; +export { DEFAULT_LEVEL, Decoder, Encoder, type EncoderOptions } from "./codec.ts"; export type { Compression } from "./compression.ts"; /** Lossy latest-value opaque publishing. */ export * as Snapshot from "./snapshot/index.ts"; diff --git a/js/flate/src/snapshot/producer.ts b/js/flate/src/snapshot/producer.ts index e4ff4b9fb8e..c9c8e6250c5 100644 --- a/js/flate/src/snapshot/producer.ts +++ b/js/flate/src/snapshot/producer.ts @@ -1,6 +1,6 @@ import type { Timed } from "@moq/net"; import * as Moq from "@moq/net"; -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "../codec.ts"; +import { Encoder as Flate } from "../codec.ts"; import { type Compression, isDeflate } from "../compression.ts"; @@ -43,19 +43,16 @@ export class Producer { * untimed track: it is never filled in. */ update({ value: payload, at }: Timed): void { - // Consumers all decode with `@moq/flate`'s default cap, so publishing past it would advertise - // a value that always fails to read. Rejected before anything is published; unlike a stream - // this is not terminal, since the previous value still stands and the next update supersedes. - if (this.#compress && payload.byteLength > DEFAULT_MAX_FRAME_SIZE) { - throw new Error(`payload larger than the decoder's ${DEFAULT_MAX_FRAME_SIZE} byte limit`); - } + // Check before opening a group. `appendGroup` publishes immediately, so letting `writeFrame` + // reject the frame would leave an empty newest group behind: a snapshot consumer jumps to the + // newest, so the previous value would be lost even though this update threw. A compressed value + // is held to the same budget by its inflated size, since that is what every consumer's decoder + // allows: compression saves bandwidth, never what a reader holds. + if (payload.byteLength > Moq.Group.MAX_GROUP_CACHE_BYTES) throw new Moq.Error.FrameTooLarge(); // One frame per group, so the window spans a single value and starts cold every time. const encoded = this.#compress ? new Flate().frame(payload) : payload; - // Check before opening a group. `appendGroup` publishes immediately, so letting `writeFrame` - // reject the frame would leave an empty newest group behind: a snapshot consumer jumps to the - // newest, so the previous value would be lost even though this update threw. if (encoded.byteLength > Moq.Group.MAX_GROUP_CACHE_BYTES) throw new Moq.Error.FrameTooLarge(); const group = this.#track.appendGroup(); diff --git a/js/flate/src/snapshot/snapshot.test.ts b/js/flate/src/snapshot/snapshot.test.ts index 2ea8ba6170e..842c459a511 100644 --- a/js/flate/src/snapshot/snapshot.test.ts +++ b/js/flate/src/snapshot/snapshot.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; -import { Time, Track } from "@moq/net"; +import { Group, Error as NetError, Time, Track } from "@moq/net"; import { Consumer, Producer } from "./index.ts"; const bytes = (...values: number[]) => new Uint8Array(values); @@ -85,6 +85,20 @@ test("compression shrinks the frame on the wire", async () => { expect(frame?.payload.byteLength).toBeLessThan(payload.byteLength / 4); }); +// Compression doesn't lift the group limit: a reader inflates the whole value, so one past the budget +// is refused however small it deflates, and the previous value stays readable. +test("a compressed value past the group budget is refused", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: "deflate" }); + producer.update({ value: bytes(1) }); + expect(() => producer.update({ value: new Uint8Array(Group.MAX_GROUP_CACHE_BYTES + 1) })).toThrow( + NetError.FrameTooLarge, + ); + producer.finish(); + + expect(await drain(track.subscribe(), true)).toEqual([bytes(1)]); +}); + test("a finished track ends the consumer", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); diff --git a/js/flate/src/stream/producer.ts b/js/flate/src/stream/producer.ts index fabc92e2a2b..fb023d26d98 100644 --- a/js/flate/src/stream/producer.ts +++ b/js/flate/src/stream/producer.ts @@ -25,8 +25,9 @@ export class Producer { #flate?: Flate; // The single group carrying the whole log, opened on the first append and never rolled. #group?: Moq.Group.Producer; - // Frames and payload bytes written to the group, checked against the group budget before each - // payload is encoded. + // Frames and bytes charged to the group, checked against the group budget before each payload is + // encoded. A compressed payload is charged its inflated size (or its slice, if larger), the budget + // every consumer's `Decoder` holds its window to. #frames = 0; #bytes = 0; @@ -60,9 +61,8 @@ export class Producer { // Check before compressing: encoding advances the window, so a payload refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. Also checked before the group is opened, so a refused - // first payload publishes nothing. The budget is below the decoder's cap, so any payload that - // fits is one every consumer can inflate. + // actual size for the same reason, and it covers the inflated size too. Also checked before + // the group is opened, so a refused first payload publishes nothing. const bound = this.#flate ? Flate.bound(payload.byteLength) : payload.byteLength; if (this.#frames >= Group.MAX_GROUP_FRAMES || this.#bytes + bound > Group.MAX_GROUP_CACHE_BYTES) { throw new NetError.GroupTooLarge(); @@ -95,7 +95,7 @@ export class Producer { } this.#frames++; - this.#bytes += encoded.byteLength; + this.#bytes += Math.max(payload.byteLength, encoded.byteLength); } /** Finish the track, closing the group. */ diff --git a/js/flate/src/stream/stream.test.ts b/js/flate/src/stream/stream.test.ts index 22d3d438edc..e3983a7e07e 100644 --- a/js/flate/src/stream/stream.test.ts +++ b/js/flate/src/stream/stream.test.ts @@ -1,7 +1,6 @@ import { expect, spyOn, test } from "bun:test"; import { Milli } from "@moq/clock"; import { Group, Error as NetError, Time, Track } from "@moq/net"; -import { DEFAULT_MAX_FRAME_SIZE } from "../codec.ts"; import { Consumer, Producer, Rolled } from "./index.ts"; // Ask for a replay window, so the superseded first group is delivered rather than skipped by the @@ -173,18 +172,14 @@ test("an ended log refuses with its own error", () => { }); // A payload past the group budget is refused before anything is written, so the log carries on: the -// next payload lands and decodes, which with compression proves the window never moved. One past the -// decoder's cap is refused the same way rather than ending the track. +// next payload lands and decodes, which with compression proves the window never moved. for (const compression of [false, true]) { test(`an oversized payload is refused and the log continues (compression=${compression})`, async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, compression: compression ? "deflate" : "none" }); producer.append({ value: payloads(1)[0] }); - const huge = new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1); - expect(() => producer.append({ value: huge.subarray(0, Group.MAX_GROUP_CACHE_BYTES) })).toThrow( - NetError.GroupTooLarge, - ); + const huge = new Uint8Array(Group.MAX_GROUP_CACHE_BYTES); expect(() => producer.append({ value: huge })).toThrow(NetError.GroupTooLarge); producer.append({ value: payloads(2)[1] }); @@ -193,16 +188,22 @@ for (const compression of [false, true]) { }); } -// The budget check stands in for the decoder's cap: a payload that fits is one every consumer can -// inflate. -test("the group budget is within the decoder's cap", () => { - expect(Group.MAX_GROUP_CACHE_BYTES).toBeLessThanOrEqual(DEFAULT_MAX_FRAME_SIZE); +// A compressed payload is charged its inflated size, since that is what every reader's decoder holds +// the group to: two payloads that each fit, and deflate to almost nothing, still overflow together. +test("a compressed log is charged its inflated size", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track, compression: "deflate" }); + const half = new Uint8Array(Group.MAX_GROUP_CACHE_BYTES / 2 + 1); + producer.append({ value: half }); + expect(() => producer.append({ value: half })).toThrow(NetError.GroupTooLarge); + producer.finish(); + expect(await drain(track.subscribe(), true)).toEqual([half]); }); test("a refused first payload publishes nothing", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, compression: "deflate" }); - expect(() => producer.append({ value: new Uint8Array(DEFAULT_MAX_FRAME_SIZE + 1) })).toThrow( + expect(() => producer.append({ value: new Uint8Array(Group.MAX_GROUP_CACHE_BYTES + 1) })).toThrow( NetError.GroupTooLarge, ); producer.finish(); diff --git a/js/json/MIGRATE.md b/js/json/MIGRATE.md index 3bf664a5d5e..ee2232e3912 100644 --- a/js/json/MIGRATE.md +++ b/js/json/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +A compressed group may inflate to at most `Group.MAX_GROUP_CACHE_BYTES` (32 MiB), the most a plain group carries. +The encoders charge each frame its plaintext: a compressed snapshot over 32 MiB throws `FrameTooLarge` (it used to pass up to 64 MiB), a window header over it throws, and deltas and ops roll a new group once a group's plaintext would pass the budget. +A compressed stream's budget is 32 MiB of record plaintext. + ### Duplicate keys are refused Every decoder throws `DuplicateKey` for a frame in which a JSON object repeats a key, where `JSON.parse` used to keep the last copy. diff --git a/js/json/src/snapshot/encoder.ts b/js/json/src/snapshot/encoder.ts index cf0fe652978..02e0a687e45 100644 --- a/js/json/src/snapshot/encoder.ts +++ b/js/json/src/snapshot/encoder.ts @@ -1,5 +1,5 @@ -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "@moq/flate"; -import { Group } from "@moq/net"; +import { Encoder as Flate } from "@moq/flate"; +import { Group, Error as NetError } from "@moq/net"; import type * as z from "@zod/mini"; import { type Compression, isDeflate } from "../compression.ts"; @@ -128,6 +128,9 @@ export class Encoder { #snapshotLen = 0; // Frames emitted into the current group, snapshot included. #groupFrames = 0; + // Bytes charged to the current group's budget: each frame's slice, or its plaintext when that is + // larger, since a reader's DEFLATE window is held to the same budget. + #groupBytes = 0; // The current group's `deflate-raw` stream, swapped for a fresh one (cold window) at each // snapshot, so a snapshot and its deltas share one stream. @@ -177,6 +180,7 @@ export class Encoder { this.#deltaBytes = 0; this.#snapshotLen = 0; this.#groupFrames = 0; + this.#groupBytes = 0; this.#resync = true; this.#pending = false; } @@ -221,16 +225,18 @@ export class Encoder { // A delta is only readable while the group still holds the snapshot it applies to. // Admitting a patch that pushes the group past that budget would abort it - // (`GroupTooLarge`), leaving a late subscriber with no value, so fall through to a - // fresh snapshot instead. + // (`GroupTooLarge`), or past a reader's inflate budget, leaving a late subscriber with no + // value, so fall through to a fresh snapshot instead. // - // Measured on the encoded payload rather than the plaintext: a sync-flushed DEFLATE frame - // can come out slightly larger than its input, so the plaintext is not an upper bound. - // Compressing first advances the window, but `#snapshot` opens a fresh one, so an + // Charged the larger of the encoded payload and the plaintext: a sync-flushed DEFLATE frame + // can come out slightly larger than its input, while the reader's window counts what it + // inflates. Compressing first advances the window, but `#snapshot` opens a fresh one, so an // over-budget delta costs only the wasted compression. - if (this.#snapshotLen + this.#deltaBytes + payload.length <= this.#maxGroupBytes) { + const charge = Math.max(payload.length, delta.length); + if (this.#groupBytes + charge <= this.#maxGroupBytes) { this.#last = json; this.#deltaBytes += payload.length; + this.#groupBytes += charge; this.#groupFrames += 1; return this.#pend({ payload, keyframe: false }); } @@ -286,19 +292,13 @@ export class Encoder { return new TextEncoder().encode(JSON.stringify(result.patch)); } - // Reject plaintext the consumer's decoder could never reproduce. Every consumer decodes with - // `@moq/flate`'s default output cap, so a frame past it would be unreadable however small it - // compresses to. Checked before anything is published or the window advances. - #checkDecodable(frame: Uint8Array): void { - if (this.#compress && frame.byteLength > DEFAULT_MAX_FRAME_SIZE) { - throw new Error(`value larger than the decoder's ${DEFAULT_MAX_FRAME_SIZE} byte limit`); - } - } - // Encode a group's snapshot (frame 0), returning its payload. On the compressed path this opens a // fresh stream (cold window), so the snapshot and its deltas share one DEFLATE window. #snapshot(frame: Uint8Array): Uint8Array { - this.#checkDecodable(frame); + // Every consumer holds a compressed group to the group budget by its inflated size, so a value + // past it would be unreadable however small it compresses to: the same `FrameTooLarge` a plain + // one meets. Checked before anything is published or the window advances. + if (this.#compress && frame.byteLength > this.#maxGroupBytes) throw new NetError.FrameTooLarge(); // Build the window locally and install it only once framing succeeds. Assigning it first would // leave a throw with a cold window in place, no resync pending, and the old group counters @@ -311,6 +311,7 @@ export class Encoder { this.#snapshotLen = payload.length; this.#deltaBytes = 0; this.#groupFrames = 1; + this.#groupBytes = Math.max(payload.length, frame.length); this.#resync = false; return payload; @@ -318,9 +319,6 @@ export class Encoder { // Compress a frame into the current group's window, or pass it through when uncompressed. #frame(frame: Uint8Array): Uint8Array { - // A delta is worse than a snapshot here: there is no keyframe after it to resynchronize on, - // so one the consumer cannot decode makes the rest of the group unreadable. - this.#checkDecodable(frame); return this.#flate ? this.#flate.frame(frame) : frame; } } diff --git a/js/json/src/snapshot/snapshot.test.ts b/js/json/src/snapshot/snapshot.test.ts index 34431979ab8..a5ff68ea2cd 100644 --- a/js/json/src/snapshot/snapshot.test.ts +++ b/js/json/src/snapshot/snapshot.test.ts @@ -402,6 +402,30 @@ test("a delta that would overflow the snapshot rolls a new one instead", async ( expect(values[values.length - 1]).toEqual({ v: "y".repeat(size) }); }); +// A compressed delta is charged its plaintext too: the reader's window counts what it inflates, so a +// patch that deflates to almost nothing still rolls the group once its plaintext would overflow the +// budget. Matches the Rust `a_compressed_delta_is_gated_on_its_inflated_size`. +test("a compressed delta is gated on its inflated size", () => { + const value = { v: "x".repeat(1000) }; + const patched = { v: "y".repeat(1000) }; + const encoder = new Encoder({ + compression: "deflate", + deltaRatio: 100, + maxGroupBytes: JSON.stringify(value).length + 500, + }); + encoder.update(value)?.commit(); + const rolled = encoder.update(patched); + expect(rolled?.keyframe).toBe(true); + expect(rolled?.payload.length).toBeLessThan(500); +}); + +// A compressed snapshot is refused by its plaintext, the way a plain one is, however small it deflates: +// no reader could inflate it. +test("a compressed snapshot past the budget is refused", () => { + const encoder = new Encoder({ compression: "deflate", deltaRatio: 0, maxGroupBytes: 100 }); + expect(() => encoder.update({ v: "x".repeat(1000) })).toThrow(NetError.FrameTooLarge); +}); + test("a compressed delta is gated on its encoded size, not its plaintext", async () => { // A sync-flushed DEFLATE frame can come out larger than its input, so the plaintext is not an // upper bound on what lands in the group. A patch that fits the budget by its plaintext but not @@ -418,8 +442,9 @@ test("a compressed delta is gated on its encoded size, not its plaintext", async expect(delta?.keyframe).toBe(false); expect(delta?.payload.length).toBeGreaterThan(plaintext); - // A budget with room for the plaintext patch but not the encoded one. - const maxGroupBytes = (snapshot?.payload.length ?? 0) + plaintext; + // A budget with room for the plaintext patch but not the encoded one. The snapshot is charged its + // plaintext, which outweighs its slice. + const maxGroupBytes = JSON.stringify(value).length + plaintext; const track = new Track.Producer("test"); const producer = new Producer({ track, compression: "deflate", maxGroupBytes }); producer.update({ value: value }); diff --git a/js/json/src/stream/encoder.ts b/js/json/src/stream/encoder.ts index 58bcd352323..034eac3ded7 100644 --- a/js/json/src/stream/encoder.ts +++ b/js/json/src/stream/encoder.ts @@ -66,9 +66,10 @@ export class Encoder { // Bumped for each record handed out, so a commit that arrives after the encoder has moved on can // tell that it is acknowledging a record that is no longer the outstanding one. #generation = 0; - // Frames and payload bytes committed to the current group, checked against the group budget - // before each record is encoded. Replaced on reset, so a commit that lands after one charges the - // old group rather than the new. + // Frames and bytes committed to the current group, checked against the group budget before each + // record is encoded. A compressed record is charged its plaintext (or its slice, if larger), since + // a reader's DEFLATE window is held to the same budget. Replaced on reset, so a commit that lands + // after one charges the old group rather than the new. #budget = { frames: 0, bytes: 0 }; constructor(config: Config = {}) { @@ -121,8 +122,7 @@ export class Encoder { // Check before compressing: encoding advances the window, so a record refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. The budget is also below `@moq/flate`'s per-frame decode - // cap, so any record that fits is one every consumer can inflate. + // actual size for the same reason; it also covers the plaintext a reader inflates. const budget = this.#budget; const bound = this.#compress ? Flate.bound(bytes.byteLength) : bytes.byteLength; if (budget.frames >= Group.MAX_GROUP_FRAMES || budget.bytes + bound > Group.MAX_GROUP_CACHE_BYTES) { @@ -142,7 +142,7 @@ export class Encoder { if (!charged) { charged = true; budget.frames++; - budget.bytes += payload.byteLength; + budget.bytes += Math.max(payload.byteLength, bytes.byteLength); } // A caller that starts the next encode before this record settles has already made the // encoder account for it. Acknowledging it now would clear the flag belonging to the newer diff --git a/js/json/src/stream/stream.test.ts b/js/json/src/stream/stream.test.ts index cbcaf38182a..8e2d208e7a5 100644 --- a/js/json/src/stream/stream.test.ts +++ b/js/json/src/stream/stream.test.ts @@ -226,6 +226,18 @@ for (const compression of [false, true]) { }); } +// A compressed record is charged its plaintext, since that is what a reader's window inflates: two +// records that each fit, and deflate to almost nothing, still overflow the budget together. +test("a compressed log is charged its inflated size", async () => { + const track = new Track.Producer("test"); + const producer = new Producer<{ n: number; pad: string }>({ track, compression: "deflate" }); + const pad = "x".repeat(Group.MAX_GROUP_CACHE_BYTES / 2); + producer.append({ value: { n: 0, pad } }); + expect(() => producer.append({ value: { n: 1, pad } })).toThrow(NetError.GroupTooLarge); + producer.finish(); + expect(await drain(track.subscribe(), true)).toEqual([0]); +}); + // The budget covers the whole log, so once its frames are spent every append is refused, and the log // written so far still finishes cleanly and reads back whole. for (const compression of [false, true]) { diff --git a/js/json/src/window/encoder.ts b/js/json/src/window/encoder.ts index fed75c145bb..6216f344ddf 100644 --- a/js/json/src/window/encoder.ts +++ b/js/json/src/window/encoder.ts @@ -1,4 +1,4 @@ -import { DEFAULT_MAX_FRAME_SIZE, Encoder as Flate } from "@moq/flate"; +import { Encoder as Flate } from "@moq/flate"; import { Group } from "@moq/net"; /** Frames (header included) in one group before a new group is forced, matching the Snapshot cap. Kept well below the net per-group cap so a roll always precedes GroupTooLarge. */ @@ -91,6 +91,9 @@ export class Encoder { // Reference size the op budget is measured against: this group's header frame. #headerLen = 0; #groupFrames = 0; + // Bytes charged to this group's budget: each frame's slice, or its plaintext when that is larger, + // since a reader's DEFLATE window is held to the same budget. + #groupBytes = 0; // Whether the next frame must be a header after a lost frame. #resync = true; // Whether the frame from the last push/pop is still unacknowledged. @@ -144,6 +147,7 @@ export class Encoder { this.#opBytes = 0; this.#headerLen = 0; this.#groupFrames = 0; + this.#groupBytes = 0; this.#resync = true; this.#pending = false; this.#generation += 1; @@ -207,17 +211,19 @@ export class Encoder { ); } - /** Compress an already-serialized op into the open group, if its header remains cached. */ + /** + * Compress an already-serialized op into the open group, if its header remains cached and + * inflatable. Charged the larger of the slice and the plaintext: a sync-flushed DEFLATE frame can + * come out slightly larger than its input, while a reader's window counts what it inflates. + */ #emitOp(text: string): Uint8Array | undefined { const bytes = new TextEncoder().encode(text); - if (bytes.length > DEFAULT_MAX_FRAME_SIZE) { - throw new Error("window frame exceeds the decoder's decompressed size limit"); - } const payload = this.#flate ? this.#flate.frame(bytes) : bytes; this.#opBytes += payload.length; this.#groupFrames += 1; - if (this.#headerLen + this.#opBytes > Group.MAX_GROUP_CACHE_BYTES) { + this.#groupBytes += Math.max(payload.length, bytes.length); + if (this.#groupBytes > Group.MAX_GROUP_CACHE_BYTES) { this.#resyncGroup(); return undefined; } @@ -243,8 +249,9 @@ export class Encoder { const checkpoint = start === offset ? "" : `,"start":${start}`; const text = `{"offset":${offset}${checkpoint},"records":[${window.join(",")}]}`; const bytes = new TextEncoder().encode(text); - if (bytes.length > DEFAULT_MAX_FRAME_SIZE) { - throw new Error("window header exceeds the decoder's decompressed size limit"); + // A reader inflates the header whole, so its plaintext meets the group budget too. + if (bytes.length > Group.MAX_GROUP_CACHE_BYTES) { + throw new Error("window header exceeds the group cache limit"); } // A fresh per-group encoder (cold window), with the header as frame 0. @@ -258,6 +265,7 @@ export class Encoder { this.#headerLen = payload.length; this.#opBytes = 0; this.#groupFrames = 1; + this.#groupBytes = Math.max(payload.length, bytes.length); this.#resync = false; return this.#pendingFrame(payload, true, edit); diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index 9afc87b48dd..b2d91c4db10 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -407,15 +407,19 @@ test("an op that would evict the header rolls first", () => { frame.commit(); }); -test("a header larger than the group cache is rejected", () => { - const encoder = new Encoder(); - - expect(() => encoder.push("x".repeat(Group.MAX_GROUP_CACHE_BYTES))).toThrow("group cache limit"); - expect(encoder.window).toEqual([]); - - const frame = encoder.push("ok"); - expect(frame.keyframe).toBeTrue(); -}); +// Compression doesn't lift the limit: a reader inflates the header whole, so one past the group budget +// is refused however small it deflates. +for (const compression of [false, true]) { + test(`a header larger than the group cache is rejected (compression=${compression})`, () => { + const encoder = new Encoder({ compression }); + + expect(() => encoder.push("x".repeat(Group.MAX_GROUP_CACHE_BYTES))).toThrow("group cache limit"); + expect(encoder.window).toEqual([]); + + const frame = encoder.push("ok"); + expect(frame.keyframe).toBeTrue(); + }); +} test("an uncommitted edit leaves the window unchanged", () => { const encoder = new Encoder(); diff --git a/rs/moq-flate/MIGRATE.md b/rs/moq-flate/MIGRATE.md index 97a02c1bc80..d9ec3809e96 100644 --- a/rs/moq-flate/MIGRATE.md +++ b/rs/moq-flate/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A compressed group is held to the group budget + +`Decoder` holds its whole window, across every frame, to `moq_net::group::MAX_CACHE_BYTES` (32 MiB), the most a plain group carries, instead of capping each frame at 64 MiB. +`DEFAULT_MAX_FRAME_SIZE` and `Decoder::with_max_frame_size` are removed; there is no other cap to set. +The producers charge a compressed payload its inflated size: a snapshot value over 32 MiB fails with `moq_net::Error::FrameTooLarge` however small it deflates, and a stream log ends its budget at 32 MiB of inflated payload. +Split a larger value across tracks. + ### Consumers return `Timed` (#5677) Snapshot and stream `next` / `poll_next` return `moq_net::Timed`: read `.value` for the payload and `.at` for the frame's exact timestamp, `None` when untimed. diff --git a/rs/moq-flate/src/lib.rs b/rs/moq-flate/src/lib.rs index 1c17293eef9..bf96de72da8 100644 --- a/rs/moq-flate/src/lib.rs +++ b/rs/moq-flate/src/lib.rs @@ -31,7 +31,9 @@ //! This is plain raw DEFLATE with a `Z_SYNC_FLUSH` after each frame, so any peer using the same //! primitive (zlib's sync flush, the browser's `deflate-raw`) interoperates on the wire. There is no //! length prefix: the caller is expected to frame each slice (moq-net already does). A small slice -//! can still inflate to far more than its own size, so [`Decoder::frame`] bounds each frame's output. +//! can still inflate to far more than its own size, so a [`Decoder`] holds its whole window to the +//! group budget ([`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES)): a compressed group never +//! costs a reader more than a plain one, and the encoders refuse or roll before a group would. //! //! A sync flush always ends in the 4-byte empty-block marker `00 00 ff ff`. That marker is fixed, so //! [`Encoder::frame`] drops it from each slice and [`Decoder::frame`] re-appends it before inflating, @@ -67,9 +69,6 @@ use flate2::{Compress, Decompress, FlushCompress, FlushDecompress, Status}; /// the small, repetitive payloads this targets. pub const DEFAULT_LEVEL: u32 = 6; -/// The default per-frame decompressed-size cap ([`Decoder::new`]): 64 MiB. -pub const DEFAULT_MAX_FRAME_SIZE: u64 = 64 * 1024 * 1024; - /// The trailing bytes of a DEFLATE sync flush, stripped on the wire and re-appended to decode. const SYNC_FLUSH_TAIL: [u8; 4] = [0x00, 0x00, 0xff, 0xff]; @@ -101,8 +100,8 @@ pub enum Error { #[error("decompression failed")] Decompress, - /// A frame's decompressed size exceeded the configured limit (zip-bomb guard). - #[error("decompressed frame exceeded {0} bytes")] + /// A group inflated past the group budget, which no conforming encoder produces (zip-bomb guard). + #[error("decompressed group exceeded {0} bytes")] TooLarge(u64), /// An error from the underlying track. @@ -198,34 +197,31 @@ impl Default for Encoder { /// Decodes a stream's frame slices back into the original payloads. Hold one per stream; feed slices /// in frame order (each frame builds on the earlier ones). +/// +/// A peer could send tiny slices that inflate hugely, so the window's total output is held to the +/// group budget, [`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES): the most a plain group can +/// carry. A compressed group then costs a reader no more than a plain one, however small its slices. pub struct Decoder { inner: Box, - max_frame_size: u64, + /// The most bytes the whole window may inflate to. A field so a test can shrink it. + budget: u64, } impl Decoder { - /// Start a fresh decoder with a cold window and the [`DEFAULT_MAX_FRAME_SIZE`] cap. + /// Start a fresh decoder with a cold window. pub fn new() -> Self { - Self::with_max_frame_size(DEFAULT_MAX_FRAME_SIZE) - } - - /// Start a fresh decoder with a cold window and a custom per-frame decompressed-size cap. - /// - /// A malicious or buggy peer could send a tiny slice that inflates hugely, so [`frame`](Self::frame) - /// stops and returns [`Error::TooLarge`] once a single frame's output would exceed `max_frame_size`. - pub fn with_max_frame_size(max_frame_size: u64) -> Self { // `false`: raw DEFLATE, matching the encoder. Self { inner: Box::new(Decompress::new(false)), - max_frame_size, + budget: moq_net::group::MAX_CACHE_BYTES, } } /// Decompress the next frame's `slice` back into its payload. /// - /// An empty slice yields an empty payload. Returns [`Error::TooLarge`] if the frame inflates past - /// the configured cap (checked as output is produced, not from any declared size), and - /// [`Error::Decompress`] on malformed input. + /// An empty slice yields an empty payload. Returns [`Error::TooLarge`] once the window's output, + /// counted across every frame so far, passes the group budget (checked as output is produced, not + /// from any declared size), and [`Error::Decompress`] on malformed input. pub fn frame(&mut self, slice: &[u8]) -> Result { let mut out = Vec::new(); self.frame_into(slice, &mut out)?; @@ -255,9 +251,9 @@ impl Decoder { .map_err(|_| Error::Decompress)?; let consumed = (self.inner.total_in() - before_in) as usize; let produced = (self.inner.total_out() - before_out) as usize; - // Bound the inflated output as it is produced; a tiny slice can expand enormously. - if out.len() as u64 + produced as u64 > self.max_frame_size { - return Err(Error::TooLarge(self.max_frame_size)); + // Bound the window's inflated output as it is produced; a tiny slice can expand enormously. + if self.inner.total_out() > self.budget { + return Err(Error::TooLarge(self.budget)); } out.extend_from_slice(&tmp[..produced]); input = &input[consumed..]; @@ -425,13 +421,16 @@ mod test { } #[test] - fn enforces_max_frame_size() { - // A tiny slice of a highly compressible payload inflates past a small cap. - let payload = vec![0u8; 1024]; - let slice = Encoder::new().frame(&payload); + fn the_budget_spans_the_whole_window() { + // Each frame fits on its own; together they inflate past the budget. + let mut enc = Encoder::new(); + let first = enc.frame(&[0u8; 1024]); + let second = enc.frame(&[0u8; 1024]); - let mut dec = Decoder::with_max_frame_size(512); - assert!(matches!(dec.frame(&slice), Err(Error::TooLarge(512)))); + let mut dec = Decoder::new(); + dec.budget = 1536; + dec.frame(&first).unwrap(); + assert!(matches!(dec.frame(&second), Err(Error::TooLarge(1536)))); } #[test] diff --git a/rs/moq-flate/src/snapshot/mod.rs b/rs/moq-flate/src/snapshot/mod.rs index 055ea9eb446..39ed7583a47 100644 --- a/rs/moq-flate/src/snapshot/mod.rs +++ b/rs/moq-flate/src/snapshot/mod.rs @@ -267,18 +267,24 @@ mod test { /// `append_group` publishes immediately, so rejecting the frame inside `write_frame` would leave /// an empty newest group behind. A snapshot consumer jumps to the newest, so the previous value - /// would vanish even though the update reported an error. + /// would vanish even though the update reported an error. Compression doesn't lift the limit: a + /// value that inflates past the group budget is refused even though it deflates to a few KiB. #[test] fn a_rejected_update_leaves_the_previous_value_readable() { - let (mut producer, track) = producer(false); - producer.update(&b"keep"[..]).unwrap(); - - let oversized = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); - assert!(producer.update(oversized).is_err()); - producer.finish().unwrap(); - - // A reader arriving now still finds the last good value, not an empty superseding group. - assert_eq!(drain(consume(track, false)), vec![Bytes::from_static(b"keep")]); + for compression in [false, true] { + let (mut producer, track) = producer(compression); + producer.update(&b"keep"[..]).unwrap(); + + let oversized = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); + assert!(matches!( + producer.update(oversized), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + )); + producer.finish().unwrap(); + + // A reader arriving now still finds the last good value, not an empty superseding group. + assert_eq!(drain(consume(track, compression)), vec![Bytes::from_static(b"keep")]); + } } /// `finish` closes the underlying track, so a later update fails rather than being silently diff --git a/rs/moq-flate/src/snapshot/producer.rs b/rs/moq-flate/src/snapshot/producer.rs index b2dcd80f075..5515d4df39a 100644 --- a/rs/moq-flate/src/snapshot/producer.rs +++ b/rs/moq-flate/src/snapshot/producer.rs @@ -71,23 +71,20 @@ impl Inner { let timestamp = payload.at; let payload = payload.value; + // Check before opening a group. `append_group` publishes immediately, so discovering the limit + // inside `write_frame` would leave an empty newest group behind: a snapshot consumer jumps to + // the newest, so the previous value would be lost even though this update reported an error. + // A compressed value is held to the same budget by its inflated size, since that is what every + // consumer's decoder allows: compression saves bandwidth, never what a reader holds. + if payload.len() as u64 > moq_net::group::MAX_CACHE_BYTES { + return Err(moq_net::Error::FrameTooLarge.into()); + } + // One frame per group, so the window spans a single value and starts cold every time. let payload = match self.compression { - true => { - // Compression can take a large value under the group's frame limit, but every consumer - // decodes with the default output cap, so publishing past it would advertise a - // value that always fails to read. Reject it here instead. - if payload.len() as u64 > crate::DEFAULT_MAX_FRAME_SIZE { - return Err(crate::Error::TooLarge(crate::DEFAULT_MAX_FRAME_SIZE)); - } - crate::Encoder::new().frame(&payload) - } + true => crate::Encoder::new().frame(&payload), false => payload, }; - - // Check before opening a group. `append_group` publishes immediately, so discovering the limit - // inside `write_frame` would leave an empty newest group behind: a snapshot consumer jumps to - // the newest, so the previous value would be lost even though this update reported an error. if payload.len() as u64 > moq_net::group::MAX_CACHE_BYTES { return Err(moq_net::Error::FrameTooLarge.into()); } diff --git a/rs/moq-flate/src/stream/mod.rs b/rs/moq-flate/src/stream/mod.rs index 96fbea32c69..01ad81bec98 100644 --- a/rs/moq-flate/src/stream/mod.rs +++ b/rs/moq-flate/src/stream/mod.rs @@ -22,9 +22,9 @@ //! Instead an append that might not fit is refused with [`moq_net::Error::GroupTooLarge`] before //! it is encoded, and the log stays intact and writable. With compression the check counts the //! payload's raw size plus DEFLATE's worst-case overhead ([`Encoder::bound`](crate::Encoder::bound)), -//! since the compressed size is only known once the window has moved. The budget is below the -//! decoder's [`DEFAULT_MAX_FRAME_SIZE`](crate::DEFAULT_MAX_FRAME_SIZE), so any payload that fits is -//! one every consumer can inflate. The budget covers the whole log rather than a single payload, so +//! since the compressed size is only known once the window has moved, and each written payload is +//! charged its inflated size: the [`Decoder`](crate::Decoder) holds its window to the same budget, so +//! compression shrinks the wire but not how much a log can hold. The budget covers the whole log rather than a single payload, so //! once it is spent every append is refused: the live stream is bounded history by design, and a //! publisher with more to say opens a new track. @@ -290,11 +290,10 @@ mod test { /// A payload past the group budget is refused before anything is written, so the log carries /// on: the next payload lands and decodes, which with compression proves the window never - /// moved. One past the decoder's cap is refused the same way rather than ending the track. + /// moved. #[test] fn an_oversized_payload_is_refused_and_the_log_continues() { - let huge = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); - let past_budget = huge.slice(..moq_net::group::MAX_CACHE_BYTES as usize); + let past_budget = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize]); for compression in [false, true] { let (mut producer, track) = producer(compression); @@ -304,7 +303,6 @@ mod test { refused(producer.append(past_budget.clone())), "compression={compression}" ); - assert!(refused(producer.append(huge.clone())), "compression={compression}"); assert!(producer.demand().wanted().is_some(), "a refusal must not end the track"); producer.append(&b"second"[..]).unwrap(); @@ -322,7 +320,7 @@ mod test { #[test] fn a_refused_first_payload_publishes_nothing() { let (mut producer, mut track) = producer(true); - let huge = Bytes::from(vec![0u8; crate::DEFAULT_MAX_FRAME_SIZE as usize + 1]); + let huge = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize + 1]); assert!(refused(producer.append(huge))); assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), @@ -330,6 +328,19 @@ mod test { ); } + /// A compressed payload is charged its inflated size, since that is what every reader's decoder + /// holds the group to: two payloads that each fit, and deflate to almost nothing, still overflow + /// the budget together. + #[test] + fn a_compressed_log_is_charged_its_inflated_size() { + let half = Bytes::from(vec![0u8; moq_net::group::MAX_CACHE_BYTES as usize / 2 + 1]); + let (mut producer, track) = producer(true); + assert!(producer.append(half.clone()).unwrap() < 64 * 1024); + assert!(refused(producer.append(half.clone()))); + producer.finish().unwrap(); + assert_eq!(drain(track, true), vec![half]); + } + /// The budget covers the whole log, so once its frames are spent every append is refused, and /// the log written so far still finishes cleanly and reads back whole. #[test] diff --git a/rs/moq-flate/src/stream/producer.rs b/rs/moq-flate/src/stream/producer.rs index 37fefe79c8e..678a0035c5a 100644 --- a/rs/moq-flate/src/stream/producer.rs +++ b/rs/moq-flate/src/stream/producer.rs @@ -84,16 +84,13 @@ struct Inner { /// The DEFLATE encoder, one window for the whole group, `Some` while compressing. flate: Option, - /// Frames and payload bytes written to the group, checked against moq-net's group budget before - /// each payload is encoded. + /// Frames and bytes charged to the group, checked against moq-net's group budget before each + /// payload is encoded. A compressed payload is charged its inflated size (or its slice, if + /// larger), the budget every consumer's [`Decoder`](crate::Decoder) holds its window to. frames: usize, bytes: u64, } -// The budget check also stands in for the decoder's cap: a payload that fits the group is one every -// consumer can inflate. -const _: () = assert!(moq_net::group::MAX_CACHE_BYTES <= crate::DEFAULT_MAX_FRAME_SIZE); - impl Inner { fn append(&mut self, payload: Timed) -> Result { let timestamp = payload.at; @@ -113,13 +110,13 @@ impl Inner { // Check before compressing: encoding advances the window, so a payload refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. Also checked before the group is opened, so a refused - // first payload publishes nothing. - let size = payload.len() as u64; + // actual size for the same reason, and it covers the inflated size too. Also checked before + // the group is opened, so a refused first payload publishes nothing. + let plain = payload.len() as u64; let bound = if self.flate.is_some() { - crate::Encoder::bound(size) + crate::Encoder::bound(plain) } else { - size + plain }; if self.frames >= moq_net::group::MAX_GROUP_FRAMES || self.bytes.saturating_add(bound) > moq_net::group::MAX_CACHE_BYTES @@ -142,7 +139,7 @@ impl Inner { let group = self.group.as_mut().expect("a group is open"); let Err(err) = group.write_frame(timestamp, payload) else { self.frames += 1; - self.bytes += size as u64; + self.bytes += plain.max(size as u64); return Ok(size); }; diff --git a/rs/moq-json/MIGRATE.md b/rs/moq-json/MIGRATE.md index a0f4a7c71df..a03a627b72d 100644 --- a/rs/moq-json/MIGRATE.md +++ b/rs/moq-json/MIGRATE.md @@ -4,9 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased -### Snapshot consumers are bounded by default +### A compressed group is held to the group budget -`snapshot::consumer::Config::default()` now sets `max_size` to `DEFAULT_MAX_SIZE` (4 MiB), so a compressed frame from a peer cannot reconstruct an unbounded value. A consumer that needs a larger state sets `max_size` itself, or `None` to keep only the DEFLATE frame cap. +A compressed group may inflate to at most `moq_net::group::MAX_CACHE_BYTES` (32 MiB), the most a plain group carries, so a peer's small frames cannot make a reader hold more than an uncompressed track would. +The encoders charge each frame its plaintext: a compressed snapshot or window header over 32 MiB fails with `moq_net::Error::FrameTooLarge` (snapshot) or the window's group cache error, where it used to pass up to 64 MiB, and deltas and ops roll a new group once a group's plaintext would pass the budget. +A compressed stream's budget is 32 MiB of record plaintext. +`snapshot::consumer::Config::max_size` still sets a tighter limit, but no longer stops inflation early; a frame inflates within the group budget and is then refused. ### Consumers return `Timed` (#5677) diff --git a/rs/moq-json/src/snapshot/consumer.rs b/rs/moq-json/src/snapshot/consumer.rs index fd1530630df..728df86586d 100644 --- a/rs/moq-json/src/snapshot/consumer.rs +++ b/rs/moq-json/src/snapshot/consumer.rs @@ -7,18 +7,11 @@ use serde::de::DeserializeOwned; use super::Decoder; use crate::{Compression, Result}; -/// The default [`Config::max_size`]. -/// -/// A peer supplies the compressed frame, so an unset limit lets a small DEFLATE frame -/// reconstruct an unbounded value. Every catalog and metadata track this crate reads -/// fits far below this, and a consumer that needs more sets its own. -pub const DEFAULT_MAX_SIZE: usize = 4 * 1024 * 1024; - /// Track-owning options for a [`Consumer`]. /// /// Build from [`Default`] and override fields (the struct is `#[non_exhaustive]`, so new options /// stay additive). -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] #[non_exhaustive] pub struct Config { /// How the frames are compressed. Must match the encoder's @@ -26,20 +19,14 @@ pub struct Config { pub compression: Compression, /// Maximum bytes in an inflated frame or the compact JSON of the reconstructed value. - /// Defaults to [`DEFAULT_MAX_SIZE`]; `None` keeps only the DEFLATE decoder's frame - /// cap and leaves state size unlimited. + /// + /// Unset, the group budget is the only bound: a compressed group inflates to at most + /// [`MAX_CACHE_BYTES`](moq_net::group::MAX_CACHE_BYTES), the most a plain one carries, so the + /// reconstructed value costs no more than an uncompressed track's. Set it to hold a smaller + /// document, at the cost of re-measuring the value after every delta. pub max_size: Option, } -impl Default for Config { - fn default() -> Self { - Self { - compression: Compression::None, - max_size: Some(DEFAULT_MAX_SIZE), - } - } -} - /// Consumes a JSON value from a track, reconstructing it from snapshots and deltas. /// /// A [`Decoder`] that owns its track: it reads groups, routes each frame by its position, and @@ -185,13 +172,6 @@ impl Consumer { mod timestamps { use super::*; - #[test] - fn default_config_bounds_the_reconstructed_value() { - // A peer picks the compressed frame, so the default must not leave the - // reconstructed value unlimited. - assert_eq!(Config::default().max_size, Some(DEFAULT_MAX_SIZE)); - } - #[test] fn exact_timestamps_survive_reads() { for compression in [crate::Compression::None, crate::Compression::Deflate] { diff --git a/rs/moq-json/src/snapshot/decoder.rs b/rs/moq-json/src/snapshot/decoder.rs index 3159b851c4e..452152dd43d 100644 --- a/rs/moq-json/src/snapshot/decoder.rs +++ b/rs/moq-json/src/snapshot/decoder.rs @@ -68,12 +68,8 @@ impl Decoder { /// only the first. pub fn snapshot(&mut self, payload: &[u8]) -> Result<()> { self.current = None; - // Every group starts a new DEFLATE window; enforce the budget while inflating. - self.flate = self.compression.then(|| { - moq_flate::Decoder::with_max_frame_size(self.max_size.map_or(moq_flate::DEFAULT_MAX_FRAME_SIZE, |size| { - (size as u64).min(moq_flate::DEFAULT_MAX_FRAME_SIZE) - })) - }); + // Every group starts a new DEFLATE window, which holds the whole group to the group budget. + self.flate = self.compression.then(moq_flate::Decoder::new); let inflated = self.flate.as_mut().map(|flate| flate.frame(payload)).transpose()?; let plain = inflated.as_deref().unwrap_or(payload); if let Some(limit) = self.max_size @@ -271,7 +267,7 @@ mod test { } else { decoder.delta(&payload) }; - assert!(matches!(result, Err(Error::Flate(moq_flate::Error::TooLarge(1024))))); + assert!(matches!(result, Err(Error::TooLarge(1024)))); assert!(decoder.value().is_none()); } } diff --git a/rs/moq-json/src/snapshot/encoder.rs b/rs/moq-json/src/snapshot/encoder.rs index c2acbb4b577..60d230fed21 100644 --- a/rs/moq-json/src/snapshot/encoder.rs +++ b/rs/moq-json/src/snapshot/encoder.rs @@ -215,6 +215,10 @@ pub struct Encoder { /// Frames emitted into the current group, snapshot included. group_frames: usize, + /// Bytes charged to the current group's budget: each frame's slice, or its plaintext when that + /// is larger, since a reader's DEFLATE window is held to the same budget. + group_bytes: u64, + /// The most bytes a group may hold, the cap a delta is admitted under. A field rather than the /// constant so a test can shrink it to a size it can reach. max_group_bytes: u64, @@ -239,6 +243,7 @@ impl Encoder { delta_bytes: 0, snapshot_len: 0, group_frames: 0, + group_bytes: 0, max_group_bytes: moq_net::group::MAX_CACHE_BYTES, resync: false, _marker: PhantomData, @@ -269,6 +274,7 @@ impl Encoder { self.delta_bytes = 0; self.snapshot_len = 0; self.group_frames = 0; + self.group_bytes = 0; self.resync = true; } } @@ -335,13 +341,6 @@ impl Encoder { // Compress into the per-group window only now, for a frame we are committed to emitting. let bytes = Bytes::from(patch); - - // Same cap as a snapshot, on the patch's plaintext: a delta that decompresses past the - // consumer's limit makes the whole group unreadable, since there is no keyframe after it to - // resynchronize on. Rejecting here leaves the encoder to reset and the group as it was. - if self.config.compression.is_deflate() && bytes.len() as u64 > moq_flate::DEFAULT_MAX_FRAME_SIZE { - return Err(moq_flate::Error::TooLarge(moq_flate::DEFAULT_MAX_FRAME_SIZE).into()); - } let payload = match self.flate.as_mut() { Some(flate) => flate.frame(&bytes), None => bytes.clone(), @@ -349,17 +348,19 @@ impl Encoder { // A delta is only readable while the group still holds the snapshot it applies to. // Admitting a patch that pushes the group past that budget would abort it - // (`GroupTooLarge`), leaving a late subscriber with no value. Roll a fresh snapshot - // instead, which is cheap next to losing the value. + // (`GroupTooLarge`), or past a reader's inflate budget, leaving a late subscriber with no + // value. Roll a fresh snapshot instead, which is cheap next to losing the value. // - // Measured on the encoded payload rather than the plaintext: a sync-flushed DEFLATE frame can - // come out slightly larger than its input, so the plaintext is not an upper bound. Compressing - // first advances the window, but [`Self::snapshot`] opens a fresh one, so an over-budget delta - // costs only the wasted compression. - if self.snapshot_len + self.delta_bytes + payload.len() as u64 > self.max_group_bytes { + // Charged the larger of the encoded payload and the plaintext: a sync-flushed DEFLATE frame + // can come out slightly larger than its input, while the reader's window counts what it + // inflates. Compressing first advances the window, but [`Self::snapshot`] opens a fresh one, + // so an over-budget delta costs only the wasted compression. + let charge = (payload.len() as u64).max(bytes.len() as u64); + if self.group_bytes + charge > self.max_group_bytes { return self.snapshot(value).map(Some); } + self.group_bytes += charge; self.delta_bytes += payload.len() as u64; self.group_frames += 1; @@ -404,11 +405,12 @@ impl Encoder { /// [`snapshot`](Self::snapshot) for a value that is already serialized, so an update that /// encoded `T` to compare it against a byte baseline does not encode it a second time. fn snapshot_encoded(&mut self, snapshot: Vec) -> Result { - // Every consumer decodes with moq-flate's default output cap, so a value past it would be - // unreadable however small it compresses to. Reject it before anything is published, so the - // previously published value stands rather than being superseded by one nothing can read. - if self.config.compression.is_deflate() && snapshot.len() as u64 > moq_flate::DEFAULT_MAX_FRAME_SIZE { - return Err(moq_flate::Error::TooLarge(moq_flate::DEFAULT_MAX_FRAME_SIZE).into()); + // Every consumer holds a compressed group to the group budget by its inflated size, so a value + // past it would be unreadable however small it compresses to: the same `FrameTooLarge` a plain + // one meets. Reject it before anything is published, so the previously published value stands + // rather than being superseded by one nothing can read. + if self.config.compression.is_deflate() && snapshot.len() as u64 > self.max_group_bytes { + return Err(moq_net::Error::FrameTooLarge.into()); } // With deltas possible, read the baseline back out of those same bytes rather than @@ -425,6 +427,7 @@ impl Encoder { // Every fallible step runs before any state changes, so a failure leaves the encoder exactly // as it was rather than half-advanced with no frame to show for it. let snapshot = Bytes::from(snapshot); + let plain_len = snapshot.len() as u64; let last = if self.config.delta_ratio == 0 { // No delta will ever diff against this, so hold the bytes instead. Uncompressed, they are // the same allocation the payload carries, so the baseline costs a refcount. @@ -450,6 +453,7 @@ impl Encoder { self.snapshot_len = payload.len() as u64; self.delta_bytes = 0; self.group_frames = 1; + self.group_bytes = self.snapshot_len.max(plain_len); self.flate = flate; self.last = Some(last); self.resync = false; @@ -856,7 +860,7 @@ mod test { // Measure the frames under the default budget, which admits the delta. let mut probe = Encoder::::new(config.clone()); - let snapshot = commit(&mut probe, &value).expect("a snapshot"); + assert!(commit(&mut probe, &value).expect("a snapshot").keyframe); let delta = commit(&mut probe, &patched).expect("a delta"); assert!(!delta.keyframe); assert!( @@ -865,9 +869,10 @@ mod test { delta.payload.len() ); - // A budget with room for the plaintext patch but not the encoded one. + // A budget with room for the plaintext patch but not the encoded one. The snapshot is charged + // its plaintext, which outweighs its slice. let mut encoder = Encoder::::new(config); - encoder.max_group_bytes = (snapshot.payload.len() + plaintext) as u64; + encoder.max_group_bytes = (serde_json::to_vec(&value).unwrap().len() + plaintext) as u64; assert!(commit(&mut encoder, &value).unwrap().keyframe); // The patch rolls a fresh snapshot instead of joining the first group. @@ -881,6 +886,52 @@ mod test { ); } + /// A compressed delta is charged its plaintext too: the reader's window counts what it inflates, + /// so a patch that deflates to almost nothing still rolls the group once its plaintext would + /// overflow the budget. Matches the JS `a compressed delta is gated on its inflated size`. + #[test] + fn a_compressed_delta_is_gated_on_its_inflated_size() { + let value = json!({ "v": "x".repeat(1000) }); + let patched = json!({ "v": "y".repeat(1000) }); + let config = Config { + delta_ratio: 100, + compression: Compression::Deflate, + }; + + let mut encoder = Encoder::::new(config); + encoder.max_group_bytes = (serde_json::to_vec(&value).unwrap().len() + 500) as u64; + assert!(commit(&mut encoder, &value).unwrap().keyframe); + let rolled = commit(&mut encoder, &patched).expect("a rolled snapshot"); + assert!( + rolled.keyframe, + "a 1 KB patch must not ride in a group with 500 bytes left" + ); + assert!( + rolled.payload.len() < 500, + "the patch deflates small, so only its plaintext overflows" + ); + } + + /// A compressed snapshot is refused by its plaintext, the way a plain one is, however small it + /// deflates: no reader could inflate it. + #[test] + fn a_compressed_snapshot_past_the_budget_is_refused() { + let config = Config { + delta_ratio: 0, + compression: Compression::Deflate, + }; + let mut encoder = Encoder::::new(config); + encoder.max_group_bytes = 100; + assert!(matches!( + encoder.update(&json!({ "v": "x".repeat(1000) })), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + )); + assert!( + encoder.value().is_none(), + "a refused snapshot leaves the encoder as it was" + ); + } + #[test] fn compressed_deltas_reuse_the_group_window() { let phrase = "Media over QUIC delivers real-time latency at massive scale"; diff --git a/rs/moq-json/src/stream/encoder.rs b/rs/moq-json/src/stream/encoder.rs index d7ecea8d69e..e48a048d8c0 100644 --- a/rs/moq-json/src/stream/encoder.rs +++ b/rs/moq-json/src/stream/encoder.rs @@ -37,6 +37,8 @@ pub struct Config { pub struct Pending<'a, T> { encoder: &'a mut Encoder, payload: Bytes, + /// What the record costs the group's budget: its plaintext or its slice, whichever is larger. + charge: u64, committed: bool, } @@ -52,7 +54,7 @@ impl Pending<'_, T> { pub fn commit(mut self) { self.committed = true; self.encoder.frames += 1; - self.encoder.bytes += self.payload.len() as u64; + self.encoder.bytes += self.charge; } } @@ -82,8 +84,9 @@ pub struct Encoder { /// consumer for the rest of the group, so encoding stops until the caller rolls a new one. desynced: bool, - /// Frames and payload bytes committed to the current group, checked against moq-net's group - /// budget before each record is encoded. + /// Frames and bytes committed to the current group, checked against moq-net's group budget + /// before each record is encoded. A compressed record is charged its plaintext (or its slice, if + /// larger), since a reader's DEFLATE window is held to the same budget. frames: usize, bytes: u64, @@ -143,8 +146,7 @@ impl Encoder { // Check before compressing: encoding advances the window, so a record refused afterwards // would leave the encoder ahead of every reader. The worst case is checked rather than the - // actual size for the same reason. The budget is also below moq-flate's per-frame decode cap, - // so any record that fits is one every consumer can inflate. + // actual size for the same reason; it also covers the plaintext a reader inflates. let size = bytes.len() as u64; let bound = if self.compression { moq_flate::Encoder::bound(size) @@ -164,6 +166,7 @@ impl Encoder { Ok(Pending { encoder: self, + charge: size.max(payload.len() as u64), payload, committed: false, }) diff --git a/rs/moq-json/src/stream/mod.rs b/rs/moq-json/src/stream/mod.rs index 7f6a2b3d8f9..4aa09493f90 100644 --- a/rs/moq-json/src/stream/mod.rs +++ b/rs/moq-json/src/stream/mod.rs @@ -29,7 +29,9 @@ //! Instead an append that might not fit is refused with [`moq_net::Error::GroupTooLarge`] before //! it is encoded, and the log stays intact and writable. With compression the check counts the //! record's raw size plus DEFLATE's worst-case overhead, since the compressed size is only known -//! once the window has moved. The budget covers the whole log rather than a single record, so once +//! once the window has moved, and each written record is charged its raw size: a reader's DEFLATE +//! window is held to the same budget, so compression shrinks the wire but not how much a log can +//! hold. The budget covers the whole log rather than a single record, so once //! it is spent every append is refused: the live stream is bounded history by design, a publisher //! with more to say opens a new track, and deep history is served from a recording. //! @@ -453,6 +455,18 @@ mod test { } } + /// A compressed record is charged its plaintext, since that is what a reader's window inflates: + /// two records that each fit, and deflate to almost nothing, still overflow the budget together. + #[test] + fn a_compressed_log_is_charged_its_inflated_size() { + let (mut producer, track) = producer(compressed()); + let half = Value::String("x".repeat(moq_net::group::MAX_CACHE_BYTES as usize / 2)); + producer.append(&half).unwrap(); + assert!(refused(producer.append(&half))); + producer.finish().unwrap(); + assert_eq!(drain(consume(track, true)), vec![half]); + } + /// The budget covers the whole log, so once its frames are spent every append is refused, and /// the log written so far still finishes cleanly and reads back whole. #[test] diff --git a/rs/moq-json/src/window/encoder.rs b/rs/moq-json/src/window/encoder.rs index f06d52b9887..b2e85066b61 100644 --- a/rs/moq-json/src/window/encoder.rs +++ b/rs/moq-json/src/window/encoder.rs @@ -185,6 +185,10 @@ pub struct Encoder { /// Frames emitted into the current group, header included. group_frames: usize, + /// Bytes charged to the current group's budget: each frame's slice, or its plaintext when that + /// is larger, since a reader's DEFLATE window is held to the same budget. + group_bytes: u64, + /// Whether the next frame must be a header because a frame was lost. Kept separate from the /// window, which a resync must never discard. resync: bool, @@ -208,6 +212,7 @@ impl Encoder { op_bytes: 0, header_len: 0, group_frames: 0, + group_bytes: 0, resync: true, _marker: PhantomData, } @@ -234,6 +239,7 @@ impl Encoder { self.op_bytes = 0; self.header_len = 0; self.group_frames = 0; + self.group_bytes = 0; self.resync = true; } @@ -268,43 +274,28 @@ impl Encoder { && self.op_bytes <= ratio * self.header_len } - /// Reject plaintext that the paired DEFLATE decoder could not produce. - fn validate_plaintext(len: usize, kind: &str) -> Result<()> { - if u64::try_from(len).unwrap_or(u64::MAX) > moq_flate::DEFAULT_MAX_FRAME_SIZE { - return Err(Error::Json(format!( - "window {kind} exceeds the decoder's decompressed size limit" - ))); - } - Ok(()) - } - - /// Compress an already-serialized op into the open group, charging it to the budget. - fn frame(&mut self, bytes: Vec) -> Result { - Self::validate_plaintext(bytes.len(), "frame")?; + /// Emit an op when the header will remain cached and inflatable, otherwise restate the window + /// in a new group. + /// + /// Charged the larger of the slice and the plaintext: a sync-flushed DEFLATE frame can come out + /// slightly larger than its input, while a reader's window counts what it inflates. + fn emit_op(&mut self, bytes: Vec) -> Result> { + let plain = bytes.len() as u64; let payload = match self.flate.as_mut() { Some(flate) => flate.frame(&bytes), None => Bytes::from(bytes), }; - self.op_bytes += payload.len() as u64; self.group_frames += 1; - - Ok(Encoded { - payload, - keyframe: false, - }) - } - - /// Emit an op when the header will remain cached, otherwise restate the window in a new group. - fn emit_op(&mut self, bytes: Vec) -> Result> { - let encoded = self.frame(bytes)?; - let group_bytes = self.header_len.saturating_add(self.op_bytes); - if group_bytes > moq_net::group::MAX_CACHE_BYTES { + self.group_bytes = self.group_bytes.saturating_add(plain.max(payload.len() as u64)); + if self.group_bytes > moq_net::group::MAX_CACHE_BYTES { self.reset(); - Ok(None) - } else { - Ok(Some(encoded)) + return Ok(None); } + Ok(Some(Encoded { + payload, + keyframe: false, + })) } /// Serialize a bounded suffix before mutably borrowing the compression state. @@ -332,7 +323,11 @@ impl Encoder { /// Encode the header restating the whole window and opening a new group. fn emit_header(&mut self, bytes: Vec) -> Result { - Self::validate_plaintext(bytes.len(), "header")?; + // A reader inflates the header whole, so its plaintext meets the group budget too. + let plain = bytes.len() as u64; + if plain > moq_net::group::MAX_CACHE_BYTES { + return Err(Error::Json("window header exceeds the group cache limit".into())); + } // Open a fresh per-group encoder (cold window) and compress the header as frame 0, recording // its wire size as the op budget's anchor. @@ -351,6 +346,7 @@ impl Encoder { self.header_len = payload.len() as u64; self.op_bytes = 0; self.group_frames = 1; + self.group_bytes = plain.max(self.header_len); self.flate = flate; self.resync = false; @@ -531,22 +527,24 @@ mod test { assert_eq!(encoder.window(), vec![Value::from(2)]); } + /// Compression doesn't lift the limit: a reader inflates the header whole, so one past the group + /// budget is refused however small it deflates. #[test] fn a_header_larger_than_the_group_cache_is_rejected() { - let mut encoder = Encoder::::new(ProducerConfig::default()); - let record = "x".repeat(moq_net::group::MAX_CACHE_BYTES as usize); - - let err = encoder.push(&record).err().expect("oversized header should fail"); - assert!(err.to_string().contains("group cache limit")); - assert!(encoder.window().is_empty()); - - let frame = encoder.push(&"ok".to_string()).unwrap(); - assert!(frame.keyframe); - } - - #[test] - fn plaintext_is_bounded_by_the_decoder_limit() { - let len = usize::try_from(moq_flate::DEFAULT_MAX_FRAME_SIZE + 1).unwrap(); - assert!(Encoder::<()>::validate_plaintext(len, "frame").is_err()); + for compression in [false, true] { + let config = ProducerConfig { + compression, + ..Default::default() + }; + let mut encoder = Encoder::::new(config); + let record = "x".repeat(moq_net::group::MAX_CACHE_BYTES as usize); + + let err = encoder.push(&record).err().expect("oversized header should fail"); + assert!(err.to_string().contains("group cache limit")); + assert!(encoder.window().is_empty()); + + let frame = encoder.push(&"ok".to_string()).unwrap(); + assert!(frame.keyframe); + } } } diff --git a/rs/moq-mux/src/catalog/msf/consumer.rs b/rs/moq-mux/src/catalog/msf/consumer.rs index 8479e394ea8..2b45d756c5c 100644 --- a/rs/moq-mux/src/catalog/msf/consumer.rs +++ b/rs/moq-mux/src/catalog/msf/consumer.rs @@ -50,7 +50,16 @@ impl Consumer { match ready!(group.poll_read_frame(waiter)?) { Some(frame) => { self.group = None; - let catalog = parse::(&frame.payload, &self.track.broadcast().path)?; + let json = std::str::from_utf8(&frame.payload).map_err(|_| Error::InvalidUtf8)?; + let msf = match moq_msf::Catalog::::from_str(json) { + Ok(msf) => msf, + Err(err) => { + tracing::warn!(error = %err, "failed to parse MSF catalog frame"); + return Poll::Ready(Err(Error::ParseFrame.into())); + } + }; + let catalog = from_msf(&msf, &self.track.broadcast().path)?; + catalog.check_renditions()?; return Poll::Ready(Ok(Some(catalog))); } None => self.group = None, @@ -74,31 +83,6 @@ impl Consumer { } } -/// Decode one catalog frame, bounding it before parsing. -/// -/// The frame comes from a peer, and JSON parsing allocates several times its bytes, -/// so a catalog must fit [`moq_json::snapshot::consumer::DEFAULT_MAX_SIZE`], the same -/// limit every compressed catalog track uses. -fn parse(payload: &[u8], base: &moq_net::Path<'_>) -> Result> { - const MAX: usize = moq_json::snapshot::consumer::DEFAULT_MAX_SIZE; - if payload.len() > MAX { - return Err(Error::FrameTooLarge { - size: payload.len(), - max: MAX, - } - .into()); - } - - let json = std::str::from_utf8(payload).map_err(|_| Error::InvalidUtf8)?; - let msf = moq_msf::Catalog::::from_str(json).map_err(|err| { - tracing::warn!(error = %err, "failed to parse MSF catalog frame"); - Error::ParseFrame - })?; - let catalog = from_msf(&msf, base)?; - catalog.check_renditions()?; - Ok(catalog) -} - impl From for Consumer { fn from(inner: moq_net::track::Subscriber) -> Self { Self::new(inner) @@ -1110,17 +1094,6 @@ mod test { ); } - #[test] - fn oversized_catalog_frame_is_refused_before_parsing() { - let payload = vec![b'{'; moq_json::snapshot::consumer::DEFAULT_MAX_SIZE + 1]; - let err = parse::<()>(&payload, &moq_net::Path::empty()).expect_err("oversized frame must be refused"); - assert!(matches!( - err, - crate::Error::Msf(Error::FrameTooLarge { size, max }) - if size == payload.len() && max == moq_json::snapshot::consumer::DEFAULT_MAX_SIZE - )); - } - #[test] fn invalid_video_codec_includes_codec_in_error() { // avc1 with a too-short profile string is a malformed structured codec. diff --git a/rs/moq-mux/src/catalog/msf/mod.rs b/rs/moq-mux/src/catalog/msf/mod.rs index 12ed707847f..de2539cc912 100644 --- a/rs/moq-mux/src/catalog/msf/mod.rs +++ b/rs/moq-mux/src/catalog/msf/mod.rs @@ -48,15 +48,6 @@ pub enum Error { #[error("MSF catalog frame is not valid UTF-8")] InvalidUtf8, - /// The catalog frame is larger than a catalog may be. - #[error("MSF catalog frame is {size} bytes, over the {max}-byte limit")] - FrameTooLarge { - /// The frame's size in bytes. - size: usize, - /// The limit it exceeded. - max: usize, - }, - /// The catalog frame cannot be decoded as an MSF catalog. #[error("failed to parse MSF catalog frame")] ParseFrame, diff --git a/rs/moq-stats/MIGRATE.md b/rs/moq-stats/MIGRATE.md index a14972577b2..a7c8ddf77f9 100644 --- a/rs/moq-stats/MIGRATE.md +++ b/rs/moq-stats/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### The `.z` flavor ends with the plain one past 32 MB + +A frame whose uncompressed JSON passes the 32 MB group budget now ends both the plain track and its `.z` sibling with `FrameTooLarge`: a reader inflates the whole group, so compression no longer carries a larger frame. +A reader that fell back to `.z` for very large frames sees the same end the plain reader does. + ### A returning group restarts under a new epoch (#4904) A stats group that returns announces a new epoch and starts its group numbers at zero, instead of continuing the old sequence. diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 34769299624..cc17ebba0c7 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -479,23 +479,26 @@ impl Snapshot { /// /// A frame over the cache limit aborts the track with /// [`moq_net::Error::FrameTooLarge`]: skipping it would leave readers on the - /// previous snapshot with no sign it stopped tracking. + /// previous snapshot with no sign it stopped tracking. A compressed frame + /// meets the same limit by its uncompressed size, which the encoder refuses. fn update(&mut self, value: &Frame) -> moq_json::Result> { if self.track.is_closed() { return Ok(None); } - let Some(frame) = self.encoder.update(value)? else { - return Ok(None); + let frame = match self.encoder.update(value) { + Ok(Some(frame)) if frame.payload.len() as u64 <= moq_net::group::MAX_CACHE_BYTES => frame, + Ok(None) => return Ok(None), + Ok(Some(_)) | Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) => { + let err = moq_net::Error::FrameTooLarge; + if let Some(group) = self.group.take() { + let _ = group.abort(err.clone()); + } + let _ = self.track.clone().abort(err.clone()); + return Err(err.into()); + } + Err(err) => return Err(err), }; let len = frame.payload.len(); - if len as u64 > moq_net::group::MAX_CACHE_BYTES { - let err = moq_net::Error::FrameTooLarge; - if let Some(group) = self.group.take() { - let _ = group.abort(err.clone()); - } - let _ = self.track.clone().abort(err.clone()); - return Err(err.into()); - } if frame.keyframe { if let Some(mut group) = self.group.take() { group.finish()?; @@ -1800,11 +1803,12 @@ mod tests { } } - /// A flavor whose frame outgrows the cache ends with `FrameTooLarge` instead - /// of leaving readers on its last snapshot, while its sibling, which DEFLATE - /// shrinks under the limit, keeps updating. + /// A frame that outgrows the cache ends both flavors with `FrameTooLarge` + /// instead of leaving readers on their last snapshot. DEFLATE shrinks the + /// `.z` frame under the limit, but a reader inflates it whole, so the + /// encoder holds it to the same budget by its uncompressed size. #[tokio::test] - async fn oversized_frame_aborts_only_its_flavor() { + async fn oversized_frame_ends_both_flavors() { use futures::FutureExt; type Value = BTreeMap; @@ -1844,17 +1848,12 @@ mod tests { let huge = "a".repeat(moq_net::group::MAX_CACHE_BYTES as usize); publish(&huge, 2); - assert!(matches!( - next(&mut plain), - Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) - )); - assert_eq!(next(&mut compressed).unwrap(), Some(Value::from([(huge, 2)]))); - - publish("small", 3); - assert_eq!( - next(&mut compressed).unwrap(), - Some(Value::from([("small".to_string(), 3)])) - ); + for reader in [&mut plain, &mut compressed] { + assert!(matches!( + next(reader), + Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) + )); + } } /// Build an origin producer, spawning its driver on the ambient runtime. From 68a338ca67ff990e92c86df2cf1066bfcfe75009 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 11:52:55 -0700 Subject: [PATCH 3/3] fix(json): resynchronize when a refused roll follows an advanced window A compressed delta that overflows the group advances the DEFLATE window before the encoder rolls to a snapshot. If that snapshot is refused too, the advanced window stayed installed and the next update rode as a delta against history no reader received. Drop the group before rolling, in Rust and JS. Co-Authored-By: Claude Opus 5.5 --- js/json/src/snapshot/encoder.ts | 5 +++-- js/json/src/snapshot/snapshot.test.ts | 14 +++++++++++++ rs/moq-json/src/snapshot/encoder.rs | 30 +++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/js/json/src/snapshot/encoder.ts b/js/json/src/snapshot/encoder.ts index 02e0a687e45..c447df3cc6c 100644 --- a/js/json/src/snapshot/encoder.ts +++ b/js/json/src/snapshot/encoder.ts @@ -230,8 +230,8 @@ export class Encoder { // // Charged the larger of the encoded payload and the plaintext: a sync-flushed DEFLATE frame // can come out slightly larger than its input, while the reader's window counts what it - // inflates. Compressing first advances the window, but `#snapshot` opens a fresh one, so an - // over-budget delta costs only the wasted compression. + // inflates. Compressing first advanced the window past what any reader holds, so drop the + // group before rolling: if the snapshot is refused too, the next update must not be a delta. const charge = Math.max(payload.length, delta.length); if (this.#groupBytes + charge <= this.#maxGroupBytes) { this.#last = json; @@ -240,6 +240,7 @@ export class Encoder { this.#groupFrames += 1; return this.#pend({ payload, keyframe: false }); } + this.reset(); } const payload = this.#snapshot(new TextEncoder().encode(text)); diff --git a/js/json/src/snapshot/snapshot.test.ts b/js/json/src/snapshot/snapshot.test.ts index a5ff68ea2cd..b3bcd4146c6 100644 --- a/js/json/src/snapshot/snapshot.test.ts +++ b/js/json/src/snapshot/snapshot.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; +import { Decoder as Flate } from "@moq/flate"; import { Group, Error as NetError, StreamCode, Time, Track } from "@moq/net"; import { Consumer } from "./consumer.ts"; import { Encoder } from "./encoder.ts"; @@ -419,6 +420,19 @@ test("a compressed delta is gated on its inflated size", () => { expect(rolled?.payload.length).toBeLessThan(500); }); +// A delta that overflows the group advances the window before the roll is attempted, so a refused roll +// must not leave that window behind: the next update is a snapshot a reader can decode. +test("a refused roll resynchronizes", () => { + const encoder = new Encoder({ compression: "deflate", deltaRatio: 100, maxGroupBytes: 200 }); + encoder.update({ a: 1 })?.commit(); + expect(() => encoder.update({ a: 1, b: "x".repeat(300) })).toThrow(NetError.FrameTooLarge); + + const next = encoder.update({ a: 2 }); + expect(next?.keyframe).toBe(true); + const decoded = new TextDecoder().decode(new Flate().frame(next?.payload ?? new Uint8Array())); + expect(JSON.parse(decoded)).toEqual({ a: 2 }); +}); + // A compressed snapshot is refused by its plaintext, the way a plain one is, however small it deflates: // no reader could inflate it. test("a compressed snapshot past the budget is refused", () => { diff --git a/rs/moq-json/src/snapshot/encoder.rs b/rs/moq-json/src/snapshot/encoder.rs index 60d230fed21..fd6fa9bf85a 100644 --- a/rs/moq-json/src/snapshot/encoder.rs +++ b/rs/moq-json/src/snapshot/encoder.rs @@ -353,10 +353,11 @@ impl Encoder { // // Charged the larger of the encoded payload and the plaintext: a sync-flushed DEFLATE frame // can come out slightly larger than its input, while the reader's window counts what it - // inflates. Compressing first advances the window, but [`Self::snapshot`] opens a fresh one, - // so an over-budget delta costs only the wasted compression. + // inflates. Compressing first advanced the window past what any reader holds, so drop the + // group before rolling: if the snapshot is refused too, the next update must not be a delta. let charge = (payload.len() as u64).max(bytes.len() as u64); if self.group_bytes + charge > self.max_group_bytes { + self.reset(); return self.snapshot(value).map(Some); } @@ -912,6 +913,31 @@ mod test { ); } + /// A delta that overflows the group advances the window before the roll is attempted, so a refused + /// roll must not leave that window behind: the next update is a snapshot a reader can decode. + #[test] + fn a_refused_roll_resynchronizes() { + let config = Config { + delta_ratio: 100, + compression: Compression::Deflate, + }; + let mut encoder = Encoder::::new(config); + encoder.max_group_bytes = 200; + assert!(commit(&mut encoder, &json!({ "a": 1 })).unwrap().keyframe); + assert!(matches!( + encoder.update(&json!({ "a": 1, "b": "x".repeat(300) })), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + )); + + let next = commit(&mut encoder, &json!({ "a": 2 })).expect("a snapshot"); + assert!(next.keyframe); + let mut decoder = moq_flate::Decoder::new(); + assert_eq!( + serde_json::from_slice::(&decoder.frame(&next.payload).unwrap()).unwrap(), + json!({ "a": 2 }) + ); + } + /// A compressed snapshot is refused by its plaintext, the way a plain one is, however small it /// deflates: no reader could inflate it. #[test]