From 22f82877be8491d781484357d2a8eb354881bc79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 15:55:47 -0700 Subject: [PATCH 1/5] chore: claim JS track compression quest Co-Authored-By: GPT-6 From c1b38e92f4e6d44ad36cb2cd1fc421fd062ab163 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 16:02:03 -0700 Subject: [PATCH 2/5] wip(net): checkpoint bounded private compression codecs Prepare the group-scoped codec foundation without changing the model, wire behavior, or advertised capabilities. The decoder enforces exact declared sizes, stops oversized inflation inside pako onData, rejects final DEFLATE blocks, and retains tight frame buffers. The encoder is isolated behind pako browser deflate for eventual lazy loading. Unresolved: synchronous writes currently require a static compressor, measured at 2870 gzip bytes beyond Inflate alone. Lazy loading requires observable pending-write timing or an explicit preparation API. Planning PR5885 and core PR5928 do not settle that JS choice. No writer timing or public API decision is implemented. Passed: 5 private codec tests (14 assertions), js/net type checks including examples and benchmarks, Biome, git diff --check. Full CI, drafts, Rust/JS exchange, interop --all, integrated bundles and fanout remain unrun because the feature is not integrated. Co-Authored-By: GPT-6 --- bun.lock | 1 + js/net/package.json | 1 + js/net/src/flate/codec.test.ts | 62 ++++++++++++++++++++++++++++ js/net/src/flate/decoder.ts | 59 +++++++++++++++++++++++++++ js/net/src/flate/encoder.ts | 74 ++++++++++++++++++++++++++++++++++ 5 files changed, 197 insertions(+) create mode 100644 js/net/src/flate/codec.test.ts create mode 100644 js/net/src/flate/decoder.ts create mode 100644 js/net/src/flate/encoder.ts diff --git a/bun.lock b/bun.lock index adca12f16a4..b374b978d69 100644 --- a/bun.lock +++ b/bun.lock @@ -264,6 +264,7 @@ "@moq/qmux": "workspace:^", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0", + "pako": "^3.0.2", "uuid": "^14.0.3", }, "devDependencies": { diff --git a/js/net/package.json b/js/net/package.json index 1b861448073..a27f70da617 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -26,6 +26,7 @@ "@moq/qmux": "workspace:^", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0", + "pako": "^3.0.2", "uuid": "^14.0.3" }, "peerDependencies": { diff --git a/js/net/src/flate/codec.test.ts b/js/net/src/flate/codec.test.ts new file mode 100644 index 00000000000..397fe7c38b5 --- /dev/null +++ b/js/net/src/flate/codec.test.ts @@ -0,0 +1,62 @@ +import { expect, spyOn, test } from "bun:test"; +import { deflateRaw, Inflate } from "pako"; +import { Decoder } from "./decoder.ts"; +import { Encoder } from "./encoder.ts"; + +const text = new TextEncoder(); + +test("group slices preserve a shared window across empty and nonempty frames", () => { + const frames = ["the quick brown fox", "", "the quick brown dog", "the lazy fox"]; + const encoder = new Encoder(); + const decoder = new Decoder(); + for (const frame of frames.map((value) => text.encode(value))) { + expect(decoder.frame(encoder.frame(frame), frame.length)).toEqual(frame); + } +}); + +test("a retained frame owns only its payload, across later pushes", () => { + const encoder = new Encoder(); + const decoder = new Decoder(); + const first = decoder.frame(encoder.frame(text.encode("first")), 5); + decoder.frame(encoder.frame(new Uint8Array(20_000)), 20_000); + expect(first).toEqual(text.encode("first")); + expect(first.buffer.byteLength).toBe(first.length); +}); + +test("each declared size is exact, including an empty slice", () => { + const slice = new Encoder().frame(new Uint8Array(1024)); + for (const size of [512, 2048]) expect(() => new Decoder().frame(slice, size)).toThrow(/declared size/); + expect(() => new Decoder().frame(new Uint8Array(), 1)).toThrow(/declared size/); +}); + +test("a bomb stops inflating at the declaration plus one scratch chunk", () => { + const slice = new Encoder().frame(new Uint8Array(1024 * 1024)); + const push = Inflate.prototype.push; + let output = 0; + const pushing = spyOn(Inflate.prototype, "push").mockImplementation(function (this: Inflate, ...args) { + const onData = this.onData; + this.onData = (chunk) => { + output += chunk.length; + onData.call(this, chunk); + }; + try { + return push.apply(this, args); + } finally { + this.onData = onData; + } + }); + try { + expect(() => new Decoder().frame(slice, 1)).toThrow(/exceeds/); + expect(output).toBe(4096); + } finally { + pushing.mockRestore(); + } +}); + +test("final blocks, malformed input, and a decoder already failed are refused", () => { + const frame = text.encode("final"); + expect(() => new Decoder().frame(deflateRaw(frame), frame.length)).toThrow(/ended/); + const decoder = new Decoder(); + expect(() => decoder.frame(new Uint8Array(64).fill(0xff), 64)).toThrow(); + expect(() => decoder.frame(new Uint8Array(), 0)).toThrow(/already failed/); +}); diff --git a/js/net/src/flate/decoder.ts b/js/net/src/flate/decoder.ts new file mode 100644 index 00000000000..3378a94ba9a --- /dev/null +++ b/js/net/src/flate/decoder.ts @@ -0,0 +1,59 @@ +// The group-scoped moq-flate decoder; applications always read inflated frames. +import { Inflate, Z_SYNC_FLUSH } from "pako"; +import { StreamCode, StreamError } from "../error.ts"; + +const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); + +function malformed(message: string): Error { + return new StreamError(StreamCode.MalformedTrack, { message }); +} + +/** Decode one group's slices in frame order, enforcing each declared size. @internal */ +export class Decoder { + #inflate = new Inflate({ raw: true, chunkSize: 4096 }); + #chunks: Uint8Array[] = []; + #total = 0; + #size = 0; + #failed = false; + + constructor() { + this.#inflate.onData = (chunk) => { + this.#total += chunk.length; + // Throw inside push, so a bomb stops doing work at its declaration plus one + // scratch chunk rather than merely discarding the rest of its output. + if (this.#total > this.#size) throw malformed("inflated frame exceeds its declared size"); + this.#chunks.push(chunk); + }; + } + + /** Inflate a slice to exactly `size` bytes, without preallocating from that declaration. */ + frame(slice: Uint8Array, size: number): Uint8Array { + if (this.#failed) throw malformed("decoder already failed"); + if (!Number.isSafeInteger(size) || size < 0) throw malformed("invalid inflated frame size"); + this.#chunks = []; + this.#total = 0; + this.#size = size; + try { + if (slice.length > 0) { + this.#inflate.push(slice, false); + if (this.#inflate.ended) throw malformed("compressed group ended its DEFLATE stream"); + this.#inflate.push(SYNC_FLUSH_TAIL, Z_SYNC_FLUSH); + if (this.#inflate.err || this.#inflate.ended) throw malformed("malformed DEFLATE slice"); + } + if (this.#total !== size) throw malformed("inflated frame differs from its declared size"); + // Retained frames must own tight buffers rather than pinning scratch storage. + const out = new Uint8Array(this.#total); + let offset = 0; + for (const chunk of this.#chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; + } catch (error) { + this.#failed = true; + throw error; + } finally { + this.#chunks = []; + } + } +} diff --git a/js/net/src/flate/encoder.ts b/js/net/src/flate/encoder.ts new file mode 100644 index 00000000000..85909b244b3 --- /dev/null +++ b/js/net/src/flate/encoder.ts @@ -0,0 +1,74 @@ +// Loaded only by an original publisher opting into track compression. +import { Z_SYNC_FLUSH } from "pako"; +import { Deflate, type DeflateOptions } from "pako/browser/deflate"; + +const DEFAULT_LEVEL = 6; +const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); + +/** Options for an {@link Encoder}. */ +interface EncoderOptions { + /** DEFLATE level, `0..=9` (higher is smaller and slower). Defaults to {@link DEFAULT_LEVEL}. */ + level?: number; +} + +/** + * Encodes a stream's frame payloads into one shared DEFLATE window, one self-delimited slice per + * frame. Hold one per stream; create a new one for each independent stream. + * + * @internal + */ +export class Encoder { + #deflate: Deflate; + #chunks: Uint8Array[] = []; + #total = 0; + + /** Start a fresh encoder with a cold window. */ + constructor(options: EncoderOptions = {}) { + // `raw`: no zlib header/trailer, matching the Rust side and the browser's `deflate-raw`. + // pako types `level` as a literal union; we accept a plain number and narrow here. + const level = (options.level ?? DEFAULT_LEVEL) as DeflateOptions["level"]; + this.#deflate = new Deflate({ raw: true, level }); + this.#deflate.onData = (chunk) => { + const bytes = chunk as Uint8Array; + this.#chunks.push(bytes); + this.#total += bytes.length; + }; + } + + /** + * The most bytes {@link frame} can return for a payload of `len` bytes, at any level and whatever + * the window already holds. + * + * zlib's `deflateBound` for its default window and memory level, which both pako and the Rust + * `moq-flate` use: incompressible input falls back to stored blocks, 5 bytes per 16 KiB. The + * constant covers the block headers and the flush, whose fixed 4-byte marker is stripped anyway. + */ + static bound(len: number): number { + // Division rather than shifts, which wrap past 2^31. + return len + Math.floor(len / 4096) + Math.floor(len / 16384) + Math.floor(len / 33554432) + 13; + } + + /** + * Compress the next frame's `payload`, returning its slice of the stream: the DEFLATE bytes minus + * the fixed sync-flush marker. Empty in yields empty out. Slices must be produced in frame order. + */ + frame(payload: Uint8Array): Uint8Array { + if (payload.length === 0) return payload; + this.#chunks = []; + this.#total = 0; + this.#deflate.push(payload, Z_SYNC_FLUSH); + + // Copy into one tight owned buffer, dropping the trailing sync-flush marker. We can't return + // pako's chunk views: a caller retains the reference and pako backs each chunk with a ~16 KB + // buffer, so a view would pin far more memory than the frame. + const out = new Uint8Array(this.#total - SYNC_FLUSH_TAIL.length); + let offset = 0; + for (const chunk of this.#chunks) { + if (offset >= out.length) break; + const take = Math.min(chunk.length, out.length - offset); + out.set(chunk.subarray(0, take), offset); + offset += take; + } + return out; + } +} From 231a78db6e24ece0f021d648c533454e85d91681 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 18:02:43 -0700 Subject: [PATCH 3/5] feat(net): publish and consume compressed lite-07 tracks Keep model writes synchronous, store slices once per group, and share exact-size inflation across plain readers. Serve older sessions plain frames and exercise Rust/JS relay exchange and lite-06 fallback. Co-Authored-By: GPT-6 --- doc/concept/moq-lite.md | 4 +- js/justfile | 2 +- js/net/bench/compression.ts | 70 ++++++++ js/net/bench/fixtures/compression-consumer.ts | 2 + js/net/bench/frames.ts | 6 +- js/net/bench/lite-varint.ts | 2 +- js/net/src/flate/encoder.ts | 5 +- js/net/src/flate/group.test.ts | 154 ++++++++++++++++++ js/net/src/flate/group.ts | 102 ++++++++++++ js/net/src/group.ts | 103 +++++++++--- js/net/src/internal.ts | 27 ++- js/net/src/lite/compression.test.ts | 73 +++++++++ js/net/src/lite/datagram.test.ts | 6 +- js/net/src/lite/datagram.ts | 34 +++- js/net/src/lite/group.test.ts | 14 +- js/net/src/lite/group.ts | 50 +++--- js/net/src/lite/publisher.ts | 44 +++-- js/net/src/lite/subscriber.ts | 36 ++-- js/net/src/resume.test.ts | 34 ++-- js/net/src/resume.ts | 7 +- js/net/src/track.ts | 65 +++++++- quest/m1/audit-size.md | 4 +- quest/m1/compress/README.md | 1 - quest/m1/compress/js.md | 29 ---- quest/m1/compress/migrate.md | 4 - quest/m1/lite07-finalize.md | 1 - test/interop/README.md | 4 + test/interop/clients/js-native/tail.ts | 23 ++- test/interop/clients/rust/tail.rs | 33 +++- test/interop/interop.sh | 13 +- test/interop/lite-varint.ts | 4 +- 31 files changed, 778 insertions(+), 178 deletions(-) create mode 100644 js/net/bench/compression.ts create mode 100644 js/net/bench/fixtures/compression-consumer.ts create mode 100644 js/net/src/flate/group.test.ts create mode 100644 js/net/src/flate/group.ts create mode 100644 js/net/src/lite/compression.test.ts delete mode 100644 quest/m1/compress/js.md diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index baa7088ea0b..732d6771742 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -289,7 +289,9 @@ JSON and not for media. Each group's frames become one DEFLATE stream compresses alone. Only the work-in-progress moq-lite 07 carries the compressed bytes. A relay forwards them as received and inflates each group once for every other subscriber, moq-transport included, so readers always see the original -frames. Rust publishes and relays compressed tracks; js/net refuses them for now. +frames. Rust and JavaScript publish compressed tracks by setting `compress: true` +on the track's info. Application writes stay synchronous and readers receive plain +payloads; compression is useful for repetitive data, not already compressed media. A relay cancels its upstream subscription once nobody subscribes, but keeps its copy of the track for 30 seconds after the last reader leaves, so a returning diff --git a/js/justfile b/js/justfile index af809fc3df9..f31074429a4 100644 --- a/js/justfile +++ b/js/justfile @@ -55,7 +55,7 @@ test-changed list: coverage package="": python3 ../sh/js/coverage.py '{{ package }}' -# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, or subscribe-drop. +# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, subscribe-drop, or compression. bench name: bun install --frozen-lockfile bun net/bench/{{ name }}.ts diff --git a/js/net/bench/compression.ts b/js/net/bench/compression.ts new file mode 100644 index 00000000000..ef0ad91dc4c --- /dev/null +++ b/js/net/bench/compression.ts @@ -0,0 +1,70 @@ +/** Sweep compressed publishers x plain readers, reporting one shared inflated copy per group. */ +import { gzipSync } from "node:zlib"; +import type { Producer as Group } from "../src/group.ts"; +import { hooks } from "../src/internal.ts"; +import { Producer as Track } from "../src/track.ts"; + +const payload = new TextEncoder().encode('{"value":123,"status":"unchanged"}\n'.repeat(64)); +const frames = 32; +const rounds = 10; +console.log("publishers,readers,publish_us,read_us,stored_bytes,inflated_bytes"); +for (const publishers of [1, 4, 16]) { + for (const readers of [1, 4, 16]) { + let publish = 0; + let read = 0; + let stored = 0; + let inflated = 0; + for (let round = 0; round < rounds; round++) { + const tracks = Array.from({ length: publishers }, () => new Track("bench").accept({ compress: true })); + const groups: Group[] = tracks.map((track) => track.appendGroup()); + const consumers = groups.map((group) => Array.from({ length: readers }, () => group.mirror())); + let started = performance.now(); + for (const group of groups) { + for (let index = 0; index < frames; index++) group.writeFrame({ payload }); + group.close(); + } + publish += performance.now() - started; + stored = groups.reduce((bytes, group) => bytes + hooks.groupCacheBytes(group), 0); + started = performance.now(); + for (const subscribers of consumers) { + let shared: Uint8Array | undefined; + for (const consumer of subscribers) { + for (let index = 0; index < frames; index++) { + const frame = consumer.tryReadFrame(); + if (!frame || frame.payload.length !== payload.length) throw new Error("missing payload"); + if (index === 0) { + if (shared && shared !== frame.payload) throw new Error("fanout duplicated inflation"); + shared = frame.payload; + } + } + consumer.close(); + } + } + read += performance.now() - started; + inflated = groups.reduce((bytes, group) => bytes + hooks.groupCacheBytes(group), 0) - stored; + if (inflated !== publishers * frames * payload.length) throw new Error("inflated bytes scale with readers"); + for (const track of tracks) track.close(); + } + console.log( + [ + publishers, + readers, + ((publish * 1000) / rounds).toFixed(1), + ((read * 1000) / rounds).toFixed(1), + stored, + inflated, + ].join(","), + ); + } +} + +// This entry is an application importing only its track consumer. Synchronous model writes +// share the module, so its bundle includes the encoder as well as the decoder. +const bundle = await Bun.build({ + entrypoints: [new URL("./fixtures/compression-consumer.ts", import.meta.url).pathname], + minify: true, + target: "browser", +}); +if (!bundle.success) throw new AggregateError(bundle.logs, "consumer bundle failed"); +const bytes = new Uint8Array(await bundle.outputs[0].arrayBuffer()); +console.log(`consumer_bundle_bytes=${bytes.length},gzip_bytes=${gzipSync(bytes).length}`); diff --git a/js/net/bench/fixtures/compression-consumer.ts b/js/net/bench/fixtures/compression-consumer.ts new file mode 100644 index 00000000000..b56b3265f56 --- /dev/null +++ b/js/net/bench/fixtures/compression-consumer.ts @@ -0,0 +1,2 @@ +/** Consumer-only entry for measuring the static application graph. @module */ +export { Consumer } from "../../src/track.ts"; diff --git a/js/net/bench/frames.ts b/js/net/bench/frames.ts index eb2ac881bf4..22aa863798b 100644 --- a/js/net/bench/frames.ts +++ b/js/net/bench/frames.ts @@ -52,9 +52,9 @@ const lite: Protocol = { async subscribe() { const open: Open = (stream) => { const producer = new Producer(0); - const done = readFrames(new Reader(stream, undefined, LiteVersion.DRAFT_06), producer, 1_000_000).then(() => - producer.close(), - ); + const done = readFrames(new Reader(stream, undefined, LiteVersion.DRAFT_06), producer, { + scale: 1_000_000, + }).then(() => producer.close()); return { group: Promise.resolve(producer.consume()), done }; }; return { open, close: () => {} }; diff --git a/js/net/bench/lite-varint.ts b/js/net/bench/lite-varint.ts index 4f335e79355..bd2a8182387 100644 --- a/js/net/bench/lite-varint.ts +++ b/js/net/bench/lite-varint.ts @@ -79,7 +79,7 @@ const subscribe: Sample = { const datagram: Sample = { name: "Datagram", encode: (w, version) => w.write(new Datagram(3n, 1_234, 1_234_567_890, new Uint8Array()).encode(version)), - decode: (r, version) => r.readAll().then((data) => Datagram.decode(data, version, () => true)), + decode: (r, version) => r.readAll().then((data) => Datagram.decode(data, version, () => ({ timed: true }))), }; const hop = randomHop(); diff --git a/js/net/src/flate/encoder.ts b/js/net/src/flate/encoder.ts index 85909b244b3..f4d01214921 100644 --- a/js/net/src/flate/encoder.ts +++ b/js/net/src/flate/encoder.ts @@ -1,6 +1,5 @@ -// Loaded only by an original publisher opting into track compression. -import { Z_SYNC_FLUSH } from "pako"; -import { Deflate, type DeflateOptions } from "pako/browser/deflate"; +// Synchronous model writes include the encoder in the static graph. +import { Deflate, type DeflateOptions, Z_SYNC_FLUSH } from "pako"; const DEFAULT_LEVEL = 6; const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); diff --git a/js/net/src/flate/group.test.ts b/js/net/src/flate/group.test.ts new file mode 100644 index 00000000000..8dd65b0614a --- /dev/null +++ b/js/net/src/flate/group.test.ts @@ -0,0 +1,154 @@ +import { expect, spyOn, test } from "bun:test"; +import { Producer as Group, MAX_GROUP_CACHE_BYTES } from "../group.ts"; +import { hooks } from "../internal.ts"; +import { Producer as Track } from "../track.ts"; +import { Decoder } from "./decoder.ts"; +import { Encoder } from "./encoder.ts"; +import { stored } from "./group.ts"; + +const payload = new TextEncoder().encode("one shared compression window ".repeat(100)); + +test("synchronous writers store slices; plain readers share one charged inflated copy", async () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const first = group.mirror(); + const second = group.mirror(); + const raw = group.mirror(); + group.writeFrame({ payload }); + group.writeFrame({ payload }); + group.close(); + const stored = hooks.groupCacheBytes(group); + expect(stored).toBeLessThan(payload.length); + const inflating = spyOn(Decoder.prototype, "frame"); + try { + const slice = await hooks.readGroupFrame(raw, undefined, true); + expect(slice?.inflated).toBe(payload.length); + expect(inflating).toHaveBeenCalledTimes(0); + slice?.complete(); + const a = await first.readFrame(); + const b = await second.readFrame(); + expect(a?.payload).toBe(b?.payload); + expect(a?.payload).toEqual(payload); + expect(inflating).toHaveBeenCalledTimes(1); + expect(hooks.groupCacheBytes(group)).toBe(stored + payload.length); + await first.readFrame(); + await second.readFrame(); + expect(inflating).toHaveBeenCalledTimes(2); + } finally { + inflating.mockRestore(); + track.close(); + } +}); + +test("received slices are forwarded byte-for-byte and exact declarations fail all plain readers", async () => { + const group = new Group(0); + hooks.bindGroupCompression(group, true); + const slice = new Encoder().frame(payload); + hooks.writeGroupFrame(group, { payload: slice }, payload.length + 1); + group.close(); + const raw = await hooks.readGroupFrame(group.mirror(), undefined, true); + expect(raw?.frame.payload).toBe(slice); + raw?.complete(); + const bad = group.mirror(); + await expect(bad.readFrame()).rejects.toThrow(/declared size/); + expect(bad.closed.peek()).toBeInstanceOf(Error); + await expect(group.mirror().readFrame()).rejects.toThrow(/declared size/); +}); + +test("declared plain bytes consume the group budget before any inflation", () => { + const group = new Group(0); + hooks.bindGroupCompression(group, true); + const inflating = spyOn(Decoder.prototype, "frame"); + try { + expect(() => hooks.writeGroupFrame(group, { payload: new Uint8Array() }, MAX_GROUP_CACHE_BYTES + 1)).toThrow(); + hooks.writeGroupFrame(group, { payload: new Uint8Array() }, MAX_GROUP_CACHE_BYTES); + expect(() => hooks.writeGroupFrame(group, { payload: new Uint8Array() }, 1)).toThrow(); + expect(inflating).toHaveBeenCalledTimes(0); + expect(hooks.groupCacheBytes(group)).toBe(0); + } finally { + inflating.mockRestore(); + group.close(); + } +}); + +test("empty frames preserve the window and prewritten groups bind without duplicating plain copies", async () => { + const group = new Group(0); + group.writeFrame({ payload }); + group.writeFrame({ payload: new Uint8Array() }); + const reader = group.mirror(); + const track = new Track("compressed").accept({ compress: true }); + track.writeGroup(group); + group.writeFrame({ payload }); + group.close(); + for (const expected of [payload, new Uint8Array(), payload]) + expect((await reader.readFrame())?.payload).toEqual(expected); + track.close(); +}); + +test("a cut compressed group preserves its committed frames before returning its error", async () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const reader = group.mirror(); + group.writeFrame({ payload }); + const cut = new Error("cut"); + group.close(cut); + expect((await reader.readFrame())?.payload).toEqual(payload); + await expect(reader.readFrame()).rejects.toThrow(cut); + track.close(); +}); + +test("a shared plain read cannot change a later synchronous write's admission", async () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const reader = group.mirror(); + group.writeFrame({ payload: new Uint8Array(MAX_GROUP_CACHE_BYTES) }); + expect((await reader.readFrame())?.payload.length).toBe(MAX_GROUP_CACHE_BYTES); + // The plain-byte budget is full, but another empty frame still fits, regardless of readers. + expect(() => group.writeFrame({ payload: new Uint8Array() })).not.toThrow(); + group.close(); + expect((await reader.readFrame())?.payload.length).toBe(0); + track.close(); +}); + +test("datagrams buffered before accept share one stored and plain form", async () => { + const track = new Track("late-info"); + const a = track.subscribe(); + const b = track.subscribe(); + track.appendDatagram(undefined, payload); + track.accept({ compress: true }); + const first = await a.recvDatagram(); + const second = await b.recvDatagram(); + if (!first || !second) throw new Error("missing datagram"); + expect(first).toBe(second); + expect(stored(first)?.inflated).toBe(payload.length); + expect(first.payload).toEqual(payload); + track.close(); +}); + +test("a malformed compressed datagram is refused before reaching readers", () => { + const track = new Track("compressed").accept({ compress: true }); + const reader = track.subscribe(); + const payload = new Uint8Array(100); + const slice = new Encoder().frame(payload); + expect(() => hooks.writeDatagram(track, { sequence: 0, payload: slice }, payload.length + 1)).toThrow( + /declared size/, + ); + expect(hooks.tryRecvDatagram(reader).kind).toBe("idle"); + track.close(); +}); + +test("refresh measures stored slices without inflating a compressed group", () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const payload = Uint8Array.from({ length: 1024 }, (_, index) => index % 251); + for (let index = 0; index < 3; index++) group.writeFrame({ payload }); + const inflating = spyOn(Decoder.prototype, "frame"); + try { + expect(hooks.overRefresh(group, 1)).toBe(false); + expect(inflating).toHaveBeenCalledTimes(0); + } finally { + inflating.mockRestore(); + group.close(); + track.close(); + } +}); diff --git a/js/net/src/flate/group.ts b/js/net/src/flate/group.ts new file mode 100644 index 00000000000..7e226084069 --- /dev/null +++ b/js/net/src/flate/group.ts @@ -0,0 +1,102 @@ +import type { Frame } from "../group.ts"; +import type { Timestamp } from "../time.ts"; +import { Decoder } from "./decoder.ts"; +import { Encoder } from "./encoder.ts"; + +/** A stored slice and its declared plain size. Package-private wire metadata. */ +export interface Stored { + /** The original compressed bytes. */ + payload: Uint8Array; + /** The exact plain payload length. */ + inflated: number; +} + +// Class identity keeps reactive equality from reading the lazy payload getter. +class SliceFrame implements Frame { + readonly timestamp: Timestamp | undefined; + #read: () => Uint8Array; + constructor(timestamp: Timestamp | undefined, read: () => Uint8Array) { + this.timestamp = timestamp; + this.#read = read; + } + get payload(): Uint8Array { + return this.#read(); + } +} + +const slices = new WeakMap(); + +/** Read a compressed frame's stored form without inflating it. */ +export function stored(frame: Frame): Stored | undefined { + return slices.get(frame); +} + +/** One group's stored form and lazy plain copy, shared by every mirror. */ +export class Compression { + #encoder?: Encoder; + #decoder?: Decoder; + #frames: Stored[] = []; + #plain: Uint8Array[] = []; + #failed?: Error; + #finished = false; + /** Plain bytes admitted against the group budget. */ + declared = 0; + /** Retained compressed payload bytes, separate from its optional plain copy. */ + stored = 0; + /** Retained slice and shared plain bytes. */ + bytes = 0; + + /** Store a local plain frame or a received slice after budget admission. */ + push(frame: Frame, inflated?: number): Frame { + let slice: Stored; + if (inflated === undefined) { + this.#encoder ??= new Encoder(); + slice = { payload: this.#encoder.frame(frame.payload), inflated: frame.payload.length }; + } else slice = { payload: frame.payload, inflated }; + this.declared += slice.inflated; + this.stored += slice.payload.length; + this.bytes += slice.payload.length; + const index = this.#frames.length; + this.#frames.push(slice); + const result = new SliceFrame(frame.timestamp, () => this.#inflate(index)); + slices.set(result, slice); + return result; + } + + #inflate(index: number): Uint8Array { + if (this.#failed) throw this.#failed; + try { + while (this.#plain.length <= index) { + const slice = this.#frames[this.#plain.length]; + this.#decoder ??= new Decoder(); + const plain = this.#decoder.frame(slice.payload, slice.inflated); + this.#plain.push(plain); + this.bytes += plain.length; + } + if (this.#finished && this.#plain.length === this.#frames.length) this.#decoder = undefined; + return this.#plain[index]; + } catch (error) { + this.#failed = error instanceof Error ? error : new Error(String(error)); + this.#decoder = undefined; + throw this.#failed; + } + } + + /** Release cached content on an abort, failing every mirror's late read. */ + abort(error: Error): void { + this.#failed = error; + this.#encoder = undefined; + this.#decoder = undefined; + this.#frames = []; + this.#plain = []; + this.bytes = 0; + this.stored = 0; + } + + /** Release codec scratch once no more frames can arrive. */ + finish(): void { + this.#finished = true; + this.#encoder = undefined; + if (this.#plain.length === this.#frames.length) this.#decoder = undefined; + } +} diff --git a/js/net/src/group.ts b/js/net/src/group.ts index b2f6274f3be..1d4909ade67 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -14,6 +14,7 @@ import { TimestampMismatch, TooFarBehind, } from "./error.ts"; +import { Compression, stored } from "./flate/group.ts"; import { type Expiry, type GroupRecover, hooks, type ReadGroupFrame, type Replacement } from "./internal.ts"; import type { Timestamp } from "./time.ts"; @@ -30,7 +31,7 @@ export const MAX_GROUP_FRAMES = 8192; * converts it into the track's negotiated timescale. */ export interface Frame { - /** The frame payload. */ + /** The plain payload, shared across readers even when the track stores compressed bytes. */ payload: Uint8Array; /** * Presentation timestamp, present exactly when the track has a timescale. A frame on a @@ -69,6 +70,8 @@ class GroupState { start = 0; cacheBytes = 0; + compressed?: boolean; + flate?: Compression; // Whether the owning track has a timescale, so every frame must carry a timestamp (true) // or none may (false). Undefined for a group no track has claimed yet, which takes either. timed?: boolean; @@ -102,7 +105,7 @@ function appendFrame(state: GroupState, frame: Frame) { state.timestamp ??= frame.timestamp; state.latest = frame.timestamp; - state.cacheBytes += frame.payload.byteLength; + state.cacheBytes += stored(frame)?.payload.length ?? frame.payload.byteLength; state.frames.mutate((frames) => { frames.push(frame); }); @@ -185,6 +188,29 @@ export class Producer { } static { + hooks.bindGroupCompression = (group, compressed) => { + const state = group.#state; + if (state.compressed === compressed) return; + if (state.compressed !== undefined) + throw new ProtocolViolation("group compression disagrees with its track"); + if (compressed && state.start !== 0) throw new ProtocolViolation("compressed group lacks its prefix"); + state.compressed = compressed; + if (!compressed) return; + const flate = new Compression(); + state.flate = flate; + const frames = state.frames.peek().map((frame) => flate.push(frame)); + state.frames.set(frames); + state.cacheBytes = flate.stored; + if (flate.stored > MAX_GROUP_CACHE_BYTES) group.#overflow(); + for (const mirror of group.#mirrors ?? []) { + mirror.frames.set(frames.slice(mirror.start - state.start)); + mirror.flate = flate; + mirror.compressed = true; + } + if (state.closed.peek() !== undefined) flate.finish(); + }; + hooks.writeGroupFrame = (group, frame, inflated) => group.#writeFrame(frame, inflated); + hooks.groupCacheBytes = (group) => group.#state.flate?.bytes ?? group.#state.cacheBytes; hooks.groupProgress = (group, fn) => { const state = group.#state; fn(state.total.peek()); @@ -209,7 +235,7 @@ export class Producer { const first = state.frames.peek()[0]; // Without its first frame the cache cannot measure the group, so it asks for nothing. if (state.start !== 0 || first === undefined) return false; - const head = first.payload.byteLength; + const head = stored(first)?.payload.length ?? first.payload.byteLength; return state.cacheBytes - head > head * ratio; }; hooks.limitGroup = (group, frames) => { @@ -257,6 +283,8 @@ export class Producer { const dst = new GroupState(this.sequence); dst.timestamp = this.#state.timestamp; dst.latest = this.#state.latest; + dst.flate = this.#state.flate; + dst.compressed = this.#state.compressed; for (const frame of this.#state.frames.peek()) appendFrame(dst, frame); dst.start = this.#state.start; // The replay only covers what is still buffered, so the count has to come from the @@ -293,13 +321,20 @@ export class Producer { /** Writes a frame to the group. */ writeFrame(frame: Frame) { + this.#writeFrame(frame); + } + + #writeFrame(frame: Frame, inflated?: number) { // Closed first: a write after a clean close at the cap would otherwise take the // overflow path, wipe committed frames, and leave readers with an empty success. if (this.#state.closed.peek() !== undefined) throw new Error("group is closed"); // A frame past the cache cap would make the group exceed its budget by itself, so // accepting it would report success for a write nothing can ever read. Rust rejects // it up front with `Error::FrameTooLarge`; do the same rather than silently dropping it. - if (frame.payload.byteLength > MAX_GROUP_CACHE_BYTES) throw new FrameTooLarge(); + if (frame.payload.byteLength > MAX_GROUP_CACHE_BYTES || (inflated ?? 0) > MAX_GROUP_CACHE_BYTES) + throw new FrameTooLarge(); + if (inflated !== undefined && (!this.#state.flate || !Number.isSafeInteger(inflated) || inflated < 0)) + throw new ProtocolViolation("unexpected inflated frame size"); checkTimed(this.#state.timed, frame); if (this.#limit !== undefined && this.#state.total.peek() >= this.#limit) { throw new Error(`frame ${this.#state.total.peek()} is past the track's end`); @@ -313,21 +348,20 @@ export class Producer { } if ( this.#state.total.peek() >= MAX_GROUP_FRAMES || - this.#state.cacheBytes + frame.payload.byteLength > MAX_GROUP_CACHE_BYTES + ((!this.#state.flate || inflated !== undefined) && + this.#state.cacheBytes + frame.payload.byteLength > MAX_GROUP_CACHE_BYTES) || + (this.#state.flate?.declared ?? this.#state.cacheBytes) + (inflated ?? frame.payload.length) > + MAX_GROUP_CACHE_BYTES ) { - const err = new GroupTooLarge(); - this.#state.frames.set([]); - this.#state.cacheBytes = 0; - if (this.#mirrors) { - for (const mirror of this.#mirrors) { - mirror.frames.set([]); - mirror.cacheBytes = 0; - } - } - this.close(err); - throw err; + this.#overflow(); } + if (this.#state.flate) { + frame = this.#state.flate.push(frame, inflated); + if (this.#state.flate.stored > MAX_GROUP_CACHE_BYTES) { + this.#overflow(); + } + } this.#activity = now(); appendFrame(this.#state, frame); if (commits) { @@ -343,6 +377,19 @@ export class Producer { } } + #overflow(): never { + const error = new GroupTooLarge(); + this.#state.frames.set([]); + this.#state.cacheBytes = 0; + this.#state.flate?.abort(error); + for (const mirror of this.#mirrors ?? []) { + mirror.frames.set([]); + mirror.cacheBytes = 0; + } + this.close(error); + throw error; + } + /** True once the group has been closed. */ get isClosed(): boolean { return this.#state.closed.peek() !== undefined; @@ -351,6 +398,7 @@ export class Producer { /** Closes the group, optionally with an error to abort readers. */ close(abort?: Error) { if (this.#state.closed.peek() !== undefined) return; + this.#state.flate?.finish(); this.#state.closed.set(abort ?? null); if (this.#mirrors) { @@ -534,7 +582,7 @@ export class Consumer { group.#closed = undefined; }; hooks.guardGroup = (group, operation) => group.#guard(operation); - hooks.readGroupFrame = (group, from) => group.#readFramePosition(true, from); + hooks.readGroupFrame = (group, from, compressed) => group.#readFramePosition(true, from, compressed); hooks.evictGroup = (group) => { group.#evict(); }; @@ -609,6 +657,7 @@ export class Consumer { this.#state.start += frames.length; } this.#state.cacheBytes = 0; + this.#state.flate = undefined; this.#state.frames.set([]); } @@ -713,17 +762,29 @@ export class Consumer { return frames; } - #readBufferedFrame(pending = false): ReadGroupFrame | undefined { + #readBufferedFrame(pending = false, compressed = false): ReadGroupFrame | undefined { const frames = this.#bufferedFrames(); const buffered = frames.shift(); if (!buffered) return undefined; + const slice = compressed ? stored(buffered) : undefined; + let frame: Frame; + try { + frame = { payload: slice?.payload ?? buffered.payload, timestamp: buffered.timestamp }; + } catch (error) { + this.#terminal = error instanceof Error ? error : new Error(String(error)); + if (this.#verdict.peek() === undefined) this.#verdict.set(this.#terminal); + this.#closed?.refresh(); + throw this.#terminal; + } + if (frames.length === 0 && this.#state.closed.peek() !== undefined) this.#state.flate = undefined; const sequence = this.#state.start++; if (pending) this.#pendingFrames++; let completed = false; return { sequence, - frame: buffered, + frame, + inflated: slice?.inflated, complete: () => { if (completed) return; completed = true; @@ -814,14 +875,14 @@ export class Consumer { return (await this.#readFramePosition())?.frame; } - async #readFramePosition(pending = false, from = 0): Promise { + async #readFramePosition(pending = false, from = 0, compressed = false): Promise { for (;;) { if (this.#terminal) { this.#abandon(); throw this.#terminal; } - const read = this.#readBufferedFrame(pending); + const read = this.#readBufferedFrame(pending, compressed); if (read) { if (read.sequence < from) { read.complete(); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index a2306e956e0..f5f8b6d2976 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -198,6 +198,8 @@ export interface ReadGroupFrame { sequence: number; /** Frame returned to the publisher. */ frame: Frame; + /** The stored slice's declared plain size, on compressed wire reads. */ + inflated?: number; /** Mark the frame delivered or deliberately skipped by the wire publisher. */ complete(): void; } @@ -507,6 +509,17 @@ export const hooks: { * Throws `TimestampMismatch` if a buffered frame already disagrees. */ bindGroupTimed: (group: GroupProducer, timed: boolean) => void; + /** Admit a datagram in its stored wire form. */ + // private: passthrough + writeDatagram: (producer: Producer, datagram: Datagram, inflated?: number) => void; + /** Bind the group to its immutable track codec. */ + // private: passthrough + bindGroupCompression: (group: GroupProducer, compressed: boolean) => void; + /** Admit a received stored slice without recompressing it. */ + // private: passthrough + writeGroupFrame: (group: GroupProducer, frame: Frame, inflated?: number) => void; + /** Payload footprint including a shared inflated copy, for measurements. */ + groupCacheBytes: (group: GroupProducer) => number; /** * Cap a group at `frames` for a track end inside it, so later writes past it throw. * Throws `ProtocolViolation` if a frame at or past the cap already exists. @@ -540,7 +553,7 @@ export const hooks: { guardGroup: (group: GroupConsumer, operation: () => Promise) => Promise; // pending: /quest/m1/public-model/serve.md /** Read a frame the wire publisher completes (or skips) once written. */ - readGroupFrame: (group: GroupConsumer, from?: number) => Promise; + readGroupFrame: (group: GroupConsumer, from?: number, compressed?: boolean) => Promise; /** Make an evicted mirror terminal while its track timeline still contains it. */ evictGroup: (group: GroupConsumer) => void; /** Attach the origin advertisement of a created broadcast. */ @@ -659,6 +672,18 @@ export const hooks: { replaceGroups: () => { throw new Error("track.ts not loaded"); }, + writeDatagram: () => { + throw new Error("track.ts not loaded"); + }, + bindGroupCompression: () => { + throw new Error("group.ts not loaded"); + }, + writeGroupFrame: () => { + throw new Error("group.ts not loaded"); + }, + groupCacheBytes: () => { + throw new Error("group.ts not loaded"); + }, bindGroupTimed: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/lite/compression.test.ts b/js/net/src/lite/compression.test.ts new file mode 100644 index 00000000000..c1aa151ebd9 --- /dev/null +++ b/js/net/src/lite/compression.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from "bun:test"; +import { Manual, Milli, Nano } from "@moq/clock"; +import { accept } from "../connection/accept.ts"; +import { connect } from "../connection/connect.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Producer } from "../origin.ts"; +import * as Path from "../path.ts"; +import { ALPN_06, ALPN_07_WIP } from "./version.ts"; + +for (const downstream of [ALPN_06, ALPN_07_WIP]) { + test(`compressed publishing on ${downstream}, FETCH and independent datagrams`, async () => { + const clock = new Manual(); + const restore = clock.install(); + const source = new Producer(); + const destination = new Producer(); + const broadcast = source.createBroadcast(Path.from("compressed")); + const track = broadcast.createTrack("data", { compress: true }); + broadcast.announce(); + const peers: { abort(): void }[] = []; + async function link(publish: Producer, consume: Producer, alpn: string) { + const pair = createMockTransportPair(alpn); + const url = new URL("https://localhost/compressed"); + peers.push( + ...(await Promise.all([ + connect({ url, transport: pair.client, consume }), + accept({ url, transport: pair.server, publish: publish.consume() }), + ])), + ); + const announcements = consume.announced(); + try { + while ((await announcements.next())?.kind !== "start") {} + } finally { + announcements.close(); + } + } + try { + await link(source, destination, downstream); + const request = destination.request(Path.from("compressed")); + while (request.active.peek() === undefined) await request.active.changed(); + const active = request.active.peek(); + if (!active) throw new Error("broadcast did not resolve"); + const remote = active.track("data"); + expect((await remote.info()).compress).toBe(downstream === ALPN_07_WIP); + const reader = remote.subscribe({ maxDelay: Milli(1000) }); + const group = track.appendGroup(); + const payloads = [new Uint8Array(4096).fill(7), new Uint8Array(), new Uint8Array(4096).fill(7)]; + for (const payload of payloads) group.writeFrame({ payload }); + group.close(); + const received = await reader.recvGroup(); + if (!received) throw new Error("missing group"); + expect(received.sequence).toBe(0); + for (const payload of payloads) expect((await received.readFrame())?.payload).toEqual(payload); + expect(await received.readFrame()).toBeUndefined(); + const fetched = await remote.fetchGroup(0); + for (const payload of payloads) expect((await fetched.readFrame())?.payload).toEqual(payload); + expect(await fetched.readFrame()).toBeUndefined(); + // Each datagram has a cold window; no lost or reordered predecessor is needed. + for (const payload of [new Uint8Array(100).fill(4), new Uint8Array(100).fill(8)]) { + track.appendDatagram(undefined, payload); + expect((await reader.recvDatagram())?.payload).toEqual(payload); + } + reader.close(); + fetched.close(); + received?.close(); + request.close(); + } finally { + for (const origin of [source, destination]) origin.close(); + for (const peer of peers) peer.abort(); + await clock.advanceAsync(Nano.zero); + restore(); + } + }); +} diff --git a/js/net/src/lite/datagram.test.ts b/js/net/src/lite/datagram.test.ts index 2e2893fe2b7..51bb21180d8 100644 --- a/js/net/src/lite/datagram.test.ts +++ b/js/net/src/lite/datagram.test.ts @@ -8,7 +8,7 @@ const dec = new TextDecoder(); test("datagram body round-trips", async () => { for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { const dg = new Datagram(7n, 42, 1000, enc.encode("hello")); - const decoded = await Datagram.decode(dg.encode(version), version, () => true); + const decoded = await Datagram.decode(dg.encode(version), version, () => ({ timed: true })); expect(decoded.subscribe).toBe(7n); expect(decoded.sequence).toBe(42); expect(decoded.timestamp).toBe(1000); @@ -33,7 +33,7 @@ test("datagram body varints follow the version's encoding", () => { test("datagram body round-trips an empty payload", async () => { const dg = new Datagram(0n, 0, 0, new Uint8Array()); - const decoded = await Datagram.decode(dg.encode(Version.DRAFT_07), Version.DRAFT_07, () => true); + const decoded = await Datagram.decode(dg.encode(Version.DRAFT_07), Version.DRAFT_07, () => ({ timed: true })); expect(decoded.payload.byteLength).toBe(0); }); @@ -43,7 +43,7 @@ test("an untimed datagram body carries no timestamp", async () => { expect(body.byteLength).toBe(7); const decoded = await Datagram.decode(body, Version.DRAFT_07, (subscribe) => { expect(subscribe).toBe(1n); - return false; + return { timed: false }; }); expect(decoded.sequence).toBe(2); expect(decoded.timestamp).toBeUndefined(); diff --git a/js/net/src/lite/datagram.ts b/js/net/src/lite/datagram.ts index be4c983558c..951f13d31e1 100644 --- a/js/net/src/lite/datagram.ts +++ b/js/net/src/lite/datagram.ts @@ -4,11 +4,12 @@ * @module */ import { encodeVarint, Reader } from "../stream.ts"; -import { type Version, wireTimescale } from "./version.ts"; +import { hasCompression, type Version, wireTimescale } from "./version.ts"; /** - * A QUIC datagram body: `subscribe (i) | sequence (i) | [timestamp (i)] | payload (b)`, with the - * timestamp present only when the subscription's track carries one (see `wireTimescale`). + * A QUIC datagram body: `subscribe (i) | sequence (i) | [timestamp (i)] | [inflated (i)] | payload (b)`, with the + * timestamp present only when the subscription's track carries one (see `wireTimescale`), + * and the inflated size only on a compressed lite-07 track. * * The payload runs to the datagram boundary, so unlike a size-prefixed lite message there is no * inner length prefix. Mirrors the Rust `lite::Datagram`; the model counterpart is @@ -23,6 +24,8 @@ export class Datagram { timestamp: number | undefined; /** The frame payload, delimited by the datagram boundary. */ payload: Uint8Array; + /** Plain payload length on a compressed lite-07 track. */ + inflated?: number; constructor(subscribe: bigint, sequence: number, timestamp: number | undefined, payload: Uint8Array) { this.subscribe = subscribe; @@ -38,12 +41,19 @@ export class Datagram { throw new Error(`version ${version} has no untimed datagram`); } + if (this.inflated !== undefined && !hasCompression(version)) + throw new Error("compressed datagram needs lite-07"); + const inflated = this.inflated === undefined ? new Uint8Array() : encodeVarint(this.inflated, version); const subscribe = encodeVarint(this.subscribe, version); const sequence = encodeVarint(this.sequence, version); const timestamp = this.timestamp === undefined ? new Uint8Array() : encodeVarint(this.timestamp, version); const out = new Uint8Array( - subscribe.byteLength + sequence.byteLength + timestamp.byteLength + this.payload.byteLength, + subscribe.byteLength + + sequence.byteLength + + timestamp.byteLength + + inflated.length + + this.payload.byteLength, ); let offset = 0; out.set(subscribe, offset); @@ -52,6 +62,8 @@ export class Datagram { offset += sequence.byteLength; out.set(timestamp, offset); offset += timestamp.byteLength; + out.set(inflated, offset); + offset += inflated.length; out.set(this.payload, offset); return out; } @@ -60,12 +72,20 @@ export class Datagram { * Decode a datagram body from the raw bytes of one QUIC datagram. `timed` reports whether the * track of the given subscribe ID carries a timestamp, which the body itself does not say. */ - static async decode(data: Uint8Array, version: Version, timed: (subscribe: bigint) => boolean): Promise { + static async decode( + data: Uint8Array, + version: Version, + framing: (subscribe: bigint) => { timed: boolean; compressed?: boolean }, + ): Promise { const r = new Reader(undefined, data, version); const subscribe = await r.u62(); const sequence = await r.u53(); - const timestamp = timed(subscribe) ? await r.u53() : undefined; + const { timed, compressed } = framing(subscribe); + const timestamp = timed ? await r.u53() : undefined; + const inflated = hasCompression(version) && compressed ? await r.u53() : undefined; const payload = await r.readAll(); - return new Datagram(subscribe, sequence, timestamp, payload); + const result = new Datagram(subscribe, sequence, timestamp, payload); + result.inflated = inflated; + return result; } } diff --git a/js/net/src/lite/group.test.ts b/js/net/src/lite/group.test.ts index d073dffe967..b0f36716873 100644 --- a/js/net/src/lite/group.test.ts +++ b/js/net/src/lite/group.test.ts @@ -37,7 +37,7 @@ test("every frame in a chunk reaches the reader before it wakes", async () => { const producer = new Producer(0); const consumer = producer.consume(); const frames = Array.from({ length: 10 }, (_, index) => ({ delta: 1, payload: [index] })); - const done = readFrames(streamOf([encode(frames)]), producer, SCALE); + const done = readFrames(streamOf([encode(frames)]), producer, { scale: SCALE }); expect((await consumer.readFrame())?.payload).toEqual(new Uint8Array([0])); expect(consumer.frameCount).toBe(10); @@ -57,7 +57,7 @@ test("frames split at every byte keep their payloads and timestamps", async () = const bytes = encode(frames); const chunks = Array.from(bytes, (byte) => new Uint8Array([byte])); - await readFrames(streamOf(chunks), producer, SCALE); + await readFrames(streamOf(chunks), producer, { scale: SCALE }); producer.close(); let ts = 0; @@ -73,7 +73,7 @@ test("frames split at every byte keep their payloads and timestamps", async () = test("frames without a timescale carry no timestamp prefix", async () => { const producer = new Producer(0); const consumer = producer.consume(); - await readFrames(streamOf([encode([{ payload: [7] }, { payload: [8, 9] }])]), producer, 0); + await readFrames(streamOf([encode([{ payload: [7] }, { payload: [8, 9] }])]), producer, { scale: 0 }); producer.close(); expect((await consumer.readFrame())?.payload).toEqual(new Uint8Array([7])); @@ -84,16 +84,16 @@ test("frames without a timescale carry no timestamp prefix", async () => { test("a stream that ends inside a frame rejects", async () => { const producer = new Producer(0); const bytes = encode([{ delta: 1, payload: [1, 2, 3] }]); - await expect(readFrames(streamOf([bytes.subarray(0, bytes.byteLength - 1)]), producer, SCALE)).rejects.toThrow( - "unexpected end of stream", - ); + await expect( + readFrames(streamOf([bytes.subarray(0, bytes.byteLength - 1)]), producer, { scale: SCALE }), + ).rejects.toThrow("unexpected end of stream"); }); test("stops once the group closes", async () => { const producer = new Producer(0); // Never ends: only the close can stop the read. const stream = new Reader(new ReadableStream(), undefined, VERSION); - const done = readFrames(stream, producer, SCALE); + const done = readFrames(stream, producer, { scale: SCALE }); producer.close(); await done; }); diff --git a/js/net/src/lite/group.ts b/js/net/src/lite/group.ts index b7e57567db1..a8350557828 100644 --- a/js/net/src/lite/group.ts +++ b/js/net/src/lite/group.ts @@ -1,5 +1,7 @@ import { race } from "@moq/signals"; -import type * as netGroup from "../group.ts"; +import { FrameTooLarge } from "../error.ts"; +import * as netGroup from "../group.ts"; +import { hooks } from "../internal.ts"; import type { Cursor, Reader, Writer } from "../stream.ts"; import * as Time from "../time.ts"; import * as Message from "./message.ts"; @@ -69,32 +71,40 @@ function unzigzag(v: bigint): bigint { return (v >> 1n) ^ -(v & 1n); } -/** - * A synchronous decode for one frame of a group or FETCH response stream. - * - * A non-zero `scale` means every frame is prefixed with a zigzag-delta timestamp (the lite-05 - * FRAME format), decoded into a Timestamp at that scale. Scale 0 (pre-lite-05) carries no - * timestamp, so frames arrive untimed. - */ -export function frameDecoder(scale: number): (c: Cursor) => netGroup.Frame { - if (scale === 0) { - return (c) => ({ payload: c.read(c.u53()) }); - } +/** A decoded frame whose payload may still be a stored slice. */ +export interface WireFrame extends netGroup.Frame { + /** The declared plain size on a compressed track. */ + inflated?: number; +} - const timescale = Time.Timescale(scale); +/** Decode one frame, with timestamp deltas when scaled and declared plain sizes when compressed. */ +export function frameDecoder(scale: number, compressed = false): (c: Cursor) => WireFrame { + const timescale = scale === 0 ? undefined : Time.Timescale(scale); let prevTs = 0n; return (c) => { - const delta = unzigzag(c.u62()); - const payload = c.read(c.u53()); - // After the last read, so a decode that ran short and gets retried adds the delta once. + const delta = timescale === undefined ? 0n : unzigzag(c.u62()); + const inflated = compressed ? c.u53() : undefined; + const size = c.u53(); + if (size > netGroup.MAX_GROUP_CACHE_BYTES || (inflated ?? 0) > netGroup.MAX_GROUP_CACHE_BYTES) + throw new FrameTooLarge(); + const payload = c.read(size); + // Commit the delta only after the last read: an incomplete decode gets retried. prevTs += delta; - return { payload, timestamp: new Time.Timestamp(Number(prevTs), timescale) }; + return { + payload, + inflated, + timestamp: timescale === undefined ? undefined : new Time.Timestamp(Number(prevTs), timescale), + }; }; } /** Write a group stream's frames into `producer` until the stream ends or the producer closes. */ -export async function readFrames(stream: Reader, producer: netGroup.Producer, scale: number): Promise { - const decode = frameDecoder(scale); +export async function readFrames( + stream: Reader, + producer: netGroup.Producer, + { scale, compress = false }: { scale: number; compress?: boolean }, +): Promise { + const decode = frameDecoder(scale, compress); for (;;) { // Every frame already buffered is written without an await, so the reader wakes once @@ -102,6 +112,6 @@ export async function readFrames(stream: Reader, producer: netGroup.Producer, sc // that closes first has already closed (or aborted) this group through its cache. const frame = stream.tryDecode(decode) ?? (await race([stream.decodeMaybe(decode), producer.closed])); if (!frame || frame instanceof Error) return; - producer.writeFrame(frame); + hooks.writeGroupFrame(producer, frame, frame.inflated); } } diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 24d29c8d2aa..d739973d339 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -8,6 +8,7 @@ import type * as broadcast from "../broadcast.ts"; import { Withdrawal } from "../connection/withdrawal.ts"; import * as DatagramStream from "../datagram_stream.ts"; import { error, NotFound, ProtocolViolation, reason, StreamCode, StreamError, unauthorized } from "../error.ts"; +import { stored } from "../flate/group.ts"; import type * as group from "../group.ts"; import { Cost, type Hop, type Route, routesEqual } from "../hop.ts"; import { finalSequence, hiddenBelow, hooks, insertRun, presented, type Run, runGaps } from "../internal.ts"; @@ -42,6 +43,7 @@ import { hasAnnounceId, hasAnnounceOk, hasAnnounceRestart, + hasCompression, hasDatagrams, hasLargest, hasProbeRtt, @@ -101,6 +103,8 @@ interface RunGroup { /** The track's wire timescale, applied to every frame timestamp, or undefined to send none. */ timescale: Timescale | undefined; + /** Whether to serve stored slices with their declared plain sizes. */ + compress: boolean; /** The subscription's ranking, which this stream joins for as long as it runs. */ priority: Priority; @@ -157,6 +161,8 @@ type ServeGroup = { sub: bigint; /** The track's wire timescale, which every frame timestamp is converted to, or undefined to send none. */ timescale: Timescale | undefined; + /** Whether to serve stored slices with their declared plain sizes. */ + compress: boolean; /** First frame to send; anything below it was excluded by the subscription. */ start: number; /** Last frame to send (inclusive), or undefined for the rest of the group. */ @@ -855,6 +861,7 @@ export class Publisher { try { // Older drafts carry no timestamps. let timescale: Timescale | undefined; + let compress = false; if (hasTrackStream(this.version)) { // Lite-05+ accepts implicitly: no SUBSCRIBE_OK (the immutable @@ -868,6 +875,7 @@ export class Publisher { // stream. const info = await untilAborted(track.info(), left.signal); timescale = wireTimescale(this.version, info.timescale); + compress = hasCompression(this.version) && !!info.compress; } else { // Older drafts acknowledge with SUBSCRIBE_OK and stream frames verbatim. const ok = new SubscribeOk({ @@ -914,6 +922,7 @@ export class Publisher { sub: msg.id, broadcast: msg.broadcast, timescale, + compress, served, bounds: { startGroup: msg.startGroup, @@ -1043,6 +1052,7 @@ export class Publisher { } await this.#runFetchGroup(group, stream.writer, { timescale: wireTimescale(this.version, info.timescale), + compress: info.compress, start: msg.startFrame, end: msg.endFrame, }); @@ -1082,11 +1092,12 @@ export class Publisher { sub: bigint; broadcast: Path.Valid; timescale: Timescale | undefined; + compress: boolean; served: Run[] | undefined; bounds: FrameBounds; }, ): Promise { - const { sub, broadcast, timescale, served, bounds } = serving; + const { sub, broadcast, timescale, compress, served, bounds } = serving; // Lite-05+ resolves the range on the subscribe stream: SUBSCRIBE_START once the // first group is known, SUBSCRIBE_END when the track finishes. const emitRange = hasTrackStream(this.version); @@ -1327,6 +1338,7 @@ export class Publisher { sub, group, timescale, + compress, priority, unsubscribed, start: range.start, @@ -1444,6 +1456,7 @@ export class Publisher { maxAge: info.maxAge, // `#serveGroup` emits each frame converted to it. timescale: info.timescale ?? undefined, + compress: hasCompression(this.version) && !!info.compress, }); })(); @@ -1500,7 +1513,10 @@ export class Publisher { // Convert the timestamp to the track's advertised timescale, matching #serveGroup. const ts = timescale === undefined ? undefined : wireTime(datagram.timestamp, timescale); - const body = new DatagramMessage(sub, datagram.sequence, ts, datagram.payload).encode(this.version); + const slice = hasCompression(this.version) ? stored(datagram) : undefined; + const message = new DatagramMessage(sub, datagram.sequence, ts, slice?.payload ?? datagram.payload); + message.inflated = slice?.inflated; + const body = message.encode(this.version); // No group fallback: drop anything that doesn't fit a single datagram. if (body.byteLength > maxSize) { @@ -1524,21 +1540,24 @@ export class Publisher { async #runFetchGroup( group: group.Consumer, stream: Writer, - { timescale, start: startFrame, end: endFrame }: ServeFetch, + { timescale, compress, start: startFrame, end: endFrame }: ServeFetch, ) { // The response carries no header, so the receiver numbers the first frame it gets // as `startFrame`. Skipping the head here is the only thing keeping those numbers // honest; a group that ends before we reach it can't be served at all. for (let i = 0; i < startFrame; i++) { - if (!(await race([group.readFrame(), stream.closed]))) { + const skipped = await race([hooks.readGroupFrame(group, undefined, compress), stream.closed]); + if (!skipped) { throw new Error(`fetch group ended at frame ${i}, before the requested start ${startFrame}`); } + skipped.complete(); } let prevTs = 0n; for (let index = startFrame; endFrame === undefined || index <= endFrame; index++) { - const frame = await race([group.readFrame(), stream.closed]); - if (!frame) break; + const read = await race([hooks.readGroupFrame(group, undefined, compress), stream.closed]); + if (!read) break; + const frame = read.frame; if (timescale !== undefined) { const ts = BigInt(wireTime(frame.timestamp, timescale)); @@ -1546,8 +1565,10 @@ export class Publisher { prevTs = ts; } + if (read.inflated !== undefined) await stream.u53(read.inflated); await stream.u53(frame.payload.byteLength); - await stream.write(frame.payload); + if (frame.payload.length > 0) await stream.write(frame.payload); + read.complete(); } } @@ -1592,7 +1613,7 @@ export class Publisher { * @internal */ async #serveGroup(stream: Writer, options: RunGroup) { - const { sub, group, timescale, priority, start: startFrame, end: endFrame } = options; + const { sub, group, timescale, compress, priority, start: startFrame, end: endFrame } = options; // This model holds whole groups, so frame `startFrame` is always reachable unless // the group ends first. Declaring it up front keeps the stream self-describing. const msg = new GroupMessage({ subscribe: sub, sequence: group.sequence, frameStart: startFrame }); @@ -1614,7 +1635,7 @@ export class Publisher { let reached = startFrame === 0; for (;;) { - const read = await race([hooks.readGroupFrame(group), stream.closed]); + const read = await race([hooks.readGroupFrame(group, undefined, compress), stream.closed]); if (!read) { // The group ended before the frame the subscriber asked to start // at, so this publisher can't serve the range at all. FINning here @@ -1639,8 +1660,11 @@ export class Publisher { prevTs = ts; } + const inflated = read.inflated; + if (inflated !== undefined) await hooks.guardGroup(group, () => stream.u53(inflated)); await hooks.guardGroup(group, () => stream.u53(read.frame.payload.byteLength)); - await hooks.guardGroup(group, () => stream.write(read.frame.payload)); + if (read.frame.payload.length > 0) + await hooks.guardGroup(group, () => stream.write(read.frame.payload)); } finally { read.complete(); } diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 48e8d18d05e..a54a1853564 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -35,7 +35,7 @@ import { import { Datagram as DatagramMessage } from "./datagram.ts"; import { closeViolation } from "./error.ts"; import { Fetch as FetchMessage } from "./fetch.ts"; -import { frameDecoder, type Group as GroupMessage, readFrames } from "./group.ts"; +import { frameDecoder, type Group as GroupMessage, readFrames, type WireFrame } from "./group.ts"; import { Probe } from "./probe.ts"; import { ProbeLevel, type Setup } from "./setup.ts"; import { StreamId } from "./stream.ts"; @@ -101,6 +101,7 @@ interface SubscribeEntry { // The write side: incoming GROUP streams are routed here. The application reads // the matching track.Subscriber it got from broadcast.Consumer.subscribe. track: track.Producer; + compress: boolean; // Per-frame timestamp scale (0 = none). undefined until it's known (from TRACK_INFO // on lite-05+, or implicit defaults on older drafts). A non-zero value means each // frame on the group stream is prefixed with a zigzag-delta timestamp varint that @@ -841,6 +842,7 @@ export class Subscriber { state.cancel.signal.throwIfAborted(); let producer: track.Producer; let drainOk = false; + let compress = false; if (hasTrackStream(this.version)) { // Fetch the immutable properties once via the TRACK stream. @@ -856,6 +858,7 @@ export class Subscriber { // register it again or send its SUBSCRIBE. state.cancel.signal.throwIfAborted(); producer = request.accept(this.#toModelInfo(info)); + compress = info.compress; timescale.set(info.timescale ?? 0); } else { // Older drafts negotiate nothing per-track: verbatim frames with no timeline. @@ -868,6 +871,7 @@ export class Subscriber { // Register before opening SUBSCRIBE so a racing GROUP stream finds the entry. const entry: SubscribeEntry = { track: producer, + compress, timescale, // The effective max delay is the stopgap grace: the wrong clock (it bounds // presentation-time drift), but it is how long the subscriber was willing to wait @@ -914,8 +918,6 @@ export class Subscriber { await stream.writer.u53(StreamId.Track); await new TrackMessage(broadcast, track, epoch).encode(stream.writer, this.version); const info = await TrackInfo.decode(stream.reader, this.version); - // Until js/net inflates, a compressed track is refused rather than read as garbage. - if (info.compress) throw new Error("compressed tracks are not supported"); return { info, stream }; }, signal, @@ -968,6 +970,7 @@ export class Subscriber { maxAge: info.maxAge === undefined ? undefined : Milli(info.maxAge), priority: info.priority, refresh: info.refresh, + compress: info.compress, }; } @@ -1104,7 +1107,8 @@ export class Subscriber { throw err; } - void this.#runFetchResponse(accepted.stream, group, accepted.info.timescale ?? 0); + hooks.bindGroupCompression(group, accepted.info.compress); + void this.#runFetchResponse(accepted.stream, group, accepted.info); } catch (err: unknown) { group.close(error(err)); throw err; @@ -1159,9 +1163,9 @@ export class Subscriber { // Read the FETCH response (bare frames, zigzag-delta-timestamped unless `timescale` is 0) // into the group, then FIN. A stream-level failure aborts the group so its reader observes // the gap. - async #runFetchResponse(stream: Stream, group: netGroup.Producer, timescale: number): Promise { + async #runFetchResponse(stream: Stream, group: netGroup.Producer, info: TrackInfo): Promise { try { - const decode = frameDecoder(timescale); + const decode = frameDecoder(info.timescale ?? 0, info.compress); // Serve until the stream FINs, the group closes, or every reader leaves. A group can // stay open indefinitely (a catalog or JSON stream), so an abandoned fetch is stopped by @@ -1175,7 +1179,7 @@ export class Subscriber { .then((): typeof idle => idle); // A decode consumes its frame whenever it lands, so one outstanding across a re-arm is // kept and awaited again rather than abandoned with its frame. - let pending: Promise | undefined; + let pending: Promise | undefined; for (;;) { // Buffered frames are written without an await, as in a group stream. let frame = pending === undefined ? stream.reader.tryDecode(decode) : undefined; @@ -1200,7 +1204,7 @@ export class Subscriber { if (!next) break; frame = next; } - group.writeFrame(frame); + hooks.writeGroupFrame(group, frame, frame.inflated); } group.close(); @@ -1453,7 +1457,10 @@ export class Subscriber { scale = timescale.peek(); } - await readFrames(stream, producer, scale); + const compress = entry.compress; + if (compress && group.frameStart !== 0) + throw new StreamError(StreamCode.TooFarBehind, { message: "compressed group lacks its prefix" }); + await readFrames(stream, producer, { scale, compress }); producer.close(); stream.stop(StreamCode.Cancel); @@ -1515,11 +1522,10 @@ export class Subscriber { // datagram (best-effort) if the subscription is unknown/closed or its timescale isn't resolved. async #routeDatagram(payload: Uint8Array): Promise { // An unknown or unresolved subscription's datagram is dropped below, however it parses. - const dg = await DatagramMessage.decode( - payload, - this.version, - (id) => !!this.#subscribes.get(id)?.timescale.peek(), - ); + const dg = await DatagramMessage.decode(payload, this.version, (id) => ({ + timed: !!this.#subscribes.get(id)?.timescale.peek(), + compressed: this.#subscribes.get(id)?.compress ?? false, + })); const entry = this.#subscribes.get(dg.subscribe); if (!entry) return; // Unknown or already-closed subscription. @@ -1533,7 +1539,7 @@ export class Subscriber { dg.timestamp === undefined ? undefined : new Time.Timestamp(dg.timestamp, Time.Timescale(scale)); // A datagram's sequence is never owed a stream, so it never holds the tail open. entry.tail.account(dg.sequence, dg.sequence + 1); - entry.track.insertDatagram(dg.sequence, timestamp, dg.payload); + hooks.writeDatagram(entry.track, { sequence: dg.sequence, timestamp, payload: dg.payload }, dg.inflated); } /** diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index fe7ac399f61..8a0c885cc82 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -705,21 +705,25 @@ for (const fails of [true, false]) { }); } -test("track info keeps one answer across routes", async () => { - const front = new Front(); - const a = new BroadcastProducer(); - const b = new BroadcastProducer(); - a.createTrack("seq", { priority: 1 }); - b.createTrack("seq", { priority: 2 }); - front.serve(a.consume()); - expect((await front.consumer.track("seq").info()).priority).toBe(1); - front.serve(b.consume()); - await expect(front.consumer.track("seq").info()).rejects.toThrow("changed its properties"); - - front.consumer.close(); - a.close(); - b.close(); -}); +for (const [field, first, second] of [ + ["priority", { priority: 1 }, { priority: 2 }], + ["compression", { compress: true }, { compress: false }], +] as const) { + test(`track info keeps one ${field} answer across routes`, async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + a.createTrack("seq", first); + b.createTrack("seq", second); + front.serve(a.consume()); + expect(await front.consumer.track("seq").info()).toMatchObject(first); + front.serve(b.consume()); + await expect(front.consumer.track("seq").info()).rejects.toThrow("changed its properties"); + front.consumer.close(); + a.close(); + b.close(); + }); +} test("a datagram reader lets go of the copies a swap replaced", async () => { const front = new Front(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 5d848c17aea..af48d221c3f 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -89,7 +89,12 @@ function verdict(err: Error): boolean { /** Two copies claiming the same content must agree on what it is. */ function compatible(a: track.Info, b: track.Info): boolean { - return a.timescale === b.timescale && a.maxAge === b.maxAge && a.priority === b.priority; + return ( + a.timescale === b.timescale && + a.maxAge === b.maxAge && + a.priority === b.priority && + !!a.compress === !!b.compress + ); } /** diff --git a/js/net/src/track.ts b/js/net/src/track.ts index e9ca1a50c02..b53adb23e35 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -17,6 +17,7 @@ import { TooFarBehind, toStreamCode, } from "./error.ts"; +import { Compression } from "./flate/group.ts"; import { type Frame, type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { type DropRun, @@ -97,6 +98,8 @@ export interface Location { * (resolved from the wire TRACK_INFO on lite-05+). They map 1:1 onto TRACK_INFO. */ export interface Info { + /** Compress frame payloads per group on lite-07; other sessions receive plain frames. Defaults to false. */ + compress?: boolean; /** * Units per second for this track's frame timestamps, or omitted (`undefined` or `null`, * which mean the same) for an untimed track, whose frames and datagrams carry no timestamp. @@ -122,7 +125,7 @@ export interface Info { /** Tie-break priority between subscriptions of equal subscriber priority (`0..=255`, higher first). Defaults to `127`. */ priority: number; /** - * How many times its first frame a group's later bytes may grow before a joining + * How many times its first frame a group's later stored bytes may grow before a joining * subscriber asks for a new group, or omitted to never ask. A join against a newest group * whose bytes after its first frame exceed this many times the first frame moves * {@link Demand.wanted} past it, so a late joiner never downloads much more than a @@ -157,7 +160,10 @@ function priorityByte(value: number): number { /** Fill in any unset {@link Info} fields with their defaults. */ export function infoDefaults(info: Partial = {}): Info { + if (info.compress !== undefined && typeof info.compress !== "boolean") + throw new TypeError("compress must be a boolean"); return { + compress: info.compress ?? false, timescale: info.timescale == null ? undefined : Timescale(info.timescale), maxAge: info.maxAge === undefined ? undefined : wireMillis("maxAge", info.maxAge), priority: priorityByte(info.priority ?? DEFAULT_PRIORITY), @@ -749,6 +755,7 @@ export class Producer { } static { + hooks.writeDatagram = (producer, datagram, inflated) => producer.#insertDatagram(datagram, inflated); hooks.producerLargest = (producer) => producer.#state.largest; hooks.joinProducer = (producer) => producer.#join(); hooks.endTrack = (producer, end) => producer.#endPending(end); @@ -892,7 +899,10 @@ export class Producer { // Anything written before accept() must match too. A mismatch closes the track, so // nothing waiting on info() hangs on a half-bound cache. try { - for (const { group } of this.#cache) hooks.bindGroupTimed(group, timed); + for (const { group } of this.#cache) { + hooks.bindGroupTimed(group, timed); + hooks.bindGroupCompression(group, resolved.compress ?? false); + } for (const sink of this.#sinks) { for (const datagram of sink.datagrams.peek()) { if ((datagram.timestamp !== undefined) !== timed) throw new TimestampMismatch(); @@ -904,7 +914,25 @@ export class Producer { } this.#state.info.set(resolved); // Propagate to any sink handed out before accept (the on-demand path). - for (const sink of this.#sinks) sink.info.set(resolved); + const compressed = new Map(); + for (const sink of this.#sinks) { + if (resolved.compress) { + sink.datagrams.mutate((datagrams) => { + for (let index = 0; index < datagrams.length; index++) { + const datagram = datagrams[index]; + let stored = compressed.get(datagram); + if (!stored) { + const flate = new Compression(); + stored = Object.assign(flate.push(datagram), { sequence: datagram.sequence }); + flate.finish(); + compressed.set(datagram, stored); + } + datagrams[index] = stored; + } + }); + } + sink.info.set(resolved); + } this.#updateSubscription(); return this; } @@ -1322,7 +1350,10 @@ export class Producer { // Hold a group to this track's timedness, once the track has committed it. #bind(group: GroupProducer): void { const info = this.#state.info.peek(); - if (info) hooks.bindGroupTimed(group, info.timescale != null); + if (info) { + hooks.bindGroupTimed(group, info.timescale != null); + hooks.bindGroupCompression(group, info.compress ?? false); + } } // Refuse a datagram whose timedness disagrees with this track's. @@ -1433,7 +1464,7 @@ export class Producer { if (payload.byteLength > MAX_DATAGRAM_BYTES) throw new Error("datagram payload too large"); this.#checkDatagram(timestamp); - this.#publishDatagram({ sequence, timestamp, payload }); + this.#insertDatagram({ sequence, timestamp, payload }); return sequence; } @@ -1445,11 +1476,29 @@ export class Producer { * {@link appendDatagram} apply. Most origin publishers want {@link appendDatagram} instead. */ insertDatagram(sequence: number, timestamp: Timestamp | undefined, payload: Uint8Array) { + this.#insertDatagram({ sequence, timestamp, payload }); + } + + #insertDatagram(datagram: Datagram, inflated?: number): void { + const { sequence, timestamp, payload } = datagram; this.#writable(sequence); - if (payload.byteLength > MAX_DATAGRAM_BYTES) throw new Error("datagram payload too large"); this.#checkDatagram(timestamp); - - this.#publishDatagram({ sequence, timestamp, payload }); + if (payload.length > MAX_DATAGRAM_BYTES || (inflated ?? 0) > MAX_DATAGRAM_BYTES) + throw new Error("datagram payload too large"); + if ( + inflated !== undefined && + (!Number.isSafeInteger(inflated) || inflated < 0 || !this.#state.info.peek()?.compress) + ) + throw new ProtocolViolation("unexpected inflated datagram size"); + if (this.#state.info.peek()?.compress) { + const flate = new Compression(); + const frame = flate.push(datagram, inflated); + // A bad best-effort datagram is dropped on arrival, before any reader receives it. + if (inflated !== undefined) void frame.payload; + flate.finish(); + datagram = Object.assign(frame, { sequence }); + } + this.#publishDatagram(datagram); } /** diff --git a/quest/m1/audit-size.md b/quest/m1/audit-size.md index 9e13fa78d66..9f1e4f1f07a 100644 --- a/quest/m1/audit-size.md +++ b/quest/m1/audit-size.md @@ -29,7 +29,7 @@ It needs the Linux release builds, which still fail to link ([moq-dev/moq#6090](https://github.com/moq-dev/moq/pull/6090) was open on 2026-10-21). If it has, its history replaces the local method below. Then re-attribute the native rows for the erasure quest, and measure `@moq/net` -once [compressed tracks in js/net](/quest/m1/compress/js.md) moves `pako` in. +with the track compression codec in `js/net`. ### Baselines @@ -90,7 +90,7 @@ Leads: - `pako` costs `watch` and `publish` 12.8 KB gzip through `@moq/json`. A player only inflates: `Inflate` alone is 9.9 KB, and the browser's `DecompressionStream("deflate-raw")` would be free if it yields each - sync-flushed frame. Carried into [compressed tracks in js/net](/quest/m1/compress/js.md). + sync-flushed frame. Synchronous track writes retain the static encoder; the maintainer accepted its measured 3 KB gzip cost. - `zod/mini` is 9 KB raw of `@moq/net`'s first load for one branded bigint (`HopSchema`), and `@moq/hang` imports it as both `zod/mini` and `@zod/mini`. Not worth a quest alone. diff --git a/quest/m1/compress/README.md b/quest/m1/compress/README.md index 173f88607ce..4b23eb0da02 100644 --- a/quest/m1/compress/README.md +++ b/quest/m1/compress/README.md @@ -81,7 +81,6 @@ This README owns the end-to-end proof and the guide once every child lands: ## Required - [IETF extension](/quest/m1/compress/ietf.md) - moq-transport draft-16+ peers that negotiate the extension exchange compressed tracks -- [JavaScript](/quest/m1/compress/js.md) - js/net publishes and consumes compressed tracks on lite-07 - [Migrate](/quest/m1/compress/migrate.md) - catalogs, stats, and timelines publish one compressed track; moq-flate, @moq/flate, and every `.z` name are gone ## Related diff --git a/quest/m1/compress/js.md b/quest/m1/compress/js.md deleted file mode 100644 index 02c70259d99..00000000000 --- a/quest/m1/compress/js.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] Compressed tracks in js/net - -## Goal - -A JS publisher sets `compress` on a track's `Info` and js/net DEFLATEs each -group on lite-07, and a JS consumer reads a compressed lite-07 track as plain -frames. On any other session a JS publisher sends plain frames without the -flag, and the refusal Core added is gone. - -## Plan - -See the [epic](/quest/m1/compress/README.md) for the decisions. - -- Move the codec from `js/flate/src/codec.ts` into js/net, crate-private. - `@moq/flate` is deleted in [Migrate](/quest/m1/compress/migrate.md). -- Mirror the Rust model: store compressed slices, one inflated copy per - group shared by plain readers, the declared size enforced while inflating. -- Docs inline beside the Rust ones in `doc/concept`. -- Size: `pako` is 12.8 KB gzip of the `watch` and `publish` bundles today, - and moving it into js/net puts it in every first load. A player only - inflates: keep deflate out of a consumer's static graph (`Inflate` alone is - 9.9 KB), or use `DecompressionStream("deflate-raw")` if it yields each - sync-flushed frame. Measure with `sh/size/bundles.ts` - ([size audit](/quest/m1/audit-size.md)). - -Tests: Rust and JS publish compressed tracks to each other over lite-07 -through moq-relay; a JS publisher on lite-06 sends plain frames. - -Public API: `Info.compress` in `@moq/net`. Wire: none beyond Core. diff --git a/quest/m1/compress/migrate.md b/quest/m1/compress/migrate.md index 1cb2d576385..68ad41cf3ae 100644 --- a/quest/m1/compress/migrate.md +++ b/quest/m1/compress/migrate.md @@ -50,7 +50,3 @@ See the [epic](/quest/m1/compress/README.md) for the decisions. Public API: removals listed above. Wire: the hang catalog loses `compression` and the `.z` tracks, in its draft too; moq-net none. - -## Required - -- [JavaScript](/quest/m1/compress/js.md) - js/publish and js/watch need the JS side before they drop `.z` diff --git a/quest/m1/lite07-finalize.md b/quest/m1/lite07-finalize.md index 968fa4ab57d..e6ab172d277 100644 --- a/quest/m1/lite07-finalize.md +++ b/quest/m1/lite07-finalize.md @@ -86,7 +86,6 @@ is published; older versions are unchanged. - [Routes and announces](/quest/m1/cluster-routing/routes.md) - lite-07 carries the route layer: ROUTE per origin node and path-less ANNOUNCE, with the hop list gone - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - SUBSCRIBE carries ranges and an order and lite FETCH is gone, in Rust and JS - [Announcement takeover](/quest/m1/announce-takeover.md) - ANNOUNCE_TAKEOVER preserves a proven instance under a new independent route -- [Compressed tracks in js/net](/quest/m1/compress/js.md) - JS reads and writes compressed lite-07 tracks instead of refusing them - [TRACKS in js/net](/quest/m1/archive/tracks-js.md) - JS speaks lite-07's Tracks Stream, which Rust already does ## Related diff --git a/test/interop/README.md b/test/interop/README.md index f84960facdb..9b9c55aabc6 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -430,3 +430,7 @@ JS publisher sends. drafts 14 through 22 through `track-status.ts`, and Rust decodes each `TRACK_STATUS_OK` and the refusal for a missing broadcast the way its subscriber reads them. + +The finite-tail matrix also publishes compressed tracks from Rust and native JS through +the relay, in both directions on lite-07 and lite-06. It checks multiple frames in each +DEFLATE window, empty frames, the plain fallback and the received track's codec flag. diff --git a/test/interop/clients/js-native/tail.ts b/test/interop/clients/js-native/tail.ts index 3339f866954..6563c4b2e2b 100644 --- a/test/interop/clients/js-native/tail.ts +++ b/test/interop/clients/js-native/tail.ts @@ -10,6 +10,7 @@ const GROUPS = 4; const BYTES = 256 * 1024; const [role, url, path, version, mode] = process.argv.slice(2); const drop = mode === "drop"; +const compress = mode === "compress"; if (!url || !path || !["publish", "subscribe"].includes(role)) { throw new Error("usage: tail.ts publish|subscribe URL BROADCAST"); } @@ -24,7 +25,7 @@ const connection = await Moq.Connection.connect({ try { if (role === "publish") { const broadcast = origin.createBroadcast(Moq.Path.from(path)); - const track = broadcast.createTrack("tail"); + const track = broadcast.createTrack("tail", { compress }); broadcast.announce(); const demand = track.demand(); while (demand.wanted.peek() === undefined) await demand.wanted.changed(); @@ -36,7 +37,8 @@ try { group.close(new Error("cut")); continue; } - group.writeFrame({ payload: new Uint8Array(BYTES).fill(sequence) }); + const sizes = compress ? [BYTES / 2, 0, BYTES / 2] : [BYTES]; + for (const size of sizes) group.writeFrame({ payload: new Uint8Array(size).fill(sequence) }); group.close(); } track.close(); @@ -60,6 +62,7 @@ try { const track = broadcast .track("tail") .subscribe({ maxDelay: Milli(drop ? 60_000 : 5000), groups: { start: { included: 0 } } }); + if (compress) assert.equal((await track.info()).compress, version === "moq-lite-07-wip"); const seen: number[] = []; try { for (;;) { @@ -67,13 +70,15 @@ try { if (!group) break; try { assert(group.sequence < GROUPS, `unexpected group ${group.sequence}`); - const frame = await group.readFrame(); - assert(frame, "empty group"); - assert.equal(frame.payload.byteLength, BYTES, "truncated group"); - assert( - frame.payload.every((byte) => byte === group.sequence), - "corrupt group", - ); + for (const size of compress ? [BYTES / 2, 0, BYTES / 2] : [BYTES]) { + const frame = await group.readFrame(); + assert(frame, "empty group"); + assert.equal(frame.payload.byteLength, size, "truncated group"); + assert( + frame.payload.every((byte) => byte === group.sequence), + "corrupt group", + ); + } assert.equal(await group.readFrame(), undefined, "extra frame"); seen.push(group.sequence); } finally { diff --git a/test/interop/clients/rust/tail.rs b/test/interop/clients/rust/tail.rs index 7aba491c29d..5c0482aae26 100644 --- a/test/interop/clients/rust/tail.rs +++ b/test/interop/clients/rust/tail.rs @@ -13,10 +13,11 @@ async fn main() -> anyhow::Result<()> { let args: Vec<_> = std::env::args().collect(); ensure!( args.len() == 4 || args.len() == 6, - "usage: interop-tail publish|subscribe URL BROADCAST [VERSION drop]" + "usage: interop-tail publish|subscribe URL BROADCAST [VERSION drop|compress]" ); let url: url::Url = args[2].parse()?; let drop = args.get(5).is_some_and(|mode| mode == "drop"); + let compress = args.get(5).is_some_and(|mode| mode == "compress"); let mut config = moq_tokio::connect::Config::default(); if let Some(version) = args.get(4) { config.version.push(version.parse().map_err(anyhow::Error::msg)?); @@ -30,7 +31,9 @@ async fn main() -> anyhow::Result<()> { let broadcast = origin.create_broadcast(&args[3])?; let mut track = broadcast.create_track( "tail", - moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_compress(compress), )?; broadcast.announce(Default::default())?; let session = client @@ -49,7 +52,10 @@ async fn main() -> anyhow::Result<()> { group.abort(moq_net::Error::App(1))?; continue; } - group.write_frame(moq_net::Timestamp::ZERO, vec![sequence as u8; BYTES])?; + let sizes: &[usize] = if compress { &[BYTES / 2, 0, BYTES / 2] } else { &[BYTES] }; + for &size in sizes { + group.write_frame(moq_net::Timestamp::ZERO, vec![sequence as u8; size])?; + } group.finish()?; } // A finished track is not a drained transport. Keep the session alive @@ -76,16 +82,25 @@ async fn main() -> anyhow::Result<()> { .with_max_delay(Duration::from_secs(if drop { 60 } else { 5 })) .with_groups(0..); let mut track = broadcast.track("tail")?.subscribe(Some(options)).await?; + if compress { + ensure!( + track.info().compress == args.get(4).is_some_and(|version| version == "moq-lite-07-wip"), + "wrong compression metadata" + ); + } let mut seen = Vec::new(); while let Some(mut group) = track.recv_group().await? { let sequence = group.sequence; - let frame = group.read_frame().await?.context("empty group")?; ensure!(sequence < GROUPS, "unexpected group {sequence}"); - ensure!(frame.payload.len() == BYTES, "truncated group {sequence}"); - ensure!( - frame.payload.iter().all(|byte| *byte == sequence as u8), - "corrupt group {sequence}" - ); + let sizes: &[usize] = if compress { &[BYTES / 2, 0, BYTES / 2] } else { &[BYTES] }; + for &size in sizes { + let frame = group.read_frame().await?.context("empty group")?; + ensure!(frame.payload.len() == size, "truncated group {sequence}"); + ensure!( + frame.payload.iter().all(|byte| *byte == sequence as u8), + "corrupt group {sequence}" + ); + } ensure!(group.read_frame().await?.is_none(), "extra frame in group {sequence}"); seen.push(sequence); } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 99d3ece29ef..f2ea313ad91 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1145,9 +1145,10 @@ run_tail_publisher() { } run_tail_pair() { - local pub="$1" sub="$2" name="tail-$1-$2" pid subscriber_pid started=$SECONDS + local pub="$1" sub="$2" mode="${3:-}" version="${4:-}" pid subscriber_pid started=$SECONDS + local name="tail-$1-$2-${mode:-plain}-${version:-default}" local fifo="$HARNESS_RUN/tail-ack" broadcast="tail-$1-$2-$$-$RANDOM" - local cell=(case=tail publisher="$pub" subscriber="$sub" log="$HARNESS_RUN/$name-sub.log") + local cell=(case="${mode:-tail}${version:+-$version}" publisher="$pub" subscriber="$sub" log="$HARNESS_RUN/$name-sub.log") if is_broken "$pub" || is_broken "$sub"; then echo " FAIL tail $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -1158,9 +1159,9 @@ run_tail_pair() { # Open both ends so neither child startup nor a failed publisher can block the harness. exec 9<>"$fifo" # Each side's token grants this broadcast alone, like a media round's. - harness_spawn "$name-pub" "$HARNESS_RUN/$name-pub.log" run_tail_publisher "$fifo" "$pub" publish "$broadcast" "$(token_url --publish "$broadcast")" + harness_spawn "$name-pub" "$HARNESS_RUN/$name-pub.log" run_tail_publisher "$fifo" "$pub" publish "$broadcast" "$(token_url --publish "$broadcast")" "$version" "$mode" pid="$HARNESS_PID" - harness_spawn "$name-sub" "$HARNESS_RUN/$name-sub.log" run_tail_client "$sub" subscribe "$broadcast" "$(token_url --subscribe "**/$broadcast")" + harness_spawn "$name-sub" "$HARNESS_RUN/$name-sub.log" run_tail_client "$sub" subscribe "$broadcast" "$(token_url --subscribe "**/$broadcast")" "$version" "$mode" subscriber_pid="$HARNESS_PID" harness_wait "$subscriber_pid" || true # Like run_native, a verified data result survives the NAPI addon's exit crash. @@ -1376,6 +1377,10 @@ if [[ "$NEGATIVE" -eq 0 && "$MEDIA" -eq 0 && "$IN_TREE" -eq 1 ]]; then if needs "$runtime"; then run_tail_pair rust "$runtime" run_tail_pair "$runtime" rust + for version in moq-lite-07-wip moq-lite-06; do + run_tail_pair rust "$runtime" compress "$version" + run_tail_pair "$runtime" rust compress "$version" + done fi done if needs js-native-node; then diff --git a/test/interop/lite-varint.ts b/test/interop/lite-varint.ts index 58faec72a44..e92dd3ac5c3 100644 --- a/test/interop/lite-varint.ts +++ b/test/interop/lite-varint.ts @@ -64,7 +64,7 @@ if (version === Version.DRAFT_07) { const setup = await Setup.decode(reader(input.setup), version); const setupBytes = await write((w) => setup.encode(w, version)); -const datagram = await Datagram.decode(Uint8Array.from(input.datagram), version, () => true); +const datagram = await Datagram.decode(Uint8Array.from(input.datagram), version, () => ({ timed: true })); const datagramBytes = [...datagram.encode(version)]; // Only lite-07 can carry an untimed track; lite-06 stamps it, so it has no untimed bytes. @@ -72,7 +72,7 @@ const untimed = version === Version.DRAFT_07; let untimedDatagramBytes: number[] = []; if (untimed) { - const datagram = await Datagram.decode(Uint8Array.from(input.untimedDatagram), version, () => false); + const datagram = await Datagram.decode(Uint8Array.from(input.untimedDatagram), version, () => ({ timed: false })); assert.equal(datagram.timestamp, undefined); untimedDatagramBytes = [...datagram.encode(version)]; } else { From 2102dba2b4e14d0fadd87ae1d4408f9f6d3e94f0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 23:11:22 -0700 Subject: [PATCH 4/5] test(interop): name the compression lane in tail results Co-Authored-By: Claude Opus 5.5 --- test/interop/interop.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/interop/interop.sh b/test/interop/interop.sh index f2ea313ad91..f896df32579 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1146,11 +1146,11 @@ run_tail_publisher() { run_tail_pair() { local pub="$1" sub="$2" mode="${3:-}" version="${4:-}" pid subscriber_pid started=$SECONDS - local name="tail-$1-$2-${mode:-plain}-${version:-default}" + local name="tail-$1-$2-${mode:-plain}-${version:-default}" label="${mode:-tail}${version:+ $version}" local fifo="$HARNESS_RUN/tail-ack" broadcast="tail-$1-$2-$$-$RANDOM" local cell=(case="${mode:-tail}${version:+-$version}" publisher="$pub" subscriber="$sub" log="$HARNESS_RUN/$name-sub.log") if is_broken "$pub" || is_broken "$sub"; then - echo " FAIL tail $pub -> $sub (client unavailable)" + echo " FAIL $label $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" overall=1 return @@ -1169,16 +1169,16 @@ run_tail_pair() { printf 'clean end\n' >&9 harness_wait "$pid" || true if grep -qx 'tail acknowledged' "$HARNESS_RUN/$name-pub.log"; then - echo " PASS tail $pub -> $sub (groups 0,1,2,3; clean end 4)" + echo " PASS $label $pub -> $sub (groups 0,1,2,3; clean end 4)" record pass "${cell[@]}" seconds=$((SECONDS - started)) else - echo " FAIL tail $pub -> $sub (publisher did not acknowledge)" + echo " FAIL $label $pub -> $sub (publisher did not acknowledge)" record fail "${cell[@]}" seconds=$((SECONDS - started)) detail="publisher did not acknowledge" overall=1 cat "$HARNESS_RUN/$name-pub.log" fi else - echo " FAIL tail $pub -> $sub (missing groups or clean end)" + echo " FAIL $label $pub -> $sub (missing groups or clean end)" record fail "${cell[@]}" seconds=$((SECONDS - started)) detail="missing groups or clean end" overall=1 cat "$HARNESS_RUN/$name-pub.log" "$HARNESS_RUN/$name-sub.log" From f87cd23c94e6e9adefa28a09c997fa39e8b21f6a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:24:13 -0700 Subject: [PATCH 5/5] quest: drop the finished JS compression blocker from native adoption Co-Authored-By: Claude Opus 5.5 --- quest/m2/native-compression-adopt.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/quest/m2/native-compression-adopt.md b/quest/m2/native-compression-adopt.md index 48c4d58abf0..942d426832c 100644 --- a/quest/m2/native-compression-adopt.md +++ b/quest/m2/native-compression-adopt.md @@ -19,11 +19,11 @@ main-thread blocking through workers, or a new preparation API. starting implementation. Constructor support alone is insufficient. Check nonfinal frame output, shared history, exact declared-size admission, early output-overrun refusal, cancellation, malformed input and group end. -- Native decoding is already an avenue in the JS compression quest and size - audit. Coordinate with that implementation and remove only duplicated - conditional scope once its actual result is known. Do not redo a landed - native decoder. Encoder and decoder adoption may land independently if - each preserves the existing contract and removes worthwhile bundled code. +- Native decoding is also an avenue in the size audit. JS compression landed + with a bundled pako codec in `js/net/src/flate`; coordinate with the audit + and remove only duplicated conditional scope. Encoder and decoder adoption + may land independently if each preserves the existing contract and removes + worthwhile bundled code. - Chromium 156 decoded the tested nonfinal frames before closure; this does not establish synchronous availability, cross-engine behavior or bounded allocation. Verify those properties rather than treating the experiment @@ -40,10 +40,6 @@ main-thread blocking through workers, or a new preparation API. change needs a new maintainer decision. Update affected docs inline; the compression guide owns the user-facing entry point, not a new page. -## Required - -- [JS compression](/quest/m1/compress/js.md) - the current compressed-track model and interoperability behavior are implemented - ## Related - [Frame flush report](/quest/m2/native-compression-flush.md) - the native nonfinal encoder flush gap