diff --git a/bun.lock b/bun.lock index fc40ac452a4..5c69ae3b04e 100644 --- a/bun.lock +++ b/bun.lock @@ -260,6 +260,7 @@ "@moq/qmux": "workspace:^", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0", + "pako": "^3.0.2", "uuid": "^14.0.3", }, "devDependencies": { diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index cbe0349ea5a..631035e4d67 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -297,7 +297,9 @@ JSON and not for media. Each group's frames become one DEFLATE stream compresses alone. Only the work-in-progress moq-lite 07 carries the compressed bytes. A relay forwards them as received and inflates each group once for every other subscriber, moq-transport included, so readers always see the original -frames. Rust publishes and relays compressed tracks; js/net refuses them for now. +frames. Rust and JavaScript publish compressed tracks by setting `compress: true` +on the track's info. Application writes stay synchronous and readers receive plain +payloads; compression is useful for repetitive data, not already compressed media. A relay cancels its upstream subscription once nobody subscribes, but keeps its copy of the track for 30 seconds after the last reader leaves, so a returning diff --git a/js/justfile b/js/justfile index 6cab4e0a74f..5f2df8d04b8 100644 --- a/js/justfile +++ b/js/justfile @@ -55,7 +55,7 @@ test-changed list: coverage package="": python3 ../sh/js/coverage.py '{{ package }}' -# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, or subscribe-drop. +# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, subscribe-drop, or compression. bench name: bun install --frozen-lockfile bun net/bench/{{ name }}.ts diff --git a/js/net/bench/compression.ts b/js/net/bench/compression.ts new file mode 100644 index 00000000000..ef0ad91dc4c --- /dev/null +++ b/js/net/bench/compression.ts @@ -0,0 +1,70 @@ +/** Sweep compressed publishers x plain readers, reporting one shared inflated copy per group. */ +import { gzipSync } from "node:zlib"; +import type { Producer as Group } from "../src/group.ts"; +import { hooks } from "../src/internal.ts"; +import { Producer as Track } from "../src/track.ts"; + +const payload = new TextEncoder().encode('{"value":123,"status":"unchanged"}\n'.repeat(64)); +const frames = 32; +const rounds = 10; +console.log("publishers,readers,publish_us,read_us,stored_bytes,inflated_bytes"); +for (const publishers of [1, 4, 16]) { + for (const readers of [1, 4, 16]) { + let publish = 0; + let read = 0; + let stored = 0; + let inflated = 0; + for (let round = 0; round < rounds; round++) { + const tracks = Array.from({ length: publishers }, () => new Track("bench").accept({ compress: true })); + const groups: Group[] = tracks.map((track) => track.appendGroup()); + const consumers = groups.map((group) => Array.from({ length: readers }, () => group.mirror())); + let started = performance.now(); + for (const group of groups) { + for (let index = 0; index < frames; index++) group.writeFrame({ payload }); + group.close(); + } + publish += performance.now() - started; + stored = groups.reduce((bytes, group) => bytes + hooks.groupCacheBytes(group), 0); + started = performance.now(); + for (const subscribers of consumers) { + let shared: Uint8Array | undefined; + for (const consumer of subscribers) { + for (let index = 0; index < frames; index++) { + const frame = consumer.tryReadFrame(); + if (!frame || frame.payload.length !== payload.length) throw new Error("missing payload"); + if (index === 0) { + if (shared && shared !== frame.payload) throw new Error("fanout duplicated inflation"); + shared = frame.payload; + } + } + consumer.close(); + } + } + read += performance.now() - started; + inflated = groups.reduce((bytes, group) => bytes + hooks.groupCacheBytes(group), 0) - stored; + if (inflated !== publishers * frames * payload.length) throw new Error("inflated bytes scale with readers"); + for (const track of tracks) track.close(); + } + console.log( + [ + publishers, + readers, + ((publish * 1000) / rounds).toFixed(1), + ((read * 1000) / rounds).toFixed(1), + stored, + inflated, + ].join(","), + ); + } +} + +// This entry is an application importing only its track consumer. Synchronous model writes +// share the module, so its bundle includes the encoder as well as the decoder. +const bundle = await Bun.build({ + entrypoints: [new URL("./fixtures/compression-consumer.ts", import.meta.url).pathname], + minify: true, + target: "browser", +}); +if (!bundle.success) throw new AggregateError(bundle.logs, "consumer bundle failed"); +const bytes = new Uint8Array(await bundle.outputs[0].arrayBuffer()); +console.log(`consumer_bundle_bytes=${bytes.length},gzip_bytes=${gzipSync(bytes).length}`); diff --git a/js/net/bench/fixtures/compression-consumer.ts b/js/net/bench/fixtures/compression-consumer.ts new file mode 100644 index 00000000000..b56b3265f56 --- /dev/null +++ b/js/net/bench/fixtures/compression-consumer.ts @@ -0,0 +1,2 @@ +/** Consumer-only entry for measuring the static application graph. @module */ +export { Consumer } from "../../src/track.ts"; diff --git a/js/net/bench/frames.ts b/js/net/bench/frames.ts index eb2ac881bf4..22aa863798b 100644 --- a/js/net/bench/frames.ts +++ b/js/net/bench/frames.ts @@ -52,9 +52,9 @@ const lite: Protocol = { async subscribe() { const open: Open = (stream) => { const producer = new Producer(0); - const done = readFrames(new Reader(stream, undefined, LiteVersion.DRAFT_06), producer, 1_000_000).then(() => - producer.close(), - ); + const done = readFrames(new Reader(stream, undefined, LiteVersion.DRAFT_06), producer, { + scale: 1_000_000, + }).then(() => producer.close()); return { group: Promise.resolve(producer.consume()), done }; }; return { open, close: () => {} }; diff --git a/js/net/bench/lite-varint.ts b/js/net/bench/lite-varint.ts index 4f335e79355..bd2a8182387 100644 --- a/js/net/bench/lite-varint.ts +++ b/js/net/bench/lite-varint.ts @@ -79,7 +79,7 @@ const subscribe: Sample = { const datagram: Sample = { name: "Datagram", encode: (w, version) => w.write(new Datagram(3n, 1_234, 1_234_567_890, new Uint8Array()).encode(version)), - decode: (r, version) => r.readAll().then((data) => Datagram.decode(data, version, () => true)), + decode: (r, version) => r.readAll().then((data) => Datagram.decode(data, version, () => ({ timed: true }))), }; const hop = randomHop(); diff --git a/js/net/package.json b/js/net/package.json index 723d081e12b..abe8c2fbf9b 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -26,6 +26,7 @@ "@moq/qmux": "workspace:^", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0", + "pako": "^3.0.2", "uuid": "^14.0.3" }, "peerDependencies": { diff --git a/js/net/src/flate/codec.test.ts b/js/net/src/flate/codec.test.ts new file mode 100644 index 00000000000..397fe7c38b5 --- /dev/null +++ b/js/net/src/flate/codec.test.ts @@ -0,0 +1,62 @@ +import { expect, spyOn, test } from "bun:test"; +import { deflateRaw, Inflate } from "pako"; +import { Decoder } from "./decoder.ts"; +import { Encoder } from "./encoder.ts"; + +const text = new TextEncoder(); + +test("group slices preserve a shared window across empty and nonempty frames", () => { + const frames = ["the quick brown fox", "", "the quick brown dog", "the lazy fox"]; + const encoder = new Encoder(); + const decoder = new Decoder(); + for (const frame of frames.map((value) => text.encode(value))) { + expect(decoder.frame(encoder.frame(frame), frame.length)).toEqual(frame); + } +}); + +test("a retained frame owns only its payload, across later pushes", () => { + const encoder = new Encoder(); + const decoder = new Decoder(); + const first = decoder.frame(encoder.frame(text.encode("first")), 5); + decoder.frame(encoder.frame(new Uint8Array(20_000)), 20_000); + expect(first).toEqual(text.encode("first")); + expect(first.buffer.byteLength).toBe(first.length); +}); + +test("each declared size is exact, including an empty slice", () => { + const slice = new Encoder().frame(new Uint8Array(1024)); + for (const size of [512, 2048]) expect(() => new Decoder().frame(slice, size)).toThrow(/declared size/); + expect(() => new Decoder().frame(new Uint8Array(), 1)).toThrow(/declared size/); +}); + +test("a bomb stops inflating at the declaration plus one scratch chunk", () => { + const slice = new Encoder().frame(new Uint8Array(1024 * 1024)); + const push = Inflate.prototype.push; + let output = 0; + const pushing = spyOn(Inflate.prototype, "push").mockImplementation(function (this: Inflate, ...args) { + const onData = this.onData; + this.onData = (chunk) => { + output += chunk.length; + onData.call(this, chunk); + }; + try { + return push.apply(this, args); + } finally { + this.onData = onData; + } + }); + try { + expect(() => new Decoder().frame(slice, 1)).toThrow(/exceeds/); + expect(output).toBe(4096); + } finally { + pushing.mockRestore(); + } +}); + +test("final blocks, malformed input, and a decoder already failed are refused", () => { + const frame = text.encode("final"); + expect(() => new Decoder().frame(deflateRaw(frame), frame.length)).toThrow(/ended/); + const decoder = new Decoder(); + expect(() => decoder.frame(new Uint8Array(64).fill(0xff), 64)).toThrow(); + expect(() => decoder.frame(new Uint8Array(), 0)).toThrow(/already failed/); +}); diff --git a/js/net/src/flate/decoder.ts b/js/net/src/flate/decoder.ts new file mode 100644 index 00000000000..3378a94ba9a --- /dev/null +++ b/js/net/src/flate/decoder.ts @@ -0,0 +1,59 @@ +// The group-scoped moq-flate decoder; applications always read inflated frames. +import { Inflate, Z_SYNC_FLUSH } from "pako"; +import { StreamCode, StreamError } from "../error.ts"; + +const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); + +function malformed(message: string): Error { + return new StreamError(StreamCode.MalformedTrack, { message }); +} + +/** Decode one group's slices in frame order, enforcing each declared size. @internal */ +export class Decoder { + #inflate = new Inflate({ raw: true, chunkSize: 4096 }); + #chunks: Uint8Array[] = []; + #total = 0; + #size = 0; + #failed = false; + + constructor() { + this.#inflate.onData = (chunk) => { + this.#total += chunk.length; + // Throw inside push, so a bomb stops doing work at its declaration plus one + // scratch chunk rather than merely discarding the rest of its output. + if (this.#total > this.#size) throw malformed("inflated frame exceeds its declared size"); + this.#chunks.push(chunk); + }; + } + + /** Inflate a slice to exactly `size` bytes, without preallocating from that declaration. */ + frame(slice: Uint8Array, size: number): Uint8Array { + if (this.#failed) throw malformed("decoder already failed"); + if (!Number.isSafeInteger(size) || size < 0) throw malformed("invalid inflated frame size"); + this.#chunks = []; + this.#total = 0; + this.#size = size; + try { + if (slice.length > 0) { + this.#inflate.push(slice, false); + if (this.#inflate.ended) throw malformed("compressed group ended its DEFLATE stream"); + this.#inflate.push(SYNC_FLUSH_TAIL, Z_SYNC_FLUSH); + if (this.#inflate.err || this.#inflate.ended) throw malformed("malformed DEFLATE slice"); + } + if (this.#total !== size) throw malformed("inflated frame differs from its declared size"); + // Retained frames must own tight buffers rather than pinning scratch storage. + const out = new Uint8Array(this.#total); + let offset = 0; + for (const chunk of this.#chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; + } catch (error) { + this.#failed = true; + throw error; + } finally { + this.#chunks = []; + } + } +} diff --git a/js/net/src/flate/encoder.ts b/js/net/src/flate/encoder.ts new file mode 100644 index 00000000000..f4d01214921 --- /dev/null +++ b/js/net/src/flate/encoder.ts @@ -0,0 +1,73 @@ +// Synchronous model writes include the encoder in the static graph. +import { Deflate, type DeflateOptions, Z_SYNC_FLUSH } from "pako"; + +const DEFAULT_LEVEL = 6; +const SYNC_FLUSH_TAIL = new Uint8Array([0x00, 0x00, 0xff, 0xff]); + +/** Options for an {@link Encoder}. */ +interface EncoderOptions { + /** DEFLATE level, `0..=9` (higher is smaller and slower). Defaults to {@link DEFAULT_LEVEL}. */ + level?: number; +} + +/** + * Encodes a stream's frame payloads into one shared DEFLATE window, one self-delimited slice per + * frame. Hold one per stream; create a new one for each independent stream. + * + * @internal + */ +export class Encoder { + #deflate: Deflate; + #chunks: Uint8Array[] = []; + #total = 0; + + /** Start a fresh encoder with a cold window. */ + constructor(options: EncoderOptions = {}) { + // `raw`: no zlib header/trailer, matching the Rust side and the browser's `deflate-raw`. + // pako types `level` as a literal union; we accept a plain number and narrow here. + const level = (options.level ?? DEFAULT_LEVEL) as DeflateOptions["level"]; + this.#deflate = new Deflate({ raw: true, level }); + this.#deflate.onData = (chunk) => { + const bytes = chunk as Uint8Array; + this.#chunks.push(bytes); + this.#total += bytes.length; + }; + } + + /** + * The most bytes {@link frame} can return for a payload of `len` bytes, at any level and whatever + * the window already holds. + * + * zlib's `deflateBound` for its default window and memory level, which both pako and the Rust + * `moq-flate` use: incompressible input falls back to stored blocks, 5 bytes per 16 KiB. The + * constant covers the block headers and the flush, whose fixed 4-byte marker is stripped anyway. + */ + static bound(len: number): number { + // Division rather than shifts, which wrap past 2^31. + return len + Math.floor(len / 4096) + Math.floor(len / 16384) + Math.floor(len / 33554432) + 13; + } + + /** + * Compress the next frame's `payload`, returning its slice of the stream: the DEFLATE bytes minus + * the fixed sync-flush marker. Empty in yields empty out. Slices must be produced in frame order. + */ + frame(payload: Uint8Array): Uint8Array { + if (payload.length === 0) return payload; + this.#chunks = []; + this.#total = 0; + this.#deflate.push(payload, Z_SYNC_FLUSH); + + // Copy into one tight owned buffer, dropping the trailing sync-flush marker. We can't return + // pako's chunk views: a caller retains the reference and pako backs each chunk with a ~16 KB + // buffer, so a view would pin far more memory than the frame. + const out = new Uint8Array(this.#total - SYNC_FLUSH_TAIL.length); + let offset = 0; + for (const chunk of this.#chunks) { + if (offset >= out.length) break; + const take = Math.min(chunk.length, out.length - offset); + out.set(chunk.subarray(0, take), offset); + offset += take; + } + return out; + } +} diff --git a/js/net/src/flate/group.test.ts b/js/net/src/flate/group.test.ts new file mode 100644 index 00000000000..8dd65b0614a --- /dev/null +++ b/js/net/src/flate/group.test.ts @@ -0,0 +1,154 @@ +import { expect, spyOn, test } from "bun:test"; +import { Producer as Group, MAX_GROUP_CACHE_BYTES } from "../group.ts"; +import { hooks } from "../internal.ts"; +import { Producer as Track } from "../track.ts"; +import { Decoder } from "./decoder.ts"; +import { Encoder } from "./encoder.ts"; +import { stored } from "./group.ts"; + +const payload = new TextEncoder().encode("one shared compression window ".repeat(100)); + +test("synchronous writers store slices; plain readers share one charged inflated copy", async () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const first = group.mirror(); + const second = group.mirror(); + const raw = group.mirror(); + group.writeFrame({ payload }); + group.writeFrame({ payload }); + group.close(); + const stored = hooks.groupCacheBytes(group); + expect(stored).toBeLessThan(payload.length); + const inflating = spyOn(Decoder.prototype, "frame"); + try { + const slice = await hooks.readGroupFrame(raw, undefined, true); + expect(slice?.inflated).toBe(payload.length); + expect(inflating).toHaveBeenCalledTimes(0); + slice?.complete(); + const a = await first.readFrame(); + const b = await second.readFrame(); + expect(a?.payload).toBe(b?.payload); + expect(a?.payload).toEqual(payload); + expect(inflating).toHaveBeenCalledTimes(1); + expect(hooks.groupCacheBytes(group)).toBe(stored + payload.length); + await first.readFrame(); + await second.readFrame(); + expect(inflating).toHaveBeenCalledTimes(2); + } finally { + inflating.mockRestore(); + track.close(); + } +}); + +test("received slices are forwarded byte-for-byte and exact declarations fail all plain readers", async () => { + const group = new Group(0); + hooks.bindGroupCompression(group, true); + const slice = new Encoder().frame(payload); + hooks.writeGroupFrame(group, { payload: slice }, payload.length + 1); + group.close(); + const raw = await hooks.readGroupFrame(group.mirror(), undefined, true); + expect(raw?.frame.payload).toBe(slice); + raw?.complete(); + const bad = group.mirror(); + await expect(bad.readFrame()).rejects.toThrow(/declared size/); + expect(bad.closed.peek()).toBeInstanceOf(Error); + await expect(group.mirror().readFrame()).rejects.toThrow(/declared size/); +}); + +test("declared plain bytes consume the group budget before any inflation", () => { + const group = new Group(0); + hooks.bindGroupCompression(group, true); + const inflating = spyOn(Decoder.prototype, "frame"); + try { + expect(() => hooks.writeGroupFrame(group, { payload: new Uint8Array() }, MAX_GROUP_CACHE_BYTES + 1)).toThrow(); + hooks.writeGroupFrame(group, { payload: new Uint8Array() }, MAX_GROUP_CACHE_BYTES); + expect(() => hooks.writeGroupFrame(group, { payload: new Uint8Array() }, 1)).toThrow(); + expect(inflating).toHaveBeenCalledTimes(0); + expect(hooks.groupCacheBytes(group)).toBe(0); + } finally { + inflating.mockRestore(); + group.close(); + } +}); + +test("empty frames preserve the window and prewritten groups bind without duplicating plain copies", async () => { + const group = new Group(0); + group.writeFrame({ payload }); + group.writeFrame({ payload: new Uint8Array() }); + const reader = group.mirror(); + const track = new Track("compressed").accept({ compress: true }); + track.writeGroup(group); + group.writeFrame({ payload }); + group.close(); + for (const expected of [payload, new Uint8Array(), payload]) + expect((await reader.readFrame())?.payload).toEqual(expected); + track.close(); +}); + +test("a cut compressed group preserves its committed frames before returning its error", async () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const reader = group.mirror(); + group.writeFrame({ payload }); + const cut = new Error("cut"); + group.close(cut); + expect((await reader.readFrame())?.payload).toEqual(payload); + await expect(reader.readFrame()).rejects.toThrow(cut); + track.close(); +}); + +test("a shared plain read cannot change a later synchronous write's admission", async () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const reader = group.mirror(); + group.writeFrame({ payload: new Uint8Array(MAX_GROUP_CACHE_BYTES) }); + expect((await reader.readFrame())?.payload.length).toBe(MAX_GROUP_CACHE_BYTES); + // The plain-byte budget is full, but another empty frame still fits, regardless of readers. + expect(() => group.writeFrame({ payload: new Uint8Array() })).not.toThrow(); + group.close(); + expect((await reader.readFrame())?.payload.length).toBe(0); + track.close(); +}); + +test("datagrams buffered before accept share one stored and plain form", async () => { + const track = new Track("late-info"); + const a = track.subscribe(); + const b = track.subscribe(); + track.appendDatagram(undefined, payload); + track.accept({ compress: true }); + const first = await a.recvDatagram(); + const second = await b.recvDatagram(); + if (!first || !second) throw new Error("missing datagram"); + expect(first).toBe(second); + expect(stored(first)?.inflated).toBe(payload.length); + expect(first.payload).toEqual(payload); + track.close(); +}); + +test("a malformed compressed datagram is refused before reaching readers", () => { + const track = new Track("compressed").accept({ compress: true }); + const reader = track.subscribe(); + const payload = new Uint8Array(100); + const slice = new Encoder().frame(payload); + expect(() => hooks.writeDatagram(track, { sequence: 0, payload: slice }, payload.length + 1)).toThrow( + /declared size/, + ); + expect(hooks.tryRecvDatagram(reader).kind).toBe("idle"); + track.close(); +}); + +test("refresh measures stored slices without inflating a compressed group", () => { + const track = new Track("compressed").accept({ compress: true }); + const group = track.appendGroup(); + const payload = Uint8Array.from({ length: 1024 }, (_, index) => index % 251); + for (let index = 0; index < 3; index++) group.writeFrame({ payload }); + const inflating = spyOn(Decoder.prototype, "frame"); + try { + expect(hooks.overRefresh(group, 1)).toBe(false); + expect(inflating).toHaveBeenCalledTimes(0); + } finally { + inflating.mockRestore(); + group.close(); + track.close(); + } +}); diff --git a/js/net/src/flate/group.ts b/js/net/src/flate/group.ts new file mode 100644 index 00000000000..7e226084069 --- /dev/null +++ b/js/net/src/flate/group.ts @@ -0,0 +1,102 @@ +import type { Frame } from "../group.ts"; +import type { Timestamp } from "../time.ts"; +import { Decoder } from "./decoder.ts"; +import { Encoder } from "./encoder.ts"; + +/** A stored slice and its declared plain size. Package-private wire metadata. */ +export interface Stored { + /** The original compressed bytes. */ + payload: Uint8Array; + /** The exact plain payload length. */ + inflated: number; +} + +// Class identity keeps reactive equality from reading the lazy payload getter. +class SliceFrame implements Frame { + readonly timestamp: Timestamp | undefined; + #read: () => Uint8Array; + constructor(timestamp: Timestamp | undefined, read: () => Uint8Array) { + this.timestamp = timestamp; + this.#read = read; + } + get payload(): Uint8Array { + return this.#read(); + } +} + +const slices = new WeakMap(); + +/** Read a compressed frame's stored form without inflating it. */ +export function stored(frame: Frame): Stored | undefined { + return slices.get(frame); +} + +/** One group's stored form and lazy plain copy, shared by every mirror. */ +export class Compression { + #encoder?: Encoder; + #decoder?: Decoder; + #frames: Stored[] = []; + #plain: Uint8Array[] = []; + #failed?: Error; + #finished = false; + /** Plain bytes admitted against the group budget. */ + declared = 0; + /** Retained compressed payload bytes, separate from its optional plain copy. */ + stored = 0; + /** Retained slice and shared plain bytes. */ + bytes = 0; + + /** Store a local plain frame or a received slice after budget admission. */ + push(frame: Frame, inflated?: number): Frame { + let slice: Stored; + if (inflated === undefined) { + this.#encoder ??= new Encoder(); + slice = { payload: this.#encoder.frame(frame.payload), inflated: frame.payload.length }; + } else slice = { payload: frame.payload, inflated }; + this.declared += slice.inflated; + this.stored += slice.payload.length; + this.bytes += slice.payload.length; + const index = this.#frames.length; + this.#frames.push(slice); + const result = new SliceFrame(frame.timestamp, () => this.#inflate(index)); + slices.set(result, slice); + return result; + } + + #inflate(index: number): Uint8Array { + if (this.#failed) throw this.#failed; + try { + while (this.#plain.length <= index) { + const slice = this.#frames[this.#plain.length]; + this.#decoder ??= new Decoder(); + const plain = this.#decoder.frame(slice.payload, slice.inflated); + this.#plain.push(plain); + this.bytes += plain.length; + } + if (this.#finished && this.#plain.length === this.#frames.length) this.#decoder = undefined; + return this.#plain[index]; + } catch (error) { + this.#failed = error instanceof Error ? error : new Error(String(error)); + this.#decoder = undefined; + throw this.#failed; + } + } + + /** Release cached content on an abort, failing every mirror's late read. */ + abort(error: Error): void { + this.#failed = error; + this.#encoder = undefined; + this.#decoder = undefined; + this.#frames = []; + this.#plain = []; + this.bytes = 0; + this.stored = 0; + } + + /** Release codec scratch once no more frames can arrive. */ + finish(): void { + this.#finished = true; + this.#encoder = undefined; + if (this.#plain.length === this.#frames.length) this.#decoder = undefined; + } +} diff --git a/js/net/src/group.ts b/js/net/src/group.ts index b2f6274f3be..1d4909ade67 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -14,6 +14,7 @@ import { TimestampMismatch, TooFarBehind, } from "./error.ts"; +import { Compression, stored } from "./flate/group.ts"; import { type Expiry, type GroupRecover, hooks, type ReadGroupFrame, type Replacement } from "./internal.ts"; import type { Timestamp } from "./time.ts"; @@ -30,7 +31,7 @@ export const MAX_GROUP_FRAMES = 8192; * converts it into the track's negotiated timescale. */ export interface Frame { - /** The frame payload. */ + /** The plain payload, shared across readers even when the track stores compressed bytes. */ payload: Uint8Array; /** * Presentation timestamp, present exactly when the track has a timescale. A frame on a @@ -69,6 +70,8 @@ class GroupState { start = 0; cacheBytes = 0; + compressed?: boolean; + flate?: Compression; // Whether the owning track has a timescale, so every frame must carry a timestamp (true) // or none may (false). Undefined for a group no track has claimed yet, which takes either. timed?: boolean; @@ -102,7 +105,7 @@ function appendFrame(state: GroupState, frame: Frame) { state.timestamp ??= frame.timestamp; state.latest = frame.timestamp; - state.cacheBytes += frame.payload.byteLength; + state.cacheBytes += stored(frame)?.payload.length ?? frame.payload.byteLength; state.frames.mutate((frames) => { frames.push(frame); }); @@ -185,6 +188,29 @@ export class Producer { } static { + hooks.bindGroupCompression = (group, compressed) => { + const state = group.#state; + if (state.compressed === compressed) return; + if (state.compressed !== undefined) + throw new ProtocolViolation("group compression disagrees with its track"); + if (compressed && state.start !== 0) throw new ProtocolViolation("compressed group lacks its prefix"); + state.compressed = compressed; + if (!compressed) return; + const flate = new Compression(); + state.flate = flate; + const frames = state.frames.peek().map((frame) => flate.push(frame)); + state.frames.set(frames); + state.cacheBytes = flate.stored; + if (flate.stored > MAX_GROUP_CACHE_BYTES) group.#overflow(); + for (const mirror of group.#mirrors ?? []) { + mirror.frames.set(frames.slice(mirror.start - state.start)); + mirror.flate = flate; + mirror.compressed = true; + } + if (state.closed.peek() !== undefined) flate.finish(); + }; + hooks.writeGroupFrame = (group, frame, inflated) => group.#writeFrame(frame, inflated); + hooks.groupCacheBytes = (group) => group.#state.flate?.bytes ?? group.#state.cacheBytes; hooks.groupProgress = (group, fn) => { const state = group.#state; fn(state.total.peek()); @@ -209,7 +235,7 @@ export class Producer { const first = state.frames.peek()[0]; // Without its first frame the cache cannot measure the group, so it asks for nothing. if (state.start !== 0 || first === undefined) return false; - const head = first.payload.byteLength; + const head = stored(first)?.payload.length ?? first.payload.byteLength; return state.cacheBytes - head > head * ratio; }; hooks.limitGroup = (group, frames) => { @@ -257,6 +283,8 @@ export class Producer { const dst = new GroupState(this.sequence); dst.timestamp = this.#state.timestamp; dst.latest = this.#state.latest; + dst.flate = this.#state.flate; + dst.compressed = this.#state.compressed; for (const frame of this.#state.frames.peek()) appendFrame(dst, frame); dst.start = this.#state.start; // The replay only covers what is still buffered, so the count has to come from the @@ -293,13 +321,20 @@ export class Producer { /** Writes a frame to the group. */ writeFrame(frame: Frame) { + this.#writeFrame(frame); + } + + #writeFrame(frame: Frame, inflated?: number) { // Closed first: a write after a clean close at the cap would otherwise take the // overflow path, wipe committed frames, and leave readers with an empty success. if (this.#state.closed.peek() !== undefined) throw new Error("group is closed"); // A frame past the cache cap would make the group exceed its budget by itself, so // accepting it would report success for a write nothing can ever read. Rust rejects // it up front with `Error::FrameTooLarge`; do the same rather than silently dropping it. - if (frame.payload.byteLength > MAX_GROUP_CACHE_BYTES) throw new FrameTooLarge(); + if (frame.payload.byteLength > MAX_GROUP_CACHE_BYTES || (inflated ?? 0) > MAX_GROUP_CACHE_BYTES) + throw new FrameTooLarge(); + if (inflated !== undefined && (!this.#state.flate || !Number.isSafeInteger(inflated) || inflated < 0)) + throw new ProtocolViolation("unexpected inflated frame size"); checkTimed(this.#state.timed, frame); if (this.#limit !== undefined && this.#state.total.peek() >= this.#limit) { throw new Error(`frame ${this.#state.total.peek()} is past the track's end`); @@ -313,21 +348,20 @@ export class Producer { } if ( this.#state.total.peek() >= MAX_GROUP_FRAMES || - this.#state.cacheBytes + frame.payload.byteLength > MAX_GROUP_CACHE_BYTES + ((!this.#state.flate || inflated !== undefined) && + this.#state.cacheBytes + frame.payload.byteLength > MAX_GROUP_CACHE_BYTES) || + (this.#state.flate?.declared ?? this.#state.cacheBytes) + (inflated ?? frame.payload.length) > + MAX_GROUP_CACHE_BYTES ) { - const err = new GroupTooLarge(); - this.#state.frames.set([]); - this.#state.cacheBytes = 0; - if (this.#mirrors) { - for (const mirror of this.#mirrors) { - mirror.frames.set([]); - mirror.cacheBytes = 0; - } - } - this.close(err); - throw err; + this.#overflow(); } + if (this.#state.flate) { + frame = this.#state.flate.push(frame, inflated); + if (this.#state.flate.stored > MAX_GROUP_CACHE_BYTES) { + this.#overflow(); + } + } this.#activity = now(); appendFrame(this.#state, frame); if (commits) { @@ -343,6 +377,19 @@ export class Producer { } } + #overflow(): never { + const error = new GroupTooLarge(); + this.#state.frames.set([]); + this.#state.cacheBytes = 0; + this.#state.flate?.abort(error); + for (const mirror of this.#mirrors ?? []) { + mirror.frames.set([]); + mirror.cacheBytes = 0; + } + this.close(error); + throw error; + } + /** True once the group has been closed. */ get isClosed(): boolean { return this.#state.closed.peek() !== undefined; @@ -351,6 +398,7 @@ export class Producer { /** Closes the group, optionally with an error to abort readers. */ close(abort?: Error) { if (this.#state.closed.peek() !== undefined) return; + this.#state.flate?.finish(); this.#state.closed.set(abort ?? null); if (this.#mirrors) { @@ -534,7 +582,7 @@ export class Consumer { group.#closed = undefined; }; hooks.guardGroup = (group, operation) => group.#guard(operation); - hooks.readGroupFrame = (group, from) => group.#readFramePosition(true, from); + hooks.readGroupFrame = (group, from, compressed) => group.#readFramePosition(true, from, compressed); hooks.evictGroup = (group) => { group.#evict(); }; @@ -609,6 +657,7 @@ export class Consumer { this.#state.start += frames.length; } this.#state.cacheBytes = 0; + this.#state.flate = undefined; this.#state.frames.set([]); } @@ -713,17 +762,29 @@ export class Consumer { return frames; } - #readBufferedFrame(pending = false): ReadGroupFrame | undefined { + #readBufferedFrame(pending = false, compressed = false): ReadGroupFrame | undefined { const frames = this.#bufferedFrames(); const buffered = frames.shift(); if (!buffered) return undefined; + const slice = compressed ? stored(buffered) : undefined; + let frame: Frame; + try { + frame = { payload: slice?.payload ?? buffered.payload, timestamp: buffered.timestamp }; + } catch (error) { + this.#terminal = error instanceof Error ? error : new Error(String(error)); + if (this.#verdict.peek() === undefined) this.#verdict.set(this.#terminal); + this.#closed?.refresh(); + throw this.#terminal; + } + if (frames.length === 0 && this.#state.closed.peek() !== undefined) this.#state.flate = undefined; const sequence = this.#state.start++; if (pending) this.#pendingFrames++; let completed = false; return { sequence, - frame: buffered, + frame, + inflated: slice?.inflated, complete: () => { if (completed) return; completed = true; @@ -814,14 +875,14 @@ export class Consumer { return (await this.#readFramePosition())?.frame; } - async #readFramePosition(pending = false, from = 0): Promise { + async #readFramePosition(pending = false, from = 0, compressed = false): Promise { for (;;) { if (this.#terminal) { this.#abandon(); throw this.#terminal; } - const read = this.#readBufferedFrame(pending); + const read = this.#readBufferedFrame(pending, compressed); if (read) { if (read.sequence < from) { read.complete(); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 9c62516bee3..b387e40894a 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -198,6 +198,8 @@ export interface ReadGroupFrame { sequence: number; /** Frame returned to the publisher. */ frame: Frame; + /** The stored slice's declared plain size, on compressed wire reads. */ + inflated?: number; /** Mark the frame delivered or deliberately skipped by the wire publisher. */ complete(): void; } @@ -518,6 +520,17 @@ export const hooks: { * Throws `TimestampMismatch` if a buffered frame already disagrees. */ bindGroupTimed: (group: GroupProducer, timed: boolean) => void; + /** Admit a datagram in its stored wire form. */ + // private: passthrough + writeDatagram: (producer: Producer, datagram: Datagram, inflated?: number) => void; + /** Bind the group to its immutable track codec. */ + // private: passthrough + bindGroupCompression: (group: GroupProducer, compressed: boolean) => void; + /** Admit a received stored slice without recompressing it. */ + // private: passthrough + writeGroupFrame: (group: GroupProducer, frame: Frame, inflated?: number) => void; + /** Payload footprint including a shared inflated copy, for measurements. */ + groupCacheBytes: (group: GroupProducer) => number; /** * Cap a group at `frames` for a track end inside it, so later writes past it throw. * Throws `ProtocolViolation` if a frame at or past the cap already exists. @@ -551,7 +564,7 @@ export const hooks: { guardGroup: (group: GroupConsumer, operation: () => Promise) => Promise; // pending: /quest/m1/public-model/serve.md /** Read a frame the wire publisher completes (or skips) once written. */ - readGroupFrame: (group: GroupConsumer, from?: number) => Promise; + readGroupFrame: (group: GroupConsumer, from?: number, compressed?: boolean) => Promise; /** Make an evicted mirror terminal while its track timeline still contains it. */ evictGroup: (group: GroupConsumer) => void; /** Attach the origin advertisement of a created broadcast. */ @@ -673,6 +686,18 @@ export const hooks: { replaceGroups: () => { throw new Error("track.ts not loaded"); }, + writeDatagram: () => { + throw new Error("track.ts not loaded"); + }, + bindGroupCompression: () => { + throw new Error("group.ts not loaded"); + }, + writeGroupFrame: () => { + throw new Error("group.ts not loaded"); + }, + groupCacheBytes: () => { + throw new Error("group.ts not loaded"); + }, bindGroupTimed: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/lite/compression.test.ts b/js/net/src/lite/compression.test.ts new file mode 100644 index 00000000000..c1aa151ebd9 --- /dev/null +++ b/js/net/src/lite/compression.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from "bun:test"; +import { Manual, Milli, Nano } from "@moq/clock"; +import { accept } from "../connection/accept.ts"; +import { connect } from "../connection/connect.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Producer } from "../origin.ts"; +import * as Path from "../path.ts"; +import { ALPN_06, ALPN_07_WIP } from "./version.ts"; + +for (const downstream of [ALPN_06, ALPN_07_WIP]) { + test(`compressed publishing on ${downstream}, FETCH and independent datagrams`, async () => { + const clock = new Manual(); + const restore = clock.install(); + const source = new Producer(); + const destination = new Producer(); + const broadcast = source.createBroadcast(Path.from("compressed")); + const track = broadcast.createTrack("data", { compress: true }); + broadcast.announce(); + const peers: { abort(): void }[] = []; + async function link(publish: Producer, consume: Producer, alpn: string) { + const pair = createMockTransportPair(alpn); + const url = new URL("https://localhost/compressed"); + peers.push( + ...(await Promise.all([ + connect({ url, transport: pair.client, consume }), + accept({ url, transport: pair.server, publish: publish.consume() }), + ])), + ); + const announcements = consume.announced(); + try { + while ((await announcements.next())?.kind !== "start") {} + } finally { + announcements.close(); + } + } + try { + await link(source, destination, downstream); + const request = destination.request(Path.from("compressed")); + while (request.active.peek() === undefined) await request.active.changed(); + const active = request.active.peek(); + if (!active) throw new Error("broadcast did not resolve"); + const remote = active.track("data"); + expect((await remote.info()).compress).toBe(downstream === ALPN_07_WIP); + const reader = remote.subscribe({ maxDelay: Milli(1000) }); + const group = track.appendGroup(); + const payloads = [new Uint8Array(4096).fill(7), new Uint8Array(), new Uint8Array(4096).fill(7)]; + for (const payload of payloads) group.writeFrame({ payload }); + group.close(); + const received = await reader.recvGroup(); + if (!received) throw new Error("missing group"); + expect(received.sequence).toBe(0); + for (const payload of payloads) expect((await received.readFrame())?.payload).toEqual(payload); + expect(await received.readFrame()).toBeUndefined(); + const fetched = await remote.fetchGroup(0); + for (const payload of payloads) expect((await fetched.readFrame())?.payload).toEqual(payload); + expect(await fetched.readFrame()).toBeUndefined(); + // Each datagram has a cold window; no lost or reordered predecessor is needed. + for (const payload of [new Uint8Array(100).fill(4), new Uint8Array(100).fill(8)]) { + track.appendDatagram(undefined, payload); + expect((await reader.recvDatagram())?.payload).toEqual(payload); + } + reader.close(); + fetched.close(); + received?.close(); + request.close(); + } finally { + for (const origin of [source, destination]) origin.close(); + for (const peer of peers) peer.abort(); + await clock.advanceAsync(Nano.zero); + restore(); + } + }); +} diff --git a/js/net/src/lite/datagram.test.ts b/js/net/src/lite/datagram.test.ts index 2e2893fe2b7..51bb21180d8 100644 --- a/js/net/src/lite/datagram.test.ts +++ b/js/net/src/lite/datagram.test.ts @@ -8,7 +8,7 @@ const dec = new TextDecoder(); test("datagram body round-trips", async () => { for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { const dg = new Datagram(7n, 42, 1000, enc.encode("hello")); - const decoded = await Datagram.decode(dg.encode(version), version, () => true); + const decoded = await Datagram.decode(dg.encode(version), version, () => ({ timed: true })); expect(decoded.subscribe).toBe(7n); expect(decoded.sequence).toBe(42); expect(decoded.timestamp).toBe(1000); @@ -33,7 +33,7 @@ test("datagram body varints follow the version's encoding", () => { test("datagram body round-trips an empty payload", async () => { const dg = new Datagram(0n, 0, 0, new Uint8Array()); - const decoded = await Datagram.decode(dg.encode(Version.DRAFT_07), Version.DRAFT_07, () => true); + const decoded = await Datagram.decode(dg.encode(Version.DRAFT_07), Version.DRAFT_07, () => ({ timed: true })); expect(decoded.payload.byteLength).toBe(0); }); @@ -43,7 +43,7 @@ test("an untimed datagram body carries no timestamp", async () => { expect(body.byteLength).toBe(7); const decoded = await Datagram.decode(body, Version.DRAFT_07, (subscribe) => { expect(subscribe).toBe(1n); - return false; + return { timed: false }; }); expect(decoded.sequence).toBe(2); expect(decoded.timestamp).toBeUndefined(); diff --git a/js/net/src/lite/datagram.ts b/js/net/src/lite/datagram.ts index be4c983558c..951f13d31e1 100644 --- a/js/net/src/lite/datagram.ts +++ b/js/net/src/lite/datagram.ts @@ -4,11 +4,12 @@ * @module */ import { encodeVarint, Reader } from "../stream.ts"; -import { type Version, wireTimescale } from "./version.ts"; +import { hasCompression, type Version, wireTimescale } from "./version.ts"; /** - * A QUIC datagram body: `subscribe (i) | sequence (i) | [timestamp (i)] | payload (b)`, with the - * timestamp present only when the subscription's track carries one (see `wireTimescale`). + * A QUIC datagram body: `subscribe (i) | sequence (i) | [timestamp (i)] | [inflated (i)] | payload (b)`, with the + * timestamp present only when the subscription's track carries one (see `wireTimescale`), + * and the inflated size only on a compressed lite-07 track. * * The payload runs to the datagram boundary, so unlike a size-prefixed lite message there is no * inner length prefix. Mirrors the Rust `lite::Datagram`; the model counterpart is @@ -23,6 +24,8 @@ export class Datagram { timestamp: number | undefined; /** The frame payload, delimited by the datagram boundary. */ payload: Uint8Array; + /** Plain payload length on a compressed lite-07 track. */ + inflated?: number; constructor(subscribe: bigint, sequence: number, timestamp: number | undefined, payload: Uint8Array) { this.subscribe = subscribe; @@ -38,12 +41,19 @@ export class Datagram { throw new Error(`version ${version} has no untimed datagram`); } + if (this.inflated !== undefined && !hasCompression(version)) + throw new Error("compressed datagram needs lite-07"); + const inflated = this.inflated === undefined ? new Uint8Array() : encodeVarint(this.inflated, version); const subscribe = encodeVarint(this.subscribe, version); const sequence = encodeVarint(this.sequence, version); const timestamp = this.timestamp === undefined ? new Uint8Array() : encodeVarint(this.timestamp, version); const out = new Uint8Array( - subscribe.byteLength + sequence.byteLength + timestamp.byteLength + this.payload.byteLength, + subscribe.byteLength + + sequence.byteLength + + timestamp.byteLength + + inflated.length + + this.payload.byteLength, ); let offset = 0; out.set(subscribe, offset); @@ -52,6 +62,8 @@ export class Datagram { offset += sequence.byteLength; out.set(timestamp, offset); offset += timestamp.byteLength; + out.set(inflated, offset); + offset += inflated.length; out.set(this.payload, offset); return out; } @@ -60,12 +72,20 @@ export class Datagram { * Decode a datagram body from the raw bytes of one QUIC datagram. `timed` reports whether the * track of the given subscribe ID carries a timestamp, which the body itself does not say. */ - static async decode(data: Uint8Array, version: Version, timed: (subscribe: bigint) => boolean): Promise { + static async decode( + data: Uint8Array, + version: Version, + framing: (subscribe: bigint) => { timed: boolean; compressed?: boolean }, + ): Promise { const r = new Reader(undefined, data, version); const subscribe = await r.u62(); const sequence = await r.u53(); - const timestamp = timed(subscribe) ? await r.u53() : undefined; + const { timed, compressed } = framing(subscribe); + const timestamp = timed ? await r.u53() : undefined; + const inflated = hasCompression(version) && compressed ? await r.u53() : undefined; const payload = await r.readAll(); - return new Datagram(subscribe, sequence, timestamp, payload); + const result = new Datagram(subscribe, sequence, timestamp, payload); + result.inflated = inflated; + return result; } } diff --git a/js/net/src/lite/group.test.ts b/js/net/src/lite/group.test.ts index d073dffe967..b0f36716873 100644 --- a/js/net/src/lite/group.test.ts +++ b/js/net/src/lite/group.test.ts @@ -37,7 +37,7 @@ test("every frame in a chunk reaches the reader before it wakes", async () => { const producer = new Producer(0); const consumer = producer.consume(); const frames = Array.from({ length: 10 }, (_, index) => ({ delta: 1, payload: [index] })); - const done = readFrames(streamOf([encode(frames)]), producer, SCALE); + const done = readFrames(streamOf([encode(frames)]), producer, { scale: SCALE }); expect((await consumer.readFrame())?.payload).toEqual(new Uint8Array([0])); expect(consumer.frameCount).toBe(10); @@ -57,7 +57,7 @@ test("frames split at every byte keep their payloads and timestamps", async () = const bytes = encode(frames); const chunks = Array.from(bytes, (byte) => new Uint8Array([byte])); - await readFrames(streamOf(chunks), producer, SCALE); + await readFrames(streamOf(chunks), producer, { scale: SCALE }); producer.close(); let ts = 0; @@ -73,7 +73,7 @@ test("frames split at every byte keep their payloads and timestamps", async () = test("frames without a timescale carry no timestamp prefix", async () => { const producer = new Producer(0); const consumer = producer.consume(); - await readFrames(streamOf([encode([{ payload: [7] }, { payload: [8, 9] }])]), producer, 0); + await readFrames(streamOf([encode([{ payload: [7] }, { payload: [8, 9] }])]), producer, { scale: 0 }); producer.close(); expect((await consumer.readFrame())?.payload).toEqual(new Uint8Array([7])); @@ -84,16 +84,16 @@ test("frames without a timescale carry no timestamp prefix", async () => { test("a stream that ends inside a frame rejects", async () => { const producer = new Producer(0); const bytes = encode([{ delta: 1, payload: [1, 2, 3] }]); - await expect(readFrames(streamOf([bytes.subarray(0, bytes.byteLength - 1)]), producer, SCALE)).rejects.toThrow( - "unexpected end of stream", - ); + await expect( + readFrames(streamOf([bytes.subarray(0, bytes.byteLength - 1)]), producer, { scale: SCALE }), + ).rejects.toThrow("unexpected end of stream"); }); test("stops once the group closes", async () => { const producer = new Producer(0); // Never ends: only the close can stop the read. const stream = new Reader(new ReadableStream(), undefined, VERSION); - const done = readFrames(stream, producer, SCALE); + const done = readFrames(stream, producer, { scale: SCALE }); producer.close(); await done; }); diff --git a/js/net/src/lite/group.ts b/js/net/src/lite/group.ts index 90b4bf4ecbe..9cf14169299 100644 --- a/js/net/src/lite/group.ts +++ b/js/net/src/lite/group.ts @@ -1,5 +1,7 @@ import { race } from "@moq/signals"; -import type * as netGroup from "../group.ts"; +import { FrameTooLarge } from "../error.ts"; +import * as netGroup from "../group.ts"; +import { hooks } from "../internal.ts"; import type { Cursor, Reader, Writer } from "../stream.ts"; import * as Time from "../time.ts"; import * as Message from "./message.ts"; @@ -69,42 +71,44 @@ function unzigzag(v: bigint): bigint { return (v >> 1n) ^ -(v & 1n); } -/** - * A synchronous decode for one frame of a group or FETCH response stream. - * - * A non-zero `scale` means every frame is prefixed with a zigzag-delta timestamp (the lite-05 - * FRAME format), decoded into a Timestamp at that scale. Scale 0 (pre-lite-05) carries no - * timestamp, so frames arrive untimed. - */ -export function frameDecoder(scale: number, timed = true): (c: Cursor) => netGroup.Frame { - if (scale === 0) { - return (c) => ({ payload: c.read(c.u53()) }); - } +/** A decoded frame whose payload may still be a stored slice. */ +export interface WireFrame extends netGroup.Frame { + /** The declared plain size on a compressed track. */ + inflated?: number; +} + +/** How {@link frameDecoder} reads each frame's optional fields. */ +export interface FrameFormat { + /** The wire timescale; non-zero means every frame carries a zigzag-delta timestamp. */ + scale: number; + /** Whether a carried timestamp is kept; false drops it where it may be a send time. */ + timed?: boolean; + /** Whether every frame declares its plain size ahead of a stored slice. */ + compress?: boolean; +} - const timescale = Time.Timescale(scale); +/** Decode one frame, with timestamp deltas when scaled and declared plain sizes when compressed. */ +export function frameDecoder({ scale, timed = true, compress = false }: FrameFormat): (c: Cursor) => WireFrame { + const timescale = scale === 0 ? undefined : Time.Timescale(scale); let prevTs = 0n; return (c) => { - const delta = unzigzag(c.u62()); - const payload = c.read(c.u53()); - // After the last read, so a decode that ran short and gets retried adds the delta once. + const delta = timescale === undefined ? 0n : unzigzag(c.u62()); + const inflated = compress ? c.u53() : undefined; + const size = c.u53(); + if (size > netGroup.MAX_GROUP_CACHE_BYTES || (inflated ?? 0) > netGroup.MAX_GROUP_CACHE_BYTES) + throw new FrameTooLarge(); + const payload = c.read(size); + // Commit the delta only after the last read: an incomplete decode gets retried. prevTs += delta; // Read to stay in step, but dropped where it may be a send time. - if (!timed) return { payload }; - return { payload, timestamp: new Time.Timestamp(Number(prevTs), timescale) }; + const timestamp = timescale === undefined || !timed ? undefined : new Time.Timestamp(Number(prevTs), timescale); + return { payload, inflated, timestamp }; }; } -/** - * Write a group stream's frames into `producer` until the stream ends or the producer closes. - * Each frame's timestamp is read at `scale` and kept only when `timed`. - */ -export async function readFrames( - stream: Reader, - producer: netGroup.Producer, - scale: number, - timed = true, -): Promise { - const decode = frameDecoder(scale, timed); +/** Write a group stream's frames into `producer` until the stream ends or the producer closes. */ +export async function readFrames(stream: Reader, producer: netGroup.Producer, format: FrameFormat): Promise { + const decode = frameDecoder(format); for (;;) { // Every frame already buffered is written without an await, so the reader wakes once @@ -112,6 +116,6 @@ export async function readFrames( // that closes first has already closed (or aborted) this group through its cache. const frame = stream.tryDecode(decode) ?? (await race([stream.decodeMaybe(decode), producer.closed])); if (!frame || frame instanceof Error) return; - producer.writeFrame(frame); + hooks.writeGroupFrame(producer, frame, frame.inflated); } } diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index de68d6b03e4..cb53ef66521 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -8,6 +8,7 @@ import type * as broadcast from "../broadcast.ts"; import { Withdrawal } from "../connection/withdrawal.ts"; import * as DatagramStream from "../datagram_stream.ts"; import { error, NotFound, ProtocolViolation, reason, StreamCode, StreamError, unauthorized } from "../error.ts"; +import { stored } from "../flate/group.ts"; import type * as group from "../group.ts"; import { Cost, type Hop, type Route, routesEqual } from "../hop.ts"; import { finalSequence, hiddenBelow, hooks, insertRun, presented, type Run, runGaps } from "../internal.ts"; @@ -43,6 +44,7 @@ import { hasAnnounceId, hasAnnounceOk, hasAnnounceRestart, + hasCompression, hasDatagrams, hasLargest, hasProbeRtt, @@ -116,6 +118,8 @@ interface RunGroup { /** How every frame is stamped, or undefined to send no timestamps. */ stamp: Stamp | undefined; + /** Whether to serve stored slices with their declared plain sizes. */ + compress: boolean; /** The subscription's ranking, which this stream joins for as long as it runs. */ priority: Priority; @@ -172,6 +176,8 @@ type ServeGroup = { sub: bigint; /** How every frame is stamped, or undefined to send no timestamps. */ stamp: Stamp | undefined; + /** Whether to serve stored slices with their declared plain sizes. */ + compress: boolean; /** First frame to send; anything below it was excluded by the subscription. */ start: number; /** Last frame to send (inclusive), or undefined for the rest of the group. */ @@ -870,6 +876,7 @@ export class Publisher { try { // Older drafts carry no timestamps. let stamp: Stamp | undefined; + let compress = false; if (hasTrackStream(this.version)) { // Lite-05+ accepts implicitly: no SUBSCRIBE_OK (the immutable @@ -883,6 +890,7 @@ export class Publisher { // stream. const info = await untilAborted(track.info(), left.signal); stamp = stamping(this.version, liteTimescale(info)); + compress = hasCompression(this.version) && !!info.compress; } else { // Older drafts acknowledge with SUBSCRIBE_OK and stream frames verbatim. const ok = new SubscribeOk({ @@ -929,6 +937,7 @@ export class Publisher { sub: msg.id, broadcast: msg.broadcast, stamp, + compress, served, bounds: { startGroup: msg.startGroup, @@ -1058,6 +1067,7 @@ export class Publisher { } await this.#runFetchGroup(group, stream.writer, { stamp: stamping(this.version, info.timescale), + compress: info.compress, start: msg.startFrame, end: msg.endFrame, }); @@ -1097,11 +1107,12 @@ export class Publisher { sub: bigint; broadcast: Path.Valid; stamp: Stamp | undefined; + compress: boolean; served: Run[] | undefined; bounds: FrameBounds; }, ): Promise { - const { sub, broadcast, stamp, served, bounds } = serving; + const { sub, broadcast, stamp, compress, served, bounds } = serving; // Lite-05+ resolves the range on the subscribe stream: SUBSCRIBE_START once the // first group is known, SUBSCRIBE_END when the track finishes. const emitRange = hasTrackStream(this.version); @@ -1373,6 +1384,7 @@ export class Publisher { sub, group, stamp, + compress, priority, unsubscribed, start: range.start, @@ -1491,6 +1503,7 @@ export class Publisher { // `#serveGroup` emits each frame converted to it. Unchecked timestamps never // enter moq-lite, whose receivers check them. timescale: liteTimescale(info), + compress: hasCompression(this.version) && !!info.compress, }); })(); @@ -1542,7 +1555,10 @@ export class Publisher { // Convert the timestamp to the track's advertised timescale, matching #serveGroup. const ts = stamp === undefined ? undefined : wireTime(datagram.timestamp, stamp); - const body = new DatagramMessage(sub, datagram.sequence, ts, datagram.payload).encode(this.version); + const slice = hasCompression(this.version) ? stored(datagram) : undefined; + const message = new DatagramMessage(sub, datagram.sequence, ts, slice?.payload ?? datagram.payload); + message.inflated = slice?.inflated; + const body = message.encode(this.version); // No group fallback: drop anything that doesn't fit a single datagram. if (body.byteLength > maxSize) { @@ -1566,21 +1582,24 @@ export class Publisher { async #runFetchGroup( group: group.Consumer, stream: Writer, - { stamp, start: startFrame, end: endFrame }: ServeFetch, + { stamp, compress, start: startFrame, end: endFrame }: ServeFetch, ) { // The response carries no header, so the receiver numbers the first frame it gets // as `startFrame`. Skipping the head here is the only thing keeping those numbers // honest; a group that ends before we reach it can't be served at all. for (let i = 0; i < startFrame; i++) { - if (!(await race([group.readFrame(), stream.closed]))) { + const skipped = await race([hooks.readGroupFrame(group, undefined, compress), stream.closed]); + if (!skipped) { throw new Error(`fetch group ended at frame ${i}, before the requested start ${startFrame}`); } + skipped.complete(); } let prevTs = 0n; for (let index = startFrame; endFrame === undefined || index <= endFrame; index++) { - const frame = await race([group.readFrame(), stream.closed]); - if (!frame) break; + const read = await race([hooks.readGroupFrame(group, undefined, compress), stream.closed]); + if (!read) break; + const frame = read.frame; if (stamp !== undefined) { const ts = BigInt(wireTime(frame.timestamp, stamp)); @@ -1588,8 +1607,10 @@ export class Publisher { prevTs = ts; } + if (read.inflated !== undefined) await stream.u53(read.inflated); await stream.u53(frame.payload.byteLength); - await stream.write(frame.payload); + if (frame.payload.length > 0) await stream.write(frame.payload); + read.complete(); } } @@ -1648,7 +1669,7 @@ export class Publisher { * @internal */ async #serveGroup(stream: Writer, options: RunGroup) { - const { sub, group, stamp, priority, start: startFrame, end: endFrame } = options; + const { sub, group, stamp, compress, priority, start: startFrame, end: endFrame } = options; // This model holds whole groups, so frame `startFrame` is always reachable unless // the group ends first. Declaring it up front keeps the stream self-describing. const msg = new GroupMessage({ subscribe: sub, sequence: group.sequence, frameStart: startFrame }); @@ -1670,7 +1691,7 @@ export class Publisher { let reached = startFrame === 0; for (;;) { - const read = await race([hooks.readGroupFrame(group), stream.closed]); + const read = await race([hooks.readGroupFrame(group, undefined, compress), stream.closed]); if (!read) { // The group ended before the frame the subscriber asked to start // at, so this publisher can't serve the range at all. FINning here @@ -1695,8 +1716,11 @@ export class Publisher { prevTs = ts; } + const inflated = read.inflated; + if (inflated !== undefined) await hooks.guardGroup(group, () => stream.u53(inflated)); await hooks.guardGroup(group, () => stream.u53(read.frame.payload.byteLength)); - await hooks.guardGroup(group, () => stream.write(read.frame.payload)); + if (read.frame.payload.length > 0) + await hooks.guardGroup(group, () => stream.write(read.frame.payload)); } finally { read.complete(); } diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index b780692a3d6..e0427a8b5e4 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -35,7 +35,7 @@ import { import { Datagram as DatagramMessage } from "./datagram.ts"; import { closeViolation } from "./error.ts"; import { Fetch as FetchMessage } from "./fetch.ts"; -import { frameDecoder, type Group as GroupMessage, readFrames } from "./group.ts"; +import { frameDecoder, type Group as GroupMessage, readFrames, type WireFrame } from "./group.ts"; import { Probe } from "./probe.ts"; import { ProbeLevel, type Setup } from "./setup.ts"; import { StreamId } from "./stream.ts"; @@ -102,6 +102,7 @@ interface SubscribeEntry { // The write side: incoming GROUP streams are routed here. The application reads // the matching track.Subscriber it got from broadcast.Consumer.subscribe. track: track.Producer; + compress: boolean; // Per-frame timestamp scale (0 = none). undefined until it's known (from TRACK_INFO // on lite-05+, or implicit defaults on older drafts). A non-zero value means each // frame on the group stream is prefixed with a zigzag-delta timestamp varint that @@ -842,6 +843,7 @@ export class Subscriber { state.cancel.signal.throwIfAborted(); let producer: track.Producer; let drainOk = false; + let compress = false; if (hasTrackStream(this.version)) { // Fetch the immutable properties once via the TRACK stream. @@ -857,6 +859,7 @@ export class Subscriber { // register it again or send its SUBSCRIBE. state.cancel.signal.throwIfAborted(); producer = request.accept(this.#toModelInfo(info)); + compress = info.compress; timescale.set(info.timescale ?? 0); } else { // Older drafts negotiate nothing per-track: verbatim frames with no timeline. @@ -871,6 +874,7 @@ export class Subscriber { // Register before opening SUBSCRIBE so a racing GROUP stream finds the entry. const entry: SubscribeEntry = { track: producer, + compress, timescale, // The effective max delay is the stopgap grace: the wrong clock (it bounds // presentation-time drift), but it is how long the subscriber was willing to wait @@ -917,8 +921,6 @@ export class Subscriber { await stream.writer.u53(StreamId.Track); await new TrackMessage(broadcast, track, epoch).encode(stream.writer, this.version); const info = await TrackInfo.decode(stream.reader, this.version); - // Until js/net inflates, a compressed track is refused rather than read as garbage. - if (info.compress) throw new Error("compressed tracks are not supported"); return { info, stream }; }, signal, @@ -972,6 +974,7 @@ export class Subscriber { maxAge: info.maxAge === undefined ? undefined : Milli(info.maxAge), priority: info.priority, refresh: info.refresh, + compress: info.compress, }; } @@ -1108,7 +1111,8 @@ export class Subscriber { throw err; } - void this.#runFetchResponse(accepted.stream, group, accepted.info.timescale ?? 0); + hooks.bindGroupCompression(group, accepted.info.compress); + void this.#runFetchResponse(accepted.stream, group, accepted.info); } catch (err: unknown) { group.close(error(err)); throw err; @@ -1163,9 +1167,13 @@ export class Subscriber { // Read the FETCH response (bare frames, zigzag-delta-timestamped unless `timescale` is 0) // into the group, then FIN. A stream-level failure aborts the group so its reader observes // the gap. - async #runFetchResponse(stream: Stream, group: netGroup.Producer, timescale: number): Promise { + async #runFetchResponse(stream: Stream, group: netGroup.Producer, info: TrackInfo): Promise { try { - const decode = frameDecoder(timescale, hasMediaTime(this.version)); + const decode = frameDecoder({ + scale: info.timescale ?? 0, + timed: hasMediaTime(this.version), + compress: info.compress, + }); // Serve until the stream FINs, the group closes, or every reader leaves. A group can // stay open indefinitely (a catalog or JSON stream), so an abandoned fetch is stopped by @@ -1179,7 +1187,7 @@ export class Subscriber { .then((): typeof idle => idle); // A decode consumes its frame whenever it lands, so one outstanding across a re-arm is // kept and awaited again rather than abandoned with its frame. - let pending: Promise | undefined; + let pending: Promise | undefined; for (;;) { // Buffered frames are written without an await, as in a group stream. let frame = pending === undefined ? stream.reader.tryDecode(decode) : undefined; @@ -1204,7 +1212,7 @@ export class Subscriber { if (!next) break; frame = next; } - group.writeFrame(frame); + hooks.writeGroupFrame(group, frame, frame.inflated); } group.close(); @@ -1457,7 +1465,10 @@ export class Subscriber { scale = timescale.peek(); } - await readFrames(stream, producer, scale, hasMediaTime(this.version)); + const compress = entry.compress; + if (compress && group.frameStart !== 0) + throw new StreamError(StreamCode.TooFarBehind, { message: "compressed group lacks its prefix" }); + await readFrames(stream, producer, { scale, timed: hasMediaTime(this.version), compress }); producer.close(); stream.stop(StreamCode.Cancel); @@ -1519,11 +1530,10 @@ export class Subscriber { // datagram (best-effort) if the subscription is unknown/closed or its timescale isn't resolved. async #routeDatagram(payload: Uint8Array): Promise { // An unknown or unresolved subscription's datagram is dropped below, however it parses. - const dg = await DatagramMessage.decode( - payload, - this.version, - (id) => !!this.#subscribes.get(id)?.timescale.peek(), - ); + const dg = await DatagramMessage.decode(payload, this.version, (id) => ({ + timed: !!this.#subscribes.get(id)?.timescale.peek(), + compressed: this.#subscribes.get(id)?.compress ?? false, + })); const entry = this.#subscribes.get(dg.subscribe); if (!entry) return; // Unknown or already-closed subscription. @@ -1540,7 +1550,7 @@ export class Subscriber { : new Time.Timestamp(dg.timestamp, Time.Timescale(scale)); // A datagram's sequence is never owed a stream, so it never holds the tail open. entry.tail.account(dg.sequence, dg.sequence + 1); - entry.track.insertDatagram(dg.sequence, timestamp, dg.payload); + hooks.writeDatagram(entry.track, { sequence: dg.sequence, timestamp, payload: dg.payload }, dg.inflated); } /** diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index d2f362b6c8f..2e89e3e82e1 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -809,21 +809,25 @@ for (const fails of [true, false]) { }); } -test("track info keeps one answer across routes", async () => { - const front = new Front(); - const a = new BroadcastProducer(); - const b = new BroadcastProducer(); - a.createTrack("seq", { priority: 1 }); - b.createTrack("seq", { priority: 2 }); - front.serve(a.consume()); - expect((await front.consumer.track("seq").info()).priority).toBe(1); - front.serve(b.consume()); - await expect(front.consumer.track("seq").info()).rejects.toThrow("changed its properties"); - - front.consumer.close(); - a.close(); - b.close(); -}); +for (const [field, first, second] of [ + ["priority", { priority: 1 }, { priority: 2 }], + ["compression", { compress: true }, { compress: false }], +] as const) { + test(`track info keeps one ${field} answer across routes`, async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + a.createTrack("seq", first); + b.createTrack("seq", second); + front.serve(a.consume()); + expect(await front.consumer.track("seq").info()).toMatchObject(first); + front.serve(b.consume()); + await expect(front.consumer.track("seq").info()).rejects.toThrow("changed its properties"); + front.consumer.close(); + a.close(); + b.close(); + }); +} test("a datagram reader lets go of the copies a swap replaced", async () => { const front = new Front(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 7f77e8570f2..785841fcfce 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -90,7 +90,12 @@ function verdict(err: Error): boolean { /** Two copies claiming the same content must agree on what it is. */ function compatible(a: track.Info, b: track.Info): boolean { - return a.timescale === b.timescale && a.maxAge === b.maxAge && a.priority === b.priority; + return ( + a.timescale === b.timescale && + a.maxAge === b.maxAge && + a.priority === b.priority && + !!a.compress === !!b.compress + ); } /** diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 3662a2f5cb1..9c90c9a8ba9 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -17,6 +17,7 @@ import { TooFarBehind, toStreamCode, } from "./error.ts"; +import { Compression } from "./flate/group.ts"; import { type Frame, type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { type DropRun, @@ -95,6 +96,8 @@ export interface Location { * (resolved from the wire TRACK_INFO on lite-05+). They map 1:1 onto TRACK_INFO. */ export interface Info { + /** Compress frame payloads per group on lite-07; other sessions receive plain frames. Defaults to false. */ + compress?: boolean; /** * Units per second for this track's frame timestamps, or omitted (`undefined` or `null`, * which mean the same) for an untimed track, whose frames and datagrams carry no timestamp. @@ -120,7 +123,7 @@ export interface Info { /** Tie-break priority between subscriptions of equal subscriber priority (`0..=255`, higher first). Defaults to `127`. */ priority: number; /** - * How many times its first frame a group's later bytes may grow before a joining + * How many times its first frame a group's later stored bytes may grow before a joining * subscriber asks for a new group, or omitted to never ask. A join against a newest group * whose bytes after its first frame exceed this many times the first frame moves * {@link Demand.wanted} past it, so a late joiner never downloads much more than a @@ -170,10 +173,13 @@ export function liteTimescale(info: Info): Timescale | undefined { /** Fill in any unset {@link Info} fields with their defaults. */ export function infoDefaults(info: Partial = {}): Info { + if (info.compress !== undefined && typeof info.compress !== "boolean") + throw new TypeError("compress must be a boolean"); const timescale = info.timescale == null ? undefined : Timescale(info.timescale); // Only a timed track has timestamps to doubt. if (info.unchecked && timescale === undefined) throw new RangeError("an untimed track can't be unchecked"); return { + compress: info.compress ?? false, timescale, maxAge: info.maxAge === undefined ? undefined : wireMillis("maxAge", info.maxAge), priority: priorityByte(info.priority ?? DEFAULT_PRIORITY), @@ -767,6 +773,7 @@ export class Producer { } static { + hooks.writeDatagram = (producer, datagram, inflated) => producer.#insertDatagram(datagram, inflated); sourceStart = (producer) => producer.#served && producer.#feedStart(); hooks.producerLargest = (producer) => producer.#state.largest; hooks.joinProducer = (producer) => producer.#join(); @@ -914,7 +921,10 @@ export class Producer { // Anything written before accept() must match too. A mismatch closes the track, so // nothing waiting on info() hangs on a half-bound cache. try { - for (const { group } of this.#cache) hooks.bindGroupTimed(group, timed); + for (const { group } of this.#cache) { + hooks.bindGroupTimed(group, timed); + hooks.bindGroupCompression(group, resolved.compress ?? false); + } for (const sink of this.#sinks) { for (const datagram of sink.datagrams.peek()) { if ((datagram.timestamp !== undefined) !== timed) throw new TimestampMismatch(); @@ -926,7 +936,25 @@ export class Producer { } this.#state.info.set(resolved); // Propagate to any sink handed out before accept (the on-demand path). - for (const sink of this.#sinks) sink.info.set(resolved); + const compressed = new Map(); + for (const sink of this.#sinks) { + if (resolved.compress) { + sink.datagrams.mutate((datagrams) => { + for (let index = 0; index < datagrams.length; index++) { + const datagram = datagrams[index]; + let stored = compressed.get(datagram); + if (!stored) { + const flate = new Compression(); + stored = Object.assign(flate.push(datagram), { sequence: datagram.sequence }); + flate.finish(); + compressed.set(datagram, stored); + } + datagrams[index] = stored; + } + }); + } + sink.info.set(resolved); + } this.#updateSubscription(); return this; } @@ -1344,7 +1372,10 @@ export class Producer { // Hold a group to this track's timedness, once the track has committed it. #bind(group: GroupProducer): void { const info = this.#state.info.peek(); - if (info) hooks.bindGroupTimed(group, info.timescale != null); + if (info) { + hooks.bindGroupTimed(group, info.timescale != null); + hooks.bindGroupCompression(group, info.compress ?? false); + } } // Refuse a datagram whose timedness disagrees with this track's. @@ -1460,7 +1491,7 @@ export class Producer { if (payload.byteLength > MAX_DATAGRAM_BYTES) throw new Error("datagram payload too large"); this.#checkDatagram(timestamp); - this.#publishDatagram({ sequence, timestamp, payload }); + this.#insertDatagram({ sequence, timestamp, payload }); return sequence; } @@ -1472,11 +1503,29 @@ export class Producer { * {@link appendDatagram} apply. Most origin publishers want {@link appendDatagram} instead. */ insertDatagram(sequence: number, timestamp: Timestamp | undefined, payload: Uint8Array) { + this.#insertDatagram({ sequence, timestamp, payload }); + } + + #insertDatagram(datagram: Datagram, inflated?: number): void { + const { sequence, timestamp, payload } = datagram; this.#writable(sequence); - if (payload.byteLength > MAX_DATAGRAM_BYTES) throw new Error("datagram payload too large"); this.#checkDatagram(timestamp); - - this.#publishDatagram({ sequence, timestamp, payload }); + if (payload.length > MAX_DATAGRAM_BYTES || (inflated ?? 0) > MAX_DATAGRAM_BYTES) + throw new Error("datagram payload too large"); + if ( + inflated !== undefined && + (!Number.isSafeInteger(inflated) || inflated < 0 || !this.#state.info.peek()?.compress) + ) + throw new ProtocolViolation("unexpected inflated datagram size"); + if (this.#state.info.peek()?.compress) { + const flate = new Compression(); + const frame = flate.push(datagram, inflated); + // A bad best-effort datagram is dropped on arrival, before any reader receives it. + if (inflated !== undefined) void frame.payload; + flate.finish(); + datagram = Object.assign(frame, { sequence }); + } + this.#publishDatagram(datagram); } /** diff --git a/quest/m1/audit-size.md b/quest/m1/audit-size.md index 9e13fa78d66..9f1e4f1f07a 100644 --- a/quest/m1/audit-size.md +++ b/quest/m1/audit-size.md @@ -29,7 +29,7 @@ It needs the Linux release builds, which still fail to link ([moq-dev/moq#6090](https://github.com/moq-dev/moq/pull/6090) was open on 2026-10-21). If it has, its history replaces the local method below. Then re-attribute the native rows for the erasure quest, and measure `@moq/net` -once [compressed tracks in js/net](/quest/m1/compress/js.md) moves `pako` in. +with the track compression codec in `js/net`. ### Baselines @@ -90,7 +90,7 @@ Leads: - `pako` costs `watch` and `publish` 12.8 KB gzip through `@moq/json`. A player only inflates: `Inflate` alone is 9.9 KB, and the browser's `DecompressionStream("deflate-raw")` would be free if it yields each - sync-flushed frame. Carried into [compressed tracks in js/net](/quest/m1/compress/js.md). + sync-flushed frame. Synchronous track writes retain the static encoder; the maintainer accepted its measured 3 KB gzip cost. - `zod/mini` is 9 KB raw of `@moq/net`'s first load for one branded bigint (`HopSchema`), and `@moq/hang` imports it as both `zod/mini` and `@zod/mini`. Not worth a quest alone. diff --git a/quest/m1/compress/README.md b/quest/m1/compress/README.md index 0557cde536d..7dd2a07581f 100644 --- a/quest/m1/compress/README.md +++ b/quest/m1/compress/README.md @@ -82,7 +82,6 @@ This README owns the end-to-end proof and the guide once every child lands: ## Required - [Mid-group subscriptions widen](/quest/m1/compress/app-range-widen.md) - an application range starting mid-group on a compressed track widens to frame 0 at the subscriber instead of failing with `Lagged` -- [JavaScript](/quest/m1/compress/js.md) - js/net publishes and consumes compressed tracks on lite-07 - [Migrate](/quest/m1/compress/migrate.md) - catalogs, stats, and timelines publish one compressed track; moq-flate, @moq/flate, and every `.z` name are gone ## Related diff --git a/quest/m1/compress/app-range-widen.md b/quest/m1/compress/app-range-widen.md index 6aab29b3b3b..75fcfed2326 100644 --- a/quest/m1/compress/app-range-widen.md +++ b/quest/m1/compress/app-range-widen.md @@ -25,7 +25,7 @@ widens, never the server. Regression test on mock time, in Rust: a mid-group application subscribe on a compressed track returns the requested frames instead of `Lagged`, and the -wire request starts at frame 0. Mirror it in `js/net` if -[JavaScript](/quest/m1/compress/js.md) has landed compressed tracks by then. +wire request starts at frame 0. Mirror it in `js/net`, which carries +compressed tracks too. Public API: none. Wire: none. diff --git a/quest/m1/compress/js.md b/quest/m1/compress/js.md deleted file mode 100644 index 02c70259d99..00000000000 --- a/quest/m1/compress/js.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] Compressed tracks in js/net - -## Goal - -A JS publisher sets `compress` on a track's `Info` and js/net DEFLATEs each -group on lite-07, and a JS consumer reads a compressed lite-07 track as plain -frames. On any other session a JS publisher sends plain frames without the -flag, and the refusal Core added is gone. - -## Plan - -See the [epic](/quest/m1/compress/README.md) for the decisions. - -- Move the codec from `js/flate/src/codec.ts` into js/net, crate-private. - `@moq/flate` is deleted in [Migrate](/quest/m1/compress/migrate.md). -- Mirror the Rust model: store compressed slices, one inflated copy per - group shared by plain readers, the declared size enforced while inflating. -- Docs inline beside the Rust ones in `doc/concept`. -- Size: `pako` is 12.8 KB gzip of the `watch` and `publish` bundles today, - and moving it into js/net puts it in every first load. A player only - inflates: keep deflate out of a consumer's static graph (`Inflate` alone is - 9.9 KB), or use `DecompressionStream("deflate-raw")` if it yields each - sync-flushed frame. Measure with `sh/size/bundles.ts` - ([size audit](/quest/m1/audit-size.md)). - -Tests: Rust and JS publish compressed tracks to each other over lite-07 -through moq-relay; a JS publisher on lite-06 sends plain frames. - -Public API: `Info.compress` in `@moq/net`. Wire: none beyond Core. diff --git a/quest/m1/compress/migrate.md b/quest/m1/compress/migrate.md index 47addf6d628..76915941907 100644 --- a/quest/m1/compress/migrate.md +++ b/quest/m1/compress/migrate.md @@ -60,6 +60,5 @@ Public API: removals listed above. Wire: the hang catalog loses ## Required -- [JavaScript](/quest/m1/compress/js.md) - js/publish and js/watch need the JS side before they drop `.z` - [Catalog HLS field](/quest/m1/archive/catalog-hls.md) - renames the timeline to `.hls`, which this compresses - [Archive consumer](/quest/m1/archive/consumer.md) - replaces the CLI archive reader that reads `.z` timelines diff --git a/quest/m1/lite07-finalize.md b/quest/m1/lite07-finalize.md index 2948209b5f4..83d90ff6813 100644 --- a/quest/m1/lite07-finalize.md +++ b/quest/m1/lite07-finalize.md @@ -94,7 +94,6 @@ is published; older versions are unchanged. - [Drop set](/quest/m1/drop-set.md) - lite-07's SUBSCRIBE_DROP carries `Max Age`, so a widening re-offers groups reported dropped - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - SUBSCRIBE carries ranges and an order and lite FETCH is gone, in Rust and JS - [Announcement takeover](/quest/m1/announce-takeover.md) - ANNOUNCE_TAKEOVER preserves a proven instance under a new independent route -- [Compressed tracks in js/net](/quest/m1/compress/js.md) - JS reads and writes compressed lite-07 tracks instead of refusing them - [TRACKS in js/net](/quest/m1/archive/tracks-js.md) - JS speaks lite-07's Tracks Stream, which Rust already does - [lite-07 padding](/quest/m1/lite07-padding.md) - lite-07 reserves the padding stream type the browser probes with diff --git a/quest/m2/native-compression-adopt.md b/quest/m2/native-compression-adopt.md index 48c4d58abf0..942d426832c 100644 --- a/quest/m2/native-compression-adopt.md +++ b/quest/m2/native-compression-adopt.md @@ -19,11 +19,11 @@ main-thread blocking through workers, or a new preparation API. starting implementation. Constructor support alone is insufficient. Check nonfinal frame output, shared history, exact declared-size admission, early output-overrun refusal, cancellation, malformed input and group end. -- Native decoding is already an avenue in the JS compression quest and size - audit. Coordinate with that implementation and remove only duplicated - conditional scope once its actual result is known. Do not redo a landed - native decoder. Encoder and decoder adoption may land independently if - each preserves the existing contract and removes worthwhile bundled code. +- Native decoding is also an avenue in the size audit. JS compression landed + with a bundled pako codec in `js/net/src/flate`; coordinate with the audit + and remove only duplicated conditional scope. Encoder and decoder adoption + may land independently if each preserves the existing contract and removes + worthwhile bundled code. - Chromium 156 decoded the tested nonfinal frames before closure; this does not establish synchronous availability, cross-engine behavior or bounded allocation. Verify those properties rather than treating the experiment @@ -40,10 +40,6 @@ main-thread blocking through workers, or a new preparation API. change needs a new maintainer decision. Update affected docs inline; the compression guide owns the user-facing entry point, not a new page. -## Required - -- [JS compression](/quest/m1/compress/js.md) - the current compressed-track model and interoperability behavior are implemented - ## Related - [Frame flush report](/quest/m2/native-compression-flush.md) - the native nonfinal encoder flush gap diff --git a/test/interop/README.md b/test/interop/README.md index 916b5c45fa9..f186857c862 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -468,3 +468,7 @@ JS publisher sends. drafts 14 through 22 through `track-status.ts`, and Rust decodes each `TRACK_STATUS_OK` and the refusal for a missing broadcast the way its subscriber reads them. + +The finite-tail matrix also publishes compressed tracks from Rust and native JS through +the relay, in both directions on lite-07 and lite-06. It checks multiple frames in each +DEFLATE window, empty frames, the plain fallback and the received track's codec flag. diff --git a/test/interop/clients/js-native/tail.ts b/test/interop/clients/js-native/tail.ts index cbeb8a60d2b..9a02279542e 100644 --- a/test/interop/clients/js-native/tail.ts +++ b/test/interop/clients/js-native/tail.ts @@ -3,7 +3,8 @@ * * `drop` cuts group 1. `join` reads from whichever group the subscription starts at: a * moq-transport subscription starts at the largest group rather than at the range asked for, so - * it must read a contiguous run of groups through the last one. + * it must read a contiguous run of groups through the last one. `compress` publishes a compressed + * track with several frames, one empty, in each group. * * @module */ @@ -19,8 +20,14 @@ const BYTES = 256 * 1024; const [role, url, path, version, mode] = process.argv.slice(2); const drop = mode === "drop"; const join = mode === "join"; -if (!url || !path || !["publish", "subscribe"].includes(role) || ![undefined, "drop", "join"].includes(mode)) { - throw new Error("usage: tail.ts publish|subscribe URL BROADCAST [VERSION [drop|join]]"); +const compress = mode === "compress"; +if ( + !url || + !path || + !["publish", "subscribe"].includes(role) || + ![undefined, "drop", "join", "compress"].includes(mode) +) { + throw new Error("usage: tail.ts publish|subscribe URL BROADCAST [VERSION [drop|join|compress]]"); } const origin = new Moq.Origin.Producer(); @@ -33,7 +40,7 @@ const connection = await Moq.Connection.connect({ try { if (role === "publish") { const broadcast = origin.createBroadcast(Moq.Path.from(path)); - const track = broadcast.createTrack("tail"); + const track = broadcast.createTrack("tail", { compress }); broadcast.announce(); const demand = track.demand(); while (demand.wanted.peek() === undefined) await demand.wanted.changed(); @@ -45,7 +52,8 @@ try { group.close(new Error("cut")); continue; } - group.writeFrame({ payload: new Uint8Array(BYTES).fill(sequence) }); + const sizes = compress ? [BYTES / 2, 0, BYTES / 2] : [BYTES]; + for (const size of sizes) group.writeFrame({ payload: new Uint8Array(size).fill(sequence) }); group.close(); } track.close(); @@ -69,6 +77,7 @@ try { const track = broadcast .track("tail") .subscribe({ maxDelay: Milli(drop ? 60_000 : 5000), groups: { start: { included: 0 } } }); + if (compress) assert.equal((await track.info()).compress, version === "moq-lite-07-wip"); const seen: number[] = []; try { for (;;) { @@ -76,13 +85,15 @@ try { if (!group) break; try { assert(group.sequence < GROUPS, `unexpected group ${group.sequence}`); - const frame = await group.readFrame(); - assert(frame, "empty group"); - assert.equal(frame.payload.byteLength, BYTES, "truncated group"); - assert( - frame.payload.every((byte) => byte === group.sequence), - "corrupt group", - ); + for (const size of compress ? [BYTES / 2, 0, BYTES / 2] : [BYTES]) { + const frame = await group.readFrame(); + assert(frame, "empty group"); + assert.equal(frame.payload.byteLength, size, "truncated group"); + assert( + frame.payload.every((byte) => byte === group.sequence), + "corrupt group", + ); + } assert.equal(await group.readFrame(), undefined, "extra frame"); seen.push(group.sequence); } finally { diff --git a/test/interop/clients/rust/tail.rs b/test/interop/clients/rust/tail.rs index a27b2d8a0f4..f4c1dc76ddd 100644 --- a/test/interop/clients/rust/tail.rs +++ b/test/interop/clients/rust/tail.rs @@ -4,7 +4,8 @@ //! moq-transport subscription starts at the largest group rather than at the range asked for, //! so it must read a contiguous run of groups through the last one. `cached` publishes every //! group before connecting, so a FETCH of a past group never races its write: moq-transport -//! refuses a FETCH that starts past the largest group rather than waiting for it. +//! refuses a FETCH that starts past the largest group rather than waiting for it. `compress` +//! publishes a compressed track with several frames, one empty, in each group. use std::time::Duration; use anyhow::{Context, ensure}; @@ -14,8 +15,13 @@ use tokio::io::AsyncBufReadExt; const GROUPS: u64 = 4; const BYTES: usize = 256 * 1024; +/// Frame sizes in each group: a compressed track spans several frames, one empty, per window. +fn sizes(compress: bool) -> &'static [usize] { + if compress { &[BYTES / 2, 0, BYTES / 2] } else { &[BYTES] } +} + /// Publish every group, cutting group 1 for `drop`. -fn write(track: &mut moq_net::track::Producer, drop: bool) -> anyhow::Result<()> { +fn write(track: &mut moq_net::track::Producer, drop: bool, compress: bool) -> anyhow::Result<()> { // Declare the end before the final group's payload is even available. track.finish_at(moq_net::track::Position::group(GROUPS))?; for sequence in 0..GROUPS { @@ -24,7 +30,9 @@ fn write(track: &mut moq_net::track::Producer, drop: bool) -> anyhow::Result<()> group.abort(moq_net::Error::App(1))?; continue; } - group.write_frame(moq_net::Timestamp::ZERO, vec![sequence as u8; BYTES])?; + for &size in sizes(compress) { + group.write_frame(moq_net::Timestamp::ZERO, vec![sequence as u8; size])?; + } group.finish()?; } Ok(()) @@ -35,17 +43,18 @@ async fn main() -> anyhow::Result<()> { let args: Vec<_> = std::env::args().collect(); ensure!( (4..=6).contains(&args.len()), - "usage: interop-tail publish|subscribe URL BROADCAST [VERSION [drop|join|cached]]" + "usage: interop-tail publish|subscribe URL BROADCAST [VERSION [drop|join|cached|compress]]" ); let url: url::Url = args[2].parse()?; let mode = args.get(5).map(String::as_str); ensure!( - matches!(mode, None | Some("drop" | "join" | "cached")), + matches!(mode, None | Some("drop" | "join" | "cached" | "compress")), "unknown mode: {mode:?}" ); let drop = mode == Some("drop"); let join = mode == Some("join"); let cached = mode == Some("cached"); + let compress = mode == Some("compress"); let mut config = moq_tokio::connect::Config::default(); // An empty version offers the defaults, so a mode can follow it. if let Some(version) = args.get(4).filter(|version| !version.is_empty()) { @@ -60,11 +69,13 @@ async fn main() -> anyhow::Result<()> { let broadcast = origin.create_broadcast(&args[3])?; let mut track = broadcast.create_track( "tail", - moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_compress(compress), )?; broadcast.announce(Default::default())?; if cached { - write(&mut track, drop)?; + write(&mut track, drop, compress)?; } let session = client .clone() @@ -75,7 +86,7 @@ async fn main() -> anyhow::Result<()> { if !cached { let mut demand = track.demand(); while demand.wanted_changed().await?.is_none() {} - write(&mut track, drop)?; + write(&mut track, drop, compress)?; } // A finished track is not a drained transport. Keep the session alive // until the harness confirms the subscriber checked every byte and EOF. @@ -101,16 +112,24 @@ async fn main() -> anyhow::Result<()> { .with_max_delay(Duration::from_secs(if drop { 60 } else { 5 })) .with_groups(0..); let mut track = broadcast.track("tail")?.subscribe(Some(options)).await?; + if compress { + ensure!( + track.info().compress == args.get(4).is_some_and(|version| version == "moq-lite-07-wip"), + "wrong compression metadata" + ); + } let mut seen = Vec::new(); while let Some(mut group) = track.recv_group().await? { let sequence = group.sequence; - let frame = group.read_frame().await?.context("empty group")?; ensure!(sequence < GROUPS, "unexpected group {sequence}"); - ensure!(frame.payload.len() == BYTES, "truncated group {sequence}"); - ensure!( - frame.payload.iter().all(|byte| *byte == sequence as u8), - "corrupt group {sequence}" - ); + for &size in sizes(compress) { + let frame = group.read_frame().await?.context("empty group")?; + ensure!(frame.payload.len() == size, "truncated group {sequence}"); + ensure!( + frame.payload.iter().all(|byte| *byte == sequence as u8), + "corrupt group {sequence}" + ); + } ensure!(group.read_frame().await?.is_none(), "extra frame in group {sequence}"); seen.push(sequence); } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 92ec8c6a3dd..bb5309fd1a4 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -301,6 +301,8 @@ SUITE=interop # The lite drafts the finite-track drop lanes cross between Rust and Node. DROP_VERSIONS=(moq-lite-05 moq-lite-06 moq-lite-07-wip) +# The lite drafts the compressed-track lanes cross: lite-07 carries the flag, lite-06 gets inflated bytes. +COMPRESS_VERSIONS=(moq-lite-07-wip moq-lite-06) # Record a case for the CI report: `record case=C [field=value...]` (see # harness_result). Every session in a case dials the same relay with each client's own @@ -371,7 +373,15 @@ plan_pass() { record "not run" case=tail publisher=rust subscriber="$runtime" record "not run" case=tail publisher="$runtime" subscriber=rust done - [[ "$VERSION" == auto ]] && needs js-native-node || return 0 + [[ "$VERSION" == auto ]] || return 0 + for runtime in js-native-node js-native-bun; do + needs "$runtime" || continue + for version in "${COMPRESS_VERSIONS[@]}"; do + record "not run" case=compress publisher=rust subscriber="$runtime" version="$version" + record "not run" case=compress publisher="$runtime" subscriber=rust version="$version" + done + done + needs js-native-node || return 0 for version in "${DROP_VERSIONS[@]}"; do record "not run" case=drop publisher=rust subscriber=js-native-node version="$version" record "not run" case=drop publisher=js-native-node subscriber=rust version="$version" @@ -1240,18 +1250,24 @@ run_tail_publisher() { } run_tail_pair() { - local pub="$1" sub="$2" name="tail-$1-$2" pid subscriber_pid started=$SECONDS from version read - local fifo="$LOGS/tail-ack" broadcast="tail-$1-$2-$$-$RANDOM" mode="" want='tail clean end=4 groups=0,1,2,3 bytes=1048576' - version=$(pinned_version) - # moq-net subscribes over moq-transport from the largest group, which can be past group 0 by - # the time the relay answers, so the reader takes any contiguous run through the last group. - if [[ "$version" == moq-transport-* ]]; then - mode="join" - want='tail join clean end=4 groups=[0-3](,[0-3])*' + # run_tail_pair [ ]: a mode pins its own version, like a drop lane. + local pub="$1" sub="$2" mode="${3:-}" version="${4:-}" pid subscriber_pid started=$SECONDS from read + local name="tail-$1-$2${mode:+-$mode-$version}" label="${mode:-tail}${version:+ $version}" + local fifo="$LOGS/tail-ack" broadcast="tail-$1-$2-$$-$RANDOM" want='tail clean end=4 groups=0,1,2,3 bytes=1048576' + local cell=(case="${mode:-tail}" publisher="$pub" subscriber="$sub" log="$LOGS/$name-sub.log") + if [[ -n "$mode" ]]; then + cell+=(version="$version") + else + version=$(pinned_version) + # moq-net subscribes over moq-transport from the largest group, which can be past group 0 by + # the time the relay answers, so the reader takes any contiguous run through the last group. + if [[ "$version" == moq-transport-* ]]; then + mode="join" + want='tail join clean end=4 groups=[0-3](,[0-3])*' + fi fi - local cell=(case=tail publisher="$pub" subscriber="$sub" log="$LOGS/$name-sub.log") if is_broken "$pub" || is_broken "$sub"; then - echo " FAIL tail $pub -> $sub (client unavailable)" + echo " FAIL $label $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" overall=1 return @@ -1270,20 +1286,20 @@ run_tail_pair() { if read=$(grep -xE "$want" "$LOGS/$name-sub.log"); then printf 'clean end\n' >&9 harness_wait "$pid" || true - if [[ -n "$version" ]] && ! check_versions "tail $pub -> $sub" "$from" "$version"; then + if [[ -n "$version" ]] && ! check_versions "$label $pub -> $sub" "$from" "$version"; then record fail "${cell[@]}" seconds=$((SECONDS - started)) detail="a relay connection negotiated another version" overall=1 elif grep -qx 'tail acknowledged' "$LOGS/$name-pub.log"; then - echo " PASS tail $pub -> $sub (${read#tail })" + echo " PASS $label $pub -> $sub (${read#tail })" record pass "${cell[@]}" seconds=$((SECONDS - started)) else - echo " FAIL tail $pub -> $sub (publisher did not acknowledge)" + echo " FAIL $label $pub -> $sub (publisher did not acknowledge)" record fail "${cell[@]}" seconds=$((SECONDS - started)) detail="publisher did not acknowledge" overall=1 cat "$LOGS/$name-pub.log" fi else - echo " FAIL tail $pub -> $sub (missing groups or clean end)" + echo " FAIL $label $pub -> $sub (missing groups or clean end)" record fail "${cell[@]}" seconds=$((SECONDS - started)) detail="missing groups or clean end" overall=1 cat "$LOGS/$name-pub.log" "$LOGS/$name-sub.log" @@ -1462,9 +1478,9 @@ run_matrix() { fi } -# This checkout's CLI FETCH and finite track tails in this pass, then the lite drop lanes. +# This checkout's CLI FETCH and finite track tails in this pass, then the lite compress and drop lanes. run_tails() { - local runtime + local runtime version echo "=== CLI FETCH close ===" run_fetch_round echo "=== finite track tails ===" @@ -1475,9 +1491,16 @@ run_tails() { run_tail_pair "$runtime" rust fi done - # The drop lanes pin their own lite versions. - if [[ "$VERSION" == auto ]] && needs js-native-node; then - local version + # The compress and drop lanes pin their own lite versions. + [[ "$VERSION" == auto ]] || return 0 + for runtime in js-native-node js-native-bun; do + needs "$runtime" || continue + for version in "${COMPRESS_VERSIONS[@]}"; do + run_tail_pair rust "$runtime" compress "$version" + run_tail_pair "$runtime" rust compress "$version" + done + done + if needs js-native-node; then for version in "${DROP_VERSIONS[@]}"; do run_drop_pair rust js-native-node "$version" run_drop_pair js-native-node rust "$version" diff --git a/test/interop/lite-varint.ts b/test/interop/lite-varint.ts index 58faec72a44..356ddc87a70 100644 --- a/test/interop/lite-varint.ts +++ b/test/interop/lite-varint.ts @@ -64,7 +64,7 @@ if (version === Version.DRAFT_07) { const setup = await Setup.decode(reader(input.setup), version); const setupBytes = await write((w) => setup.encode(w, version)); -const datagram = await Datagram.decode(Uint8Array.from(input.datagram), version, () => true); +const datagram = await Datagram.decode(Uint8Array.from(input.datagram), version, () => ({ timed: true })); const datagramBytes = [...datagram.encode(version)]; // Only lite-07 can carry an untimed track; lite-06 stamps it, so it has no untimed bytes. @@ -72,7 +72,7 @@ const untimed = version === Version.DRAFT_07; let untimedDatagramBytes: number[] = []; if (untimed) { - const datagram = await Datagram.decode(Uint8Array.from(input.untimedDatagram), version, () => false); + const datagram = await Datagram.decode(Uint8Array.from(input.untimedDatagram), version, () => ({ timed: false })); assert.equal(datagram.timestamp, undefined); untimedDatagramBytes = [...datagram.encode(version)]; } else { @@ -86,7 +86,7 @@ const trackInfoBytes = await write((w) => trackInfo.encode(w, version)); // A GROUP header, then frames until the stream ends. const g = reader(input.group); const group = await Group.decode(g, version); -const decode = frameDecoder(1_000_000); +const decode = frameDecoder({ scale: 1_000_000 }); const frames: { timestamp: bigint; payload: Uint8Array }[] = []; for (;;) { const frame = await g.decodeMaybe(decode); @@ -113,7 +113,7 @@ if (untimed) { const untimedGroup = await Group.decode(u, version); const untimedFrames: Uint8Array[] = []; for (;;) { - const frame = await u.decodeMaybe(frameDecoder(0)); + const frame = await u.decodeMaybe(frameDecoder({ scale: 0 })); if (!frame) break; assert.equal(frame.timestamp, undefined); untimedFrames.push(frame.payload);