diff --git a/bun.lock b/bun.lock index 60b672d8524..fa90358b6fa 100644 --- a/bun.lock +++ b/bun.lock @@ -177,6 +177,15 @@ "typescript": "7.0.2", }, }, + "js/media-sync": { + "name": "@moq/media-sync", + "version": "0.1.0", + "devDependencies": { + "@types/bun": "^1.4.2", + "rimraf": "^6.1.3", + "typescript": "7.0.2", + }, + }, "js/moq-boy": { "name": "@moq/boy", "version": "0.4.1", @@ -708,6 +717,8 @@ "@moq/loc": ["@moq/loc@workspace:js/loc"], + "@moq/media-sync": ["@moq/media-sync@workspace:js/media-sync"], + "@moq/msf": ["@moq/msf@workspace:js/msf"], "@moq/net": ["@moq/net@workspace:js/net"], diff --git a/js/media-sync/README.md b/js/media-sync/README.md new file mode 100644 index 00000000000..a86b0c95b90 --- /dev/null +++ b/js/media-sync/README.md @@ -0,0 +1,24 @@ +# @moq/media-sync + +Transport-neutral media clock, pacing, and generation-safe per-track delivery queues. +No dependencies: no transport, container, decoder, or renderer. + +```ts +import { MediaSync } from "@moq/media-sync"; + +const sync = new MediaSync({ buffer: { video: 50, audio: 100 }, timestampScale: 1_000 }); // µs timestamps +sync.enqueue(frame.timestamp, "video", async (signal) => { + const decoded = await decode(frame, signal); // stage the effect + return () => render(decoded); // commit only if the generation is still current +}); +sync.cancel("video"); // retire the video generation: staged commits are discarded +``` + +- `received(timestamp)` seeds the clock. It must run before `wait()`, which throws on an unseeded clock. +- `timestampScale` is the number of media timestamp units per clock unit, so callers keep their own units. +- `cancel(kind)` and `reset()` retire a generation: nothing staged before the retirement commits after it, + including stages that are mid-`await`, and timestamps may be reused afterwards. +- `reset()` also clears the clock and wakes every waiter, so parked waits release. + +`@moq/media-sync/vectors` exports the clock, pacing, and retirement vectors, so a wrapper that adapts units +or callbacks can be tested against the same cases as the core. diff --git a/js/media-sync/package.json b/js/media-sync/package.json new file mode 100644 index 00000000000..601c23c819b --- /dev/null +++ b/js/media-sync/package.json @@ -0,0 +1,28 @@ +{ + "name": "@moq/media-sync", + "type": "module", + "version": "0.1.0", + "description": "Transport-neutral media clock, pacing, and generation-safe delivery queues.", + "license": "(MIT OR Apache-2.0)", + "repository": { + "type": "git", + "url": "git+https://github.com/moq-dev/moq.git", + "directory": "js/media-sync" + }, + "sideEffects": false, + "exports": { + ".": "./src/index.ts", + "./vectors": "./src/vectors.ts" + }, + "scripts": { + "build": "rimraf dist && tsc -b tsconfig.build.json && bun ../common/package.ts", + "check": "tsc --noEmit", + "test": "bun test --only-failures", + "release": "bun ../common/release.ts" + }, + "devDependencies": { + "@types/bun": "^1.4.2", + "rimraf": "^6.1.3", + "typescript": "7.0.2" + } +} diff --git a/js/media-sync/src/index.test.ts b/js/media-sync/src/index.test.ts new file mode 100644 index 00000000000..4288db48794 --- /dev/null +++ b/js/media-sync/src/index.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from "bun:test"; +import { MediaSync } from "./index.ts"; +import { MEDIA_SYNC_CLOCK_VECTORS, MEDIA_SYNC_DELIVERY_VECTORS, MEDIA_SYNC_PACING_VECTORS } from "./vectors.ts"; + +describe("MediaSync", () => { + for (const vector of MEDIA_SYNC_CLOCK_VECTORS) { + it(`runs the shared clock vector: ${vector.name}`, () => { + let now: number = vector.anchorClock; + const sync = new MediaSync({ + now: () => now, + buffer: vector.buffer, + timestampScale: vector.timestampScale, + }); + sync.received(vector.anchorTimestamp); + now = vector.readClock; + + expect(sync.reference).toBe(vector.expected.reference); + expect(sync.timestamp).toBe(vector.expected.timestamp); + expect(sync.now("video")).toBe(vector.expected.videoNow); + expect(sync.now("audio")).toBe(vector.expected.audioNow); + }); + } + + for (const vector of MEDIA_SYNC_PACING_VECTORS) { + it(`runs the shared pacing vector: ${vector.name}`, async () => { + let now: number = vector.anchorClockMs; + const sync = new MediaSync({ now: () => now, buffer: vector.bufferMs }); + sync.received(vector.anchorTimestampMs); + now = vector.earlyClockMs; + sync.received(vector.deliveryTimestampMs); + let delivered = false; + sync.enqueue(vector.deliveryTimestampMs, vector.kind, () => () => (delivered = true)); + await Promise.resolve(); + await Promise.resolve(); + expect(delivered).toBe(false); + + now = vector.dueClockMs; + sync.received(vector.anchorTimestampMs); + await sync.drain(vector.kind); + expect(delivered).toBe(true); + }); + } + + it("clears clock state on reset", () => { + const sync = new MediaSync({ now: () => 1_000 }); + sync.received(900); + sync.reset(); + expect(sync.reference).toBeUndefined(); + expect(sync.timestamp).toBeUndefined(); + }); + + for (const vector of MEDIA_SYNC_DELIVERY_VECTORS) { + it(`prevents post-await and queued commits after ${vector.name}`, async () => { + const sync = new MediaSync({ now: () => 0 }); + const started = Promise.withResolvers(); + const release = Promise.withResolvers(); + const delivered: number[] = []; + const signal = sync.captureSignal(vector.kind); + const retire = () => { + if (vector.retire === "cancel") sync.cancel(vector.kind); + else sync.reset(); + }; + + sync.enqueue( + vector.firstTimestampMs, + vector.kind, + async () => { + started.resolve(); + await release.promise; + return () => delivered.push(1); + }, + undefined, + signal, + false, + ); + sync.enqueue( + vector.queuedTimestampMs, + vector.kind, + () => () => delivered.push(2), + undefined, + signal, + false, + ); + + await started.promise; + retire(); + release.resolve(); + await Promise.resolve(); + await Promise.resolve(); + + expect(signal.aborted).toBe(true); + expect(delivered).toEqual([]); + }); + + it(`accepts timestamp reuse after ${vector.name}`, async () => { + const sync = new MediaSync({ now: () => 0 }); + let delivered = 0; + const retire = () => { + if (vector.retire === "cancel") sync.cancel(vector.kind); + else sync.reset(); + }; + const enqueue = () => + sync.enqueue( + vector.reusedTimestampMs, + vector.kind, + () => () => delivered++, + undefined, + undefined, + false, + ); + enqueue(); + await sync.drain(vector.kind); + retire(); + enqueue(); + await sync.drain(vector.kind); + expect(delivered).toBe(2); + }); + } + + it("does not advance clock state for a canceled speculative delivery", () => { + let now = 1_000; + const sync = new MediaSync({ now: () => now }); + sync.received(900); + + now = 1_010; + sync.enqueue(950, "video", () => () => undefined, undefined, undefined, false); + sync.cancel("video"); + + expect(sync.reference).toBe(100); + expect(sync.timestamp).toBe(900); + }); + + it("reports a synchronous immediate-stage failure", () => { + const sync = new MediaSync(); + let failure: unknown; + sync.deliver( + "video", + () => { + throw new Error("stage failed"); + }, + (error) => { + failure = error; + }, + ); + expect(failure).toBeInstanceOf(Error); + }); +}); diff --git a/js/media-sync/src/index.ts b/js/media-sync/src/index.ts new file mode 100644 index 00000000000..10be9c17895 --- /dev/null +++ b/js/media-sync/src/index.ts @@ -0,0 +1,252 @@ +/** + * Transport-neutral media clock, pacing, and generation-safe delivery queues. + * + * {@link MediaSync} anchors media timestamps to a local clock, paces each delivery until its + * timestamp is due, and serializes deliveries per {@link MediaKind}. Each delivery stages its + * effects and returns a commit; {@link MediaSync.cancel} and {@link MediaSync.reset} retire the + * current generation so a stage that resolves afterwards is discarded instead of committed. + * + * @module + */ + +/** The independently-queued media kinds. */ +export type MediaKind = "video" | "audio"; + +/** A staged delivery: `commit` runs only while its generation is current, `discard` otherwise. */ +export type MediaDelivery = { + commit(): void; + discard?(): void; +}; + +/** + * Stage a delivery. Return a commit function, a {@link MediaDelivery}, or `null` for nothing to commit. + * `signal` aborts when the generation is retired. + */ +export type MediaDeliveryStage = ( + signal: AbortSignal, +) => null | (() => void) | MediaDelivery | Promise void) | MediaDelivery>; + +/** Options for {@link MediaSync}. */ +export interface MediaSyncOptions { + /** Clock source, in clock units (default: `performance.now()` milliseconds). */ + now?: () => number; + /** Playout delay in clock units, shared or per kind (default: zero). */ + buffer?: number | Partial>; + /** Number of media timestamp units in one clock/buffer unit. */ + timestampScale?: number; + /** Called whenever the reference or latest timestamp changes, including on reset. */ + onState?: (reference: number | undefined, timestamp: number | undefined) => void; +} + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +/** Transport-neutral media clock and generation-safe per-track delivery queue. */ +export class MediaSync { + #now: () => number; + #buffers: Record; + #timestampScale: number; + #reference: number | undefined; + #timestamp: number | undefined; + #onState?: MediaSyncOptions["onState"]; + #update = deferred(); + #tails: Record> = { video: Promise.resolve(), audio: Promise.resolve() }; + #independent: Record>> = { video: new Set(), audio: new Set() }; + #controllers: Record = { + video: new AbortController(), + audio: new AbortController(), + }; + + constructor(options: MediaSyncOptions = {}) { + this.#now = options.now ?? (() => performance.now()); + const buffer = options.buffer ?? 0; + this.#buffers = + typeof buffer === "number" + ? { video: buffer, audio: buffer } + : { video: buffer.video ?? 0, audio: buffer.audio ?? 0 }; + this.#timestampScale = options.timestampScale ?? 1; + if (!Number.isFinite(this.#timestampScale) || this.#timestampScale <= 0) { + throw new Error("timestampScale must be finite and greater than zero"); + } + this.#onState = options.onState; + } + + /** The local clock time of media timestamp zero, or undefined until seeded. */ + get reference(): number | undefined { + return this.#reference; + } + + /** The latest media timestamp received, or undefined until seeded. */ + get timestamp(): number | undefined { + return this.#timestamp; + } + + /** The playout delay for `kind`, in clock units. */ + buffer(kind: MediaKind): number { + return this.#buffers[kind]; + } + + /** Set the playout delay for `kind`, or for both kinds, and re-evaluate parked waits. */ + setBuffer(buffer: number, kind?: MediaKind): void { + if (kind) { + if (buffer === this.#buffers[kind]) return; + this.#buffers[kind] = buffer; + } else { + if (buffer === this.#buffers.video && buffer === this.#buffers.audio) return; + this.#buffers.video = buffer; + this.#buffers.audio = buffer; + } + this.#wake(); + } + + /** Seed or advance the clock with a received media timestamp. Call before {@link MediaSync.wait}. */ + received(timestamp: number): void { + if (this.#timestamp === undefined || timestamp > this.#timestamp) this.#timestamp = timestamp; + const reference = this.#now() - timestamp / this.#timestampScale; + if (this.#reference === undefined || reference < this.#reference) this.#reference = reference; + this.#onState?.(this.#reference, this.#timestamp); + this.#wake(); + } + + /** The media timestamp due for presentation now, in timestamp units, or undefined until seeded. */ + now(kind: MediaKind = "video"): number | undefined { + if (this.#reference === undefined) return undefined; + return (this.#now() - this.#reference - this.#buffers[kind]) * this.#timestampScale; + } + + /** Sleep until `timestamp` is due, the clock resets, or `signal` aborts. Throws if the clock was never seeded. */ + async wait(timestamp: number, kind: MediaKind = "video", signal?: AbortSignal): Promise { + if (signal?.aborted) return; + if (this.#reference === undefined) throw new Error("reference not set; call received() first"); + + for (;;) { + const reference = this.#reference; + if (reference === undefined || signal?.aborted) return; + const sleep = reference - (this.#now() - timestamp / this.#timestampScale) + this.#buffers[kind]; + if (sleep < 5) return; + + await new Promise((resolve) => { + let done = false; + const finish = () => { + if (done) return; + done = true; + clearTimeout(timeout); + signal?.removeEventListener("abort", finish); + resolve(); + }; + const timeout = setTimeout(finish, sleep); + signal?.addEventListener("abort", finish, { once: true }); + void this.#update.promise.then(finish); + if (signal?.aborted) finish(); + }); + } + } + + /** The signal of the current `kind` generation; it aborts when that generation is retired. */ + captureSignal(kind: MediaKind): AbortSignal { + return this.#controllers[kind].signal; + } + + /** Queue a staged delivery for `kind`, paced to `timestamp` and serialized behind earlier deliveries unless + * `pace`/`serialize` are false. A delivery captured under a retired `capturedSignal` is dropped. */ + enqueue( + timestamp: number, + kind: MediaKind, + stage: MediaDeliveryStage, + onError?: (error: unknown) => void, + capturedSignal?: AbortSignal, + pace = true, + serialize = true, + ): void { + const signal = capturedSignal ?? this.captureSignal(kind); + if (!this.#accepts(kind, signal)) return; + const run = async () => { + if (!this.#accepts(kind, signal)) return; + if (pace) await this.wait(timestamp, kind, signal); + if (!this.#accepts(kind, signal)) return; + const staged = await stage(signal); + if (!staged) return; + const delivery = typeof staged === "function" ? { commit: staged } : staged; + if (this.#accepts(kind, signal)) delivery.commit(); + else delivery.discard?.(); + }; + const execute = () => + run().catch((error) => { + if (!signal.aborted) onError?.(error); + }); + if (!serialize) { + const independent = this.#independent[kind]; + const pending = execute().finally(() => independent.delete(pending)); + independent.add(pending); + return; + } + const independent = [...this.#independent[kind]]; + this.#tails[kind] = Promise.all([this.#tails[kind].catch(() => undefined), ...independent]).then(execute); + } + + /** Start an unpaced stage immediately while retaining generation ownership. */ + deliver( + kind: MediaKind, + stage: MediaDeliveryStage, + onError?: (error: unknown) => void, + capturedSignal?: AbortSignal, + ): void { + const signal = capturedSignal ?? this.captureSignal(kind); + if (!this.#accepts(kind, signal)) return; + let staged: ReturnType; + try { + staged = stage(signal); + } catch (error) { + onError?.(error); + return; + } + void Promise.resolve(staged) + .then((staged) => { + if (!staged) return; + const delivery = typeof staged === "function" ? { commit: staged } : staged; + if (this.#accepts(kind, signal)) delivery.commit(); + else delivery.discard?.(); + }) + .catch((error) => { + if (!signal.aborted) onError?.(error); + }); + } + + /** Resolve once every delivery queued for `kind` so far has settled. */ + async drain(kind: MediaKind): Promise { + await this.#tails[kind]; + await Promise.all(this.#independent[kind]); + } + + /** Retire the `kind` generation: queued and in-flight deliveries are discarded rather than committed. */ + cancel(kind: MediaKind): void { + this.#controllers[kind].abort(); + this.#controllers[kind] = new AbortController(); + this.#tails[kind] = Promise.resolve(); + this.#independent[kind] = new Set(); + } + + /** Retire both generations, clear the clock, and wake every waiter. */ + reset(): void { + this.cancel("video"); + this.cancel("audio"); + this.#reference = undefined; + this.#timestamp = undefined; + this.#onState?.(undefined, undefined); + this.#wake(); + } + + #accepts(kind: MediaKind, signal: AbortSignal): boolean { + return !signal.aborted && signal === this.#controllers[kind].signal; + } + + #wake(): void { + this.#update.resolve(); + this.#update = deferred(); + } +} diff --git a/js/media-sync/src/vectors.ts b/js/media-sync/src/vectors.ts new file mode 100644 index 00000000000..0e4ec7de6d7 --- /dev/null +++ b/js/media-sync/src/vectors.ts @@ -0,0 +1,87 @@ +/** + * Shared clock, pacing, and retirement vectors for {@link MediaSync} and the wrappers that adapt its units. + * + * @module + */ + +/** Clock anchoring and `now()` per kind, in millisecond and microsecond timestamp units. */ +export const MEDIA_SYNC_CLOCK_VECTORS = [ + { + name: "millisecond clock", + timestampScale: 1, + anchorClock: 1_000, + anchorTimestamp: 900, + readClock: 1_030, + buffer: { video: 20, audio: 40 }, + expected: { reference: 100, timestamp: 900, videoNow: 910, audioNow: 890 }, + }, + { + name: "microsecond media timestamps", + timestampScale: 1_000, + anchorClock: 1_000, + anchorTimestamp: 900_000, + readClock: 1_030, + buffer: { video: 20, audio: 40 }, + expected: { reference: 100, timestamp: 900_000, videoNow: 910_000, audioNow: 890_000 }, + }, +] as const; + +/** A paced delivery is held while early and committed once due. */ +export const MEDIA_SYNC_PACING_VECTORS = [ + { + name: "paced video delivery", + kind: "video", + anchorClockMs: 1_000, + anchorTimestampMs: 900, + deliveryTimestampMs: 1_000, + earlyClockMs: 1_030, + dueClockMs: 1_051, + bufferMs: 20, + }, + { + name: "paced audio delivery", + kind: "audio", + anchorClockMs: 2_000, + anchorTimestampMs: 1_900, + deliveryTimestampMs: 2_000, + earlyClockMs: 2_030, + dueClockMs: 2_051, + bufferMs: 20, + }, +] as const; + +/** Cancel and reset retire in-flight and queued deliveries, and allow timestamp reuse afterwards. */ +export const MEDIA_SYNC_DELIVERY_VECTORS = [ + { + name: "audio cancellation retirement", + kind: "audio", + retire: "cancel", + firstTimestampMs: 7, + queuedTimestampMs: 8, + reusedTimestampMs: 7, + }, + { + name: "audio reset retirement", + kind: "audio", + retire: "reset", + firstTimestampMs: 17, + queuedTimestampMs: 18, + reusedTimestampMs: 17, + }, + { + name: "video cancellation retirement", + kind: "video", + retire: "cancel", + firstTimestampMs: 27, + queuedTimestampMs: 28, + reusedTimestampMs: 27, + }, + { + name: "video reset retirement", + kind: "video", + retire: "reset", + firstTimestampMs: 37, + queuedTimestampMs: 38, + reusedTimestampMs: 37, + }, +] as const; diff --git a/js/media-sync/tsconfig.build.json b/js/media-sync/tsconfig.build.json new file mode 100644 index 00000000000..b0d79fdc2cb --- /dev/null +++ b/js/media-sync/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + // Build-only variant of tsconfig.json that keeps tests out of dist/, so they + // are neither published nor discovered by `bun test` at the package root. + // `tsc --noEmit` still uses tsconfig.json, so tests stay type-checked. + "extends": "./tsconfig.json", + "exclude": ["src/**/*.test.ts", "src/**/*.test.tsx"] +} diff --git a/js/media-sync/tsconfig.json b/js/media-sync/tsconfig.json new file mode 100644 index 00000000000..bb55d7c43fc --- /dev/null +++ b/js/media-sync/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "./src", + "types": ["bun"] + }, + "include": ["src"] +} diff --git a/package.json b/package.json index b38308d70a0..46e1f7f1298 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "doc", "js/net", "js/clock", + "js/media-sync", "js/pattern", "js/auth", "js/flate",