diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 6a96aca4a39..a1314d8f7b1 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -547,6 +547,7 @@ A group with no decodable frames is a walk-now discontinuity: one empty codec pa A consumer MUST NOT submit the marker to a decoder. Empty groups (zero objects) are permitted and mean nothing. After a discontinuity the timeline continues forward. +A publisher that stops producing and may resume on the same track (e.g. an encoder idle for lack of demand) SHOULD publish a discontinuity marker when it stops, so the group before the pause does not reach across the gap and read as live. A group whose timestamps fall below the live edge earlier groups reached is malformed. A delivered sequence hole is a playhead event unless the boundary is contiguous within 1 ms. A consumer re-applies startup delay and skip at a playhead event; it does not reset codec state. @@ -565,8 +566,8 @@ A consumer MUST skip it and MUST NOT submit it to a decoder. It does not mean the track ended. For audio, an empty codec payload retains its terminal-trimming meaning: its timestamp is the exclusive endpoint of the source media. -When a codec must receive additional packets to emit buffered source samples, the marker MUST precede those terminal packets. -A consumer MUST NOT submit the marker to the codec decoder, MUST decode the terminal packets, and MUST discard decoded samples at or after the endpoint. +When a codec must receive additional packets to emit buffered source samples, the marker MUST precede those terminal packets in the same group. +A consumer MUST NOT submit the marker to the codec decoder, MUST decode the terminal packets, and MUST discard decoded samples at or after the endpoint until a later group begins. Audio publishers do not append per-group duration markers because the codec defines each packet's duration. Data tracks retain empty payloads as data, without endpoint semantics. @@ -1072,6 +1073,8 @@ A publisher MAY estimate an unknown final duration from the frame cadence, but M - Required exclusive DVR restart recovery to remove unreferenced group objects left by interrupted expiration. - Replaced the catalog root `timeline` field with `archive`, carrying the timeline track plus optional `replay`, `store`, and recording `version`. - A marker group of one empty frame declares a discontinuity. Empty groups mean nothing. Timestamps only move forward; a group below the live edge is malformed. A delivered sequence hole is a playhead event unless contiguous within 1 ms. +- A publisher that stops producing and may resume on the same track SHOULD publish a discontinuity marker when it stops. +- An audio endpoint bounds only the terminal packets that follow it in its own group. - Replaced the archive timeline `wall` field with a root `clock` section (`wall` plus `timescale`): one fixed broadcast mapping every track and the archive index convert into, independent of any archive. Zero timescales and walls past the JSON-safe integer range are refused. # Acknowledgments diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 5f369b3d4f8..69c8400a968 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -227,6 +227,90 @@ test("Legacy Producer refuses a keyframe that rewinds the timeline", () => { producer.close(); }); +/** Read every group a subscriber is served until `last`, as (sequence, [timestamp, payload size][]). */ +async function readGroups(subscriber: Track.Subscriber, last: number) { + const groups: [number, [number, number][]][] = []; + for (;;) { + const group = await subscriber.recvGroup(); + if (!group) throw new Error("track ended before the last group"); + const frames: [number, number][] = []; + for (;;) { + const frame = await group.readFrame(); + if (!frame) break; + const [timestamp, payload] = Varint.decode(frame.payload); + frames.push([timestamp, payload.byteLength]); + } + groups.push([group.sequence, frames]); + if (group.sequence === last) return groups; + } +} + +test("Legacy Producer cut marks the break with one empty frame at the live edge", async () => { + const track = new Track.Producer("test"); + const subscriber = replay(track); + const producer = new LegacyProducer(track, new LegacyFormat("audio")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.encode(new Uint8Array([1]), 20_000 as Time.Micro, true); + producer.cut(); + producer.cut(); // nothing new to mark + producer.encode(new Uint8Array([1]), 5_000_000 as Time.Micro, true); + producer.close(); + + expect(await readGroups(subscriber, 3)).toEqual([ + [0, [[0, 1]]], + [1, [[20_000, 1]]], + [2, [[20_000, 0]]], + [3, [[5_000_000, 1]]], + ]); +}); + +test("Legacy Producer cut marks the break at the caller's end", async () => { + const track = new Track.Producer("test"); + const subscriber = replay(track); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.cut(33_000 as Time.Micro); + producer.close(); + + expect(await readGroups(subscriber, 1)).toEqual([ + [ + 0, + [ + [0, 1], + [33_000, 0], + ], + ], + [1, [[33_000, 0]]], + ]); +}); + +test("Legacy Producer cut marks nothing on a data track or before any frame", async () => { + const data = new Track.Producer("data"); + const producer = new LegacyProducer(data, new LegacyFormat("data")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.cut(); + expect(data.appendGroup().sequence).toBe(1); + + const empty = new Track.Producer("empty"); + new LegacyProducer(empty, new LegacyFormat("video")).cut(); + expect(empty.appendGroup().sequence).toBe(0); +}); + +// A group's reach runs to its successor's first frame, so without the marker the group before a +// pause would stretch across the whole gap and read as live to anyone joining after the resume. +test("Legacy Producer cut keeps pre-pause media from reading as live", async () => { + const track = new Track.Producer("test"); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + producer.encode(new Uint8Array([1]), 0 as Time.Micro, true); + producer.encode(new Uint8Array([1]), 33_000 as Time.Micro, false); + producer.cut(); + producer.encode(new Uint8Array([1]), 5_000_000 as Time.Micro, true); + producer.close(); + + const subscriber = track.subscribe({ maxAge: Time.Milli(1_000) }); + expect((await readGroups(subscriber, 2)).map(([sequence]) => sequence)).toEqual([1, 2]); +}); + test("LegacyFormat throws on truncated input", () => { const format = new LegacyFormat("data"); // A varint that indicates more bytes follow but is truncated diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index cf8647fddf1..c066dbe8311 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -84,6 +84,8 @@ export class Producer { #liveEdge?: Time.Micro; // Gap between consecutive timestamps, used to close the last group when no successor exists. #interval?: Time.Micro; + // A cut's marker is the newest group, so another one would say nothing new. + #marked = false; /** Wrap a track to publish legacy-container frames into it. */ constructor(track: Moq.Track.Producer, format: Format, props: ProducerProps = {}) { @@ -94,9 +96,10 @@ export class Producer { /** Encode and append a frame; a keyframe starts a new group. Throws if the first frame is not a keyframe, or if the timestamp sits below the live edge earlier groups reached. */ encode(data: Uint8Array | Source, timestamp: Time.Micro, keyframe: boolean) { + this.#marked = false; if (keyframe) { const rewound = this.#previous !== undefined && timestamp < this.#previous; - this.cut(rewound ? undefined : timestamp); + this.#close(rewound ? undefined : timestamp); if (rewound) this.#interval = undefined; this.#refuse(timestamp); this.#group = this.#track.appendGroup(); @@ -122,8 +125,40 @@ export class Producer { if (this.#end === undefined || timestamp > this.#end) this.#end = timestamp; } - /** Flush and close the current group at the supplied or estimated end timestamp. */ + /** + * Close the current group and mark a break in the timeline: whatever comes next does not + * continue it. Call it when the timeline is about to jump, e.g. an encoder pausing for lack of + * demand or switching source; the next keyframe already rolls the group over on its own. + * + * `end` is where the content stops, estimated from the frame cadence when omitted. After closing + * the group, this publishes a marker group of one empty frame at `end`, or at the live edge + * without one. Without the marker, a group's reach runs to its successor's first frame, so the + * group before a pause reads as live until whatever resumes it, and a subscriber joining + * mid-break is handed that stale media. The marker bounds it, and it is the latest group a + * joiner lands on. Data tracks only close the group, since an empty payload is data. No marker + * is written until a frame follows the last one. Throws if `end` precedes the last video frame. + */ cut(end?: Time.Micro) { + this.#close(end); + // Nothing is measured across the break. + this.#interval = undefined; + const timestamp = end ?? this.#liveEdge; + if (this.#format.kind === "data" || this.#marked || timestamp === undefined) return; + + const group = this.#track.appendGroup(); + this.#timeline?.record(group.sequence, timestamp, false); + this.#timeline?.end(timestamp); + group.writeFrame({ + payload: encodeFrame(new Uint8Array(), timestamp), + timestamp: Time.Timestamp.fromMicros(timestamp), + }); + group.close(); + this.#liveEdge = this.#liveEdge === undefined ? timestamp : (Math.max(this.#liveEdge, timestamp) as Time.Micro); + this.#marked = true; + } + + // Flush and close the current group at the supplied or estimated end timestamp. + #close(end?: Time.Micro) { if (!this.#group) return; if ( this.#format.kind === "video" && @@ -161,7 +196,7 @@ export class Producer { /** Close the track and current group, optionally with an error. */ close(err?: Error) { - if (!err) this.cut(); + if (!err) this.#close(); this.#group?.close(err); this.#track.close(err); } diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 21ceed0ec43..9bfabb30f14 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -155,6 +155,10 @@ export class Encoder { // discontinuity and re-anchors on. #pipeline: Pipeline | undefined; + // The exclusive end of the newest frame written to the live track, where a demand gap's + // discontinuity marker goes. Cleared once the marker is written. + #end: Time.Micro | undefined; + // The fatal error an AudioEncoder reported, if any. That instance can never encode again and // reconfiguring it would be a retry, so the rendition stays down for the life of this encoder. #fatal = new Signal(undefined); @@ -258,6 +262,23 @@ export class Encoder { this.#encode(rendition.track, format, effect); }); + // When demand disappears, end the epoch with a discontinuity marker (see + // Container.Legacy.Producer.cut) so a later subscriber resumes on the same track without the + // pre-gap frames reading as live. Its empty payload marks where the source media ends. + effect.run((effect) => { + const track = effect.get(rendition.track); + if (!track) return; + effect.cleanup(() => { + const end = this.#end; + this.#end = undefined; + if (end === undefined || track.closed.peek() !== undefined) return; + track.writeFrame({ + payload: Container.Legacy.encodeFrame(new Uint8Array(), end), + timestamp: Time.Timestamp.fromMicros(end), + }); + }); + }); + effect.run((effect) => { const enabled = effect.get(this.in.enabled); const capture = effect.get(this.in.capture); @@ -384,10 +405,13 @@ export class Encoder { // Each audio frame is its own group so the relay can forward it without // waiting for a group boundary. Loss is handled by the codec's PLC. - track.peek()?.writeFrame({ + const live = track.peek(); + if (!live) return; + live.writeFrame({ payload: Container.Legacy.encodeFrame(frame, frame.timestamp as Time.Micro), timestamp: Time.Timestamp.fromMicros(frame.timestamp as Time.Micro), }); + this.#end = (frame.timestamp + (frame.duration ?? 0)) as Time.Micro; }, error: (err) => { console.error("encoder error", err); diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 0d6c6b9cc7a..74759200d65 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -90,7 +90,7 @@ test("encoding tracks encoder config in its child effect", async () => { } }); -test("a demand gap leaves the broadcast-owned track open for resume", async () => { +test("a demand gap cuts the group and leaves the broadcast-owned track open for resume", async () => { using _videoEncoder = installFakeVideoEncoder(); const cut = spyOn(Container.Legacy.Producer.prototype, "cut"); @@ -119,6 +119,7 @@ test("a demand gap leaves the broadcast-owned track open for resume", async () = live.set(undefined); await settle(); expect(track.closed.peek()).toBeUndefined(); + expect(cut).toHaveBeenCalledTimes(1); live.set(track); await settle(); diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index fe7695f4c11..b22325c6381 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -235,9 +235,10 @@ export class Encoder { this.#lastCaptureWall = performance.now(); const producer = new Container.Legacy.Producer(track, new Container.Legacy.Format("video")); - // The broadcast owns this static track across demand gaps. End only the current - // group when demand disappears so a later subscriber can resume on the same track. - // A fatal encoder error still aborts the track through producer.close(err) below. + // The broadcast owns this static track across demand gaps. When demand disappears, cut the + // current group, marking the break so a later subscriber resumes on the same track without + // the pre-gap group reading as live. A fatal encoder error still aborts the track through + // producer.close(err) below. effect.cleanup(() => { if (track.closed.peek() === undefined) producer.cut(); }); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 54fa26490e4..65bb0162a47 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -616,7 +616,12 @@ export class Decoder { // Apply ordered container metadata before handling the result. An endpoint that also // starts a new epoch must survive the reset so its following drain is trimmed. - #onNext(next: { discontinuity: number; end?: Time.Micro; frame?: { timestamp: Time.Micro } }): boolean { + #onNext(next: { + discontinuity: number; + group: number; + end?: Time.Micro; + frame?: { timestamp: Time.Micro }; + }): boolean { if (!this.#terminal.update(next)) return false; this.#ring?.reset(); this.sync.reset(); diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 1a88c54c10d..6c6d9607cad 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -4,19 +4,19 @@ import { Terminal } from "./terminal"; test("Terminal trims a partial packet to the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, frame: { timestamp: 0 as Time.Micro }, end: 17_916 as Time.Micro }); + terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 17_916 as Time.Micro }); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(860); }); test("Terminal maps Opus delay back before trimming terminal output", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); const body = terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }); expect(body).toEqual({ timestamp: 0 as Time.Micro, frameOffset: 312, frames: 648 }); - terminal.update({ discontinuity: 0, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 0, frame: { timestamp: 20_000 as Time.Micro } }); + terminal.update({ discontinuity: 0, group: 0, end: 20_000 as Time.Micro }); + terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 20_000 as Time.Micro } }); const drain = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(drain).toEqual({ timestamp: 13_500 as Time.Micro, frameOffset: 0, frames: 312 }); expect(body.frames + drain.frames).toBe(960); @@ -24,20 +24,20 @@ test("Terminal maps Opus delay back before trimming terminal output", () => { test("Terminal preserves samples before the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, frame: { timestamp: 0 as Time.Micro }, end: 20_000 as Time.Micro }); + terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 20_000 as Time.Micro }); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); test("a rewound endpoint survives the discontinuity reset and trims its terminal packet", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, end: 40_000 as Time.Micro }); + terminal.update({ discontinuity: 0, group: 0, end: 40_000 as Time.Micro }); - const reset = terminal.update({ discontinuity: 1, end: 20_000 as Time.Micro }); + const reset = terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); expect(reset).toBe(true); expect(terminal.end).toBe(20_000 as Time.Micro); - terminal.update({ discontinuity: 1, frame: { timestamp: 20_000 as Time.Micro } }); + terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 20_000 as Time.Micro } }); const span = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(span).toEqual({ timestamp: 20_000 as Time.Micro, frameOffset: 312, frames: 0 }); }); @@ -45,10 +45,20 @@ test("a rewound endpoint survives the discontinuity reset and trims its terminal test("a discontinuity reapplies Opus pre-skip in the resumed epoch", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frameOffset).toBe(312); - terminal.update({ discontinuity: 1, frame: { timestamp: 1_000_000 as Time.Micro } }); + terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 1_000_000 as Time.Micro } }); const resumed = terminal.span({ timestamp: 1_000_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(resumed).toEqual({ timestamp: 1_000_000 as Time.Micro, frameOffset: 312, frames: 648 }); }); + +test("an endpoint does not trim media resumed in a later group", () => { + const terminal = new Terminal(); + terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + // A discontinuity marker: its own group, ending the epoch at the live edge. + terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); + terminal.update({ discontinuity: 1, group: 2, frame: { timestamp: 5_000_000 as Time.Micro } }); + expect(terminal.end).toBeUndefined(); + expect(terminal.span({ timestamp: 5_000_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); +}); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index f241fa264d1..f89a8b15e74 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -8,6 +8,7 @@ interface SampleSpan { interface Update { readonly discontinuity: number; + readonly group: number; readonly end?: Time.Micro; readonly frame?: { readonly timestamp: Time.Micro }; } @@ -26,6 +27,8 @@ export interface DecodedSpan { export class Terminal { #discontinuity = 0; #end?: Time.Micro; + // The group that carried `#end`. It bounds only the terminal packets that follow it there. + #endGroup?: number; #epoch?: Time.Micro; #preSkip = 0; #preSkipRemaining?: number; @@ -51,8 +54,13 @@ export class Terminal { this.#end = undefined; this.#resetEpoch(); } + // A frame from a later group is new media, e.g. resuming after a discontinuity marker. + if (next.frame && next.group !== this.#endGroup) this.#end = undefined; if (next.frame && this.#epoch === undefined) this.#epoch = next.frame.timestamp; - if (next.end !== undefined) this.#end = next.end; + if (next.end !== undefined) { + this.#end = next.end; + this.#endGroup = next.group; + } return reset; }