diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a1ef1633de5..e1553b18f15 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -171,6 +171,10 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just js catalog-bench + - name: JS arrival window benchmark + if: ${{ !cancelled() }} + run: nix develop --command just js arrivals-bench + # Times Effect create and close beside a bare `new Error()`. No threshold: it only # has to keep running, and its numbers track the creation stack's cost. - name: JS signals effect benchmark diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index ee727d762f2..d58a887abc4 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -148,6 +148,30 @@ Pass a signal from [`@moq/signals`](/lib/js/signals) for any control you want to change later, such as `muted` or `delay`. `Player` is the pipeline inside ``; its parts are exported for custom composition. +To chart when each frame arrived against when it plays, open an `arrivals()` +handle on a decoder. It yields every frame received while it is open, stamped +when the frame came off the transport, and a gap wherever media after an earlier +frame will never arrive (a latency skip, a hole): + +```ts +const arrivals = player.audio.arrivals(); +for await (const entry of arrivals) { + if (entry.kind === "gap") { + // Break the plotted line; nothing arrived from `timestamp` for `duration`. + continue; + } + const { timestamp, arrival, late } = entry; + // `late`: arrived after its playback deadline. +} +``` + +A gap's span is an upper bound: without per-frame durations it starts at the +last frame's timestamp, and it covers any pause the publisher declared. + +Unread frames queue on the handle, so keep reading and keep your own window. +Leaving the loop or calling `arrivals.close()` stops observing and releases +them; nothing is retained without an open handle. Video reports the rendition on screen. + ## Buffered playback By default the player minimizes latency: it skips ahead whenever media piles diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 82255ac22ce..a29e578d10f 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -31,8 +31,10 @@ export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. A container read yields frames plus two markers in delivery order: a gap where -media will never arrive (the join, a skip, a hole) and an end where the -publisher declared its media stops (a pause, a source switch, the finish). +media after what was read will never arrive (a skip, a hole), carrying where it +starts and how long it lasts as an upper bound, and an end where the publisher +declared its media stops (a pause, a source switch, the finish). Joining +mid-track is not a gap. Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 17447495cf3..cdcfe3c5c77 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -34,7 +34,6 @@ for (const count of [1, 16, 256]) { expect(warn).toHaveBeenLastCalledWith(`skipping slow groups: track=tone 0 -> ${count * 2} count=${count}`); expect(consumer.buffered.peek()[0]?.start).toBe(Time.Milli(count)); // The retained group is still delivered, and another catch-up reports separately. - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(await consumer.next()).toMatchObject({ kind: "frame", group: count * 2 }); newest.writeFrame({ payload: Varint.encode(count * 1000), timestamp: Time.Timestamp.now() }); write(count * 2 + 2, (count + 1) * 1000); diff --git a/js/hang/src/container/consumer.startup.test.ts b/js/hang/src/container/consumer.startup.test.ts index 5d8ecfacd8f..4d827ecac30 100644 --- a/js/hang/src/container/consumer.startup.test.ts +++ b/js/hang/src/container/consumer.startup.test.ts @@ -35,10 +35,10 @@ function setup(maxDelay = 100) { return { track, consumer, results, warn }; } -// Each item as "gap", "@<µs>" for a frame, or "end @<µs>". +// Each item as "gap <µs>+<µs>", "@<µs>" for a frame, or "end @<µs>". function items(results: Read[]): string[] { return results.map((read) => { - if (read.kind === "gap") return "gap"; + if (read.kind === "gap") return `gap ${read.timestamp}+${read.duration}`; if (read.kind === "end") return `end ${read.group}@${read.end}`; return `${read.group}@${read.frame.timestamp}`; }); @@ -79,8 +79,8 @@ for (const emptySuccessor of [false, true]) { for (let i = 0; i < 5; i++) write(live, 1_000_000 + i * 33_333); await flush(); + // Nothing was read before the live group, so the stale one leaves no gap. expect(items(results)).toEqual([ - "gap", ...[1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map( (timestamp) => `${sequence}@${timestamp}`, ), @@ -98,14 +98,15 @@ test("Consumer preserves a declared marker before its first media across a hole" write(marker, 0, true); marker.close(); await flush(); - expect(items(results)).toEqual(["gap", "end 16@0"]); + expect(items(results)).toEqual(["end 16@0"]); const live = new Group.Producer(18); track.writeGroup(live); write(live, 50_000); await flush(); - // Group 17 never arrived, so the media after the marker is past a gap. - expect(items(results)).toEqual(["gap", "end 16@0", "gap", "18@50000"]); + // Group 17 never arrived, so the media after the marker is past a gap. It spans the declared + // pause too: the reader can't tell where group 17 would have started. + expect(items(results)).toEqual(["end 16@0", "gap 0+50000", "18@50000"]); expect(frames(results)[0].frame.keyframe).toBe(true); expect(warn).not.toHaveBeenCalled(); }); @@ -117,7 +118,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(first, 0); first.close(); await flush(); - expect(items(results)).toEqual(["gap", "16@0"]); + expect(items(results)).toEqual(["16@0"]); if (reset) { const marker = new Group.Producer(17); @@ -125,7 +126,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(marker, 0, true); marker.close(); await flush(); - expect(items(results)).toEqual(["gap", "16@0", "end 17@0"]); + expect(items(results)).toEqual(["16@0", "end 17@0"]); } const sequence = reset ? 20 : 19; const later = new Group.Producer(sequence); @@ -141,13 +142,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(bridge, 0); bridge.close(); await flush(); - // The bridge closes the hole, so nothing reads as a gap past the first. - expect(items(results)).toEqual([ - "gap", - "16@0", - ...(reset ? ["end 17@0"] : []), - `${sequence - 1}@0`, - `${sequence}@50000`, - ]); + // The bridge closes the hole, so nothing reads as a gap. + expect(items(results)).toEqual(["16@0", ...(reset ? ["end 17@0"] : []), `${sequence - 1}@0`, `${sequence}@50000`]); expect(warn).not.toHaveBeenCalled(); }); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 8d9a9f8f4c1..5d6e1e2ae0e 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -529,9 +529,9 @@ async function drainFrames( return frames; } -/** An item as "gap", a frame's µs, or "end <µs>". */ +/** An item as "gap <µs>+<µs>", a frame's µs, or "end <µs>". */ function describe(read: Read): string { - if (read.kind === "gap") return "gap"; + if (read.kind === "gap") return `gap ${read.timestamp}+${read.duration}`; if (read.kind === "end") return `end ${read.end}`; return `${read.frame.timestamp}`; } @@ -745,7 +745,7 @@ test("Consumer delivers a group that arrives below the cursor", async () => { // --- Group.Producer boundary signals --- -test("Consumer next() yields a gap before the first frame only", async () => { +test("Consumer next() yields no gap before the first frame", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -765,10 +765,28 @@ test("Consumer next() yields a gap before the first frame only", async () => { seen.push(describe(result)); } - expect(seen).toEqual(["gap", "0", "33000", "66000"]); + expect(seen).toEqual(["0", "33000", "66000"]); consumer.close(); }); +test("Consumer reads nothing after close, even the frame a gap was returned ahead of", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + // Group 1 never arrives, so group 2's frame is deferred behind a gap. + writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); + + consumer.close(); + expect(await consumer.next()).toBeUndefined(); + track.close(); +}); + test("Consumer keeps a trailing duration marker internal", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { @@ -790,7 +808,6 @@ test("Consumer keeps a trailing duration marker internal", async () => { track.close(); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); expect(media.keyframe).toBe(true); @@ -820,7 +837,6 @@ test("Consumer reads a leading marker as an end and keeps the first media keyfra track.writeGroup(group); track.close(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(await consumer.next()).toEqual({ kind: "end", group: 0, end: 20_000 as Time.Micro }); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); @@ -838,7 +854,6 @@ test("Consumer reads an empty LOC payload as the frame's duration", async () => const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); expect(media.duration).toBeUndefined(); @@ -856,7 +871,6 @@ test("Consumer passes over the duration markers the LOC producer writes", async const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); const first = frameOf(await consumer.next()); expect(first.payload).toEqual(new Uint8Array([0xde, 0xad])); expect(first.keyframe).toBe(true); @@ -877,7 +891,6 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { producer.close(); const format = kind === "legacy" ? new LegacyFormat("data") : new LocFormat(); const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Time.Milli }); - expect(await consumer.next()).toEqual({ kind: "gap" }); const frame = frameOf(await consumer.next()); expect(frame.payload).toEqual(new Uint8Array()); expect(frame.keyframe).toBe(true); @@ -934,7 +947,6 @@ test("Consumer treats B-frame reordering within a group as continuous", async () writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 66_000 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 66_000, 33_000]) { expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } @@ -957,7 +969,6 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro, 50_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 33_000, 66_000, 50_000]) { expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } @@ -973,7 +984,6 @@ test("Consumer reads no gap when retiring a fully delivered group", async () => writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); @@ -1000,7 +1010,6 @@ test("Consumer reads no gap when retiring a group holding only its endpoint", as group.close(); track.writeGroup(group); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); @@ -1022,7 +1031,6 @@ test("Consumer reads a gap when retiring a truncated group", async () => { group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); track.writeGroup(group); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); group.close(new NetError.Stream(StreamCode.Cancel)); @@ -1030,7 +1038,8 @@ test("Consumer reads a gap when retiring a truncated group", async () => { writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + // Legacy carries no duration, so the gap starts at the last frame read. + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); @@ -1045,14 +1054,14 @@ test("Consumer reads a gap when retiring an open group", async () => { group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); track.writeGroup(group); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + // Legacy carries no duration, so the gap starts at the last frame read. + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); group.close(); @@ -1522,9 +1531,9 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 90_000, 1)); - // The missing group's span was never delivered, so a gap comes first. + // The missing group's span was never delivered, so a gap from A's end to B comes first. const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).toEqual({ kind: "gap" }); + expect(result).toEqual({ kind: "gap", timestamp: 33_333 as Time.Micro, duration: 966_667 as Time.Micro }); expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); @@ -1561,7 +1570,7 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc } const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).toEqual({ kind: "gap" }); + expect(result).toEqual({ kind: "gap", timestamp: 33_333 as Time.Micro, duration: 33_334 as Time.Micro }); expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); @@ -1579,7 +1588,6 @@ test("Consumer yields no gap while nothing is dropped", async () => { writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); // nothing precedes the first frame expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); expect(frameOf(await consumer.next()).timestamp).toBe(33_000 as Time.Micro); @@ -1599,7 +1607,6 @@ test("Consumer reads a marker group as an end, not a gap", async () => { writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); expect(await consumer.next()).toEqual({ kind: "end", group: 1, end: 0 as Time.Micro }); // A declared stop loses nothing, so no gap follows it. @@ -1619,7 +1626,6 @@ test("Consumer empty groups mean nothing", async () => { writeGroupWithLegacyFrames(track, 2, [1_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); expect(frameOf(await consumer.next()).timestamp).toBe(1_000 as Time.Micro); @@ -1637,7 +1643,7 @@ test("Consumer reads a latency skip as one gap", async () => { writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); @@ -1654,7 +1660,11 @@ test("Consumer reads a timestamp hole as a gap", async () => { writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ + kind: "gap", + timestamp: 0 as Time.Micro, + duration: 1_000_000 as Time.Micro, + }); expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); @@ -1678,7 +1688,7 @@ test("Consumer reads a shed marker group as a gap", async () => { await settle(); // The zero budget sheds the marker unread, and what it declared is lost with it. - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 500 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(500 as Time.Micro); group0.close(); @@ -1728,8 +1738,7 @@ test("a live video frame is delivered before its duration marker arrives", async }); await settle(); expect(delivered).toBe(true); - expect(await read).toEqual({ kind: "gap" }); - expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await read).timestamp).toBe(0 as Time.Micro); group.writeFrame({ payload: encodeLegacyFrame(15_000 as Time.Micro, new Uint8Array()), timestamp: Time.Timestamp.now(), @@ -1776,7 +1785,11 @@ for (const code of [StreamCode.Cancel, StreamCode.Internal, StreamCode.Old, Stre }); next.close(); // The reset cut the group's tail. - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ + kind: "gap", + timestamp: 0 as Time.Micro, + duration: 1 as Time.Micro, + }); const resumed = frameOf(await consumer.next()); expect(resumed.payload).toEqual(new Uint8Array([2])); expect(resumed.keyframe).toBe(true); @@ -1851,7 +1864,7 @@ for (const end of [ const resumed = await Promise.race([pending, settle(100).then(() => "stalled" as const)]); expect(resumed).not.toBe("stalled"); if (resumed === "stalled") return; - expect(resumed).toEqual({ kind: "gap" }); + expect(resumed).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 1_000_000 as Time.Micro }); expect(await consumer.next()).toMatchObject({ kind: "frame", group: 2, frame: { timestamp: 1_000_000 } }); if (end) await expect(consumer.next()).rejects.toBe(end); else expect(await consumer.next()).toBeUndefined(); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a4bf7c3c188..65852712752 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -4,7 +4,7 @@ import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq import type { Format } from "./format"; import { Jitter } from "./jitter"; -import type { BufferedRanges, Frame } from "./types"; +import type { BufferedRanges, ReceivedFrame } from "./types"; /** Options for constructing a {@link Consumer}; the obsolete `maxAge` prop is refused. */ export interface ConsumerProps { @@ -24,13 +24,17 @@ export interface ConsumerProps { /** One item of a {@link Consumer.next} read, in delivery order. Mirrors Rust's `container::Read`. */ export type Read = /** A media frame from `group`. Only group equality means anything: numbers may skip. */ - | { kind: "frame"; group: number; frame: Frame } + | { kind: "frame"; group: number; frame: ReceivedFrame } /** - * Media before the next item will never reach this reader: it joined mid-track, or a group was - * skipped, evicted, truncated, or never arrived. A decoder resumes without the references or - * state that media built. + * Media after an item already read will never reach this reader: a group was skipped, evicted, + * truncated, or never arrived. A decoder resumes without the references or state that media + * built. Joining mid-track is not a gap: nothing was read before it. + * + * The span is an upper bound. `timestamp` is the furthest point already read; without a + * per-frame duration that is the last frame's timestamp, one frame early. `duration` runs to + * where the next item resumes, so it also covers any declared pause in between. */ - | { kind: "gap" } + | { kind: "gap"; timestamp: Time.Micro; duration: Time.Micro } /** * The publisher declared that its media stops at `end`: it paused, switched source, or finished. * Frames that follow from the same `group` are the stopped encoder's final packets (an audio @@ -40,7 +44,7 @@ export type Read = interface Group { consumer: Moq.Group.Consumer; - frames: Frame[]; // decode order + frames: ReceivedFrame[]; // decode order empty: boolean; // no wire frame was published; empty groups mean nothing media: boolean; // a decodable (non-marker) frame was buffered start?: Time.Micro; // First decoded timestamp @@ -98,9 +102,11 @@ export class Consumer { // Group of the last media frame next() returned, so an endpoint after it reads as that frame's // duration rather than a declared stop. #mediaGroup?: number; - // Media went missing since the last item read, so the next one is preceded by a gap. Starts set: - // nothing before the first frame was read. - #lost = true; + // Media went missing since the last item read, so the next one is preceded by a gap. + #lost = false; + // The furthest presentation point read so far: max(timestamp + duration), endpoints included. + // Undefined until the first item, so a loss before it is no gap. + #edge?: Time.Micro; // The item a gap was returned ahead of. #next?: Read; // The first media timestamp of the latest delivered group, and its predecessor's start. @@ -213,7 +219,8 @@ export class Consumer { this.#jitter.observe(arrival, Moq.Time.Milli.fromMicro(sample.timestamp)); this.#spread.set(this.#jitter.measured); } - const frame: Frame = { + const frame: ReceivedFrame = { + arrival, payload: sample.payload, timestamp: sample.timestamp, // Protocol invariant: groups always start at a keyframe. @@ -449,7 +456,7 @@ export class Consumer { /** * Returns the next {@link Read} in delivery order, awaiting one if needed: a frame, a gap where - * media will never arrive (always one before the first frame), or an end the publisher declared. + * media after an earlier item will never arrive, or an end the publisher declared. * An unmarked forward timestamp jump is not a gap, since nothing on the wire says media is missing. * After buffered groups drain, a finished track returns undefined and an aborted track throws. */ @@ -517,8 +524,11 @@ export class Consumer { const end = this.#format.end?.(frame); if (end !== undefined) { this.#updateBuffered(); - // A trailing endpoint is the last frame's duration, which only the skip logic needs. - if (seq === this.#mediaGroup) continue; + // A trailing endpoint is the last frame's duration: it moves the edge, nothing else. + if (seq === this.#mediaGroup) { + this.#advance(end); + continue; + } return this.#emit({ kind: "end", group: seq, end }); } this.#mediaGroup = seq; @@ -588,12 +598,25 @@ export class Consumer { } } - // Return `read`, or a gap ahead of it when media went missing first. - #emit(read: Read): Read { - if (!this.#lost) return read; + // Return `read`, or a gap ahead of it when media went missing after an earlier item. + #emit(read: Exclude): Read { + const resume = read.kind === "frame" ? read.frame.timestamp : read.end; + const edge = this.#edge; + const lost = this.#lost; this.#lost = false; + this.#advance( + read.kind === "frame" ? Moq.Time.Micro.add(resume, read.frame.duration ?? Moq.Time.Micro.zero) : resume, + ); + if (!lost || edge === undefined) return read; + + // A next group may start before the edge (an overlapping endpoint), so the span bottoms out at zero. + const timestamp = Moq.Time.Micro.min(edge, resume); this.#next = read; - return { kind: "gap" }; + return { kind: "gap", timestamp, duration: Moq.Time.Micro.sub(resume, timestamp) }; + } + + #advance(end: Time.Micro): void { + if (this.#edge === undefined || end > this.#edge) this.#edge = end; } #updateBuffered(): void { @@ -624,6 +647,8 @@ export class Consumer { /** Stop consuming and release the track and all buffered groups. */ close(): void { + // The item a gap was returned ahead of is buffered media too. + this.#next = undefined; this.#finish(null); this.#signals.close(); } diff --git a/js/hang/src/container/types.ts b/js/hang/src/container/types.ts index 1bab11a26ae..02e6579cb6f 100644 --- a/js/hang/src/container/types.ts +++ b/js/hang/src/container/types.ts @@ -17,6 +17,12 @@ export interface Frame { duration?: Time.Micro; } +/** A media frame carrying its local transport ingress time. */ +export interface ReceivedFrame extends Frame { + /** Monotonic ingress time in milliseconds, before reordering or decoder backpressure. */ + arrival: Time.Milli; +} + /** A contiguous span of buffered media, in milliseconds. */ export interface BufferedRange { /** Start of the range in milliseconds. */ diff --git a/js/justfile b/js/justfile index 051baffb93a..7a710a58316 100644 --- a/js/justfile +++ b/js/justfile @@ -95,3 +95,8 @@ qmux-bench: bun install --frozen-lockfile bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/qmux.ts + +# Sweep open arrival handles against their unread backlog. +arrivals-bench: + bun install --frozen-lockfile + bun watch/bench/arrivals.ts diff --git a/js/watch/bench/arrivals.ts b/js/watch/bench/arrivals.ts new file mode 100644 index 00000000000..33dde39e5cc --- /dev/null +++ b/js/watch/bench/arrivals.ts @@ -0,0 +1,29 @@ +import { Time } from "@moq/net"; +import { Producer } from "../src/arrivals"; + +// Sweep open handles against each handle's unread backlog: a write should cost one push per +// handle, never a slope in the backlog. +const frame = { kind: "frame" as const, timestamp: Time.Milli(0), arrival: Time.Milli(0), late: false }; +const WRITES = 10_000; + +console.log("handles,backlog,ns_per_handle_write"); +for (const count of [1, 4, 16]) { + for (const backlog of [0, 256, 4096]) { + const producer = new Producer(); + const handles = Array.from({ length: count }, () => producer.consume()); + for (let i = 0; i < backlog; i++) producer.write(frame); + + const start = performance.now(); + for (let n = 0; n < WRITES; n++) { + producer.write(frame); + for (const handle of handles) await handle.next(); + } + const ns = ((performance.now() - start) * 1e6) / (WRITES * count); + + for (const handle of handles) { + for (let i = 0; i < backlog; i++) if ((await handle.next()) === undefined) throw new Error("lost arrival"); + handle.close(); + } + console.log(`${count},${backlog},${ns.toFixed(0)}`); + } +} diff --git a/js/watch/src/arrivals.test.ts b/js/watch/src/arrivals.test.ts new file mode 100644 index 00000000000..a5e6518622c --- /dev/null +++ b/js/watch/src/arrivals.test.ts @@ -0,0 +1,70 @@ +import { expect, test } from "bun:test"; +import { Time } from "@moq/net"; +import { Producer } from "./arrivals"; + +const frame = (arrival: number) => ({ + kind: "frame" as const, + timestamp: Time.Milli(arrival), + arrival: Time.Milli(arrival), + late: false, +}); + +test("a handle receives every frame written while it is open", async () => { + const producer = new Producer(); + producer.write(frame(0)); + const handle = producer.consume(); + for (let i = 1; i <= 3; i++) producer.write(frame(i)); + for (let i = 1; i <= 3; i++) expect(await handle.next()).toEqual(frame(i)); + + const waiting = handle.next(); + producer.write(frame(4)); + expect(await waiting).toEqual(frame(4)); + handle.close(); +}); + +test("closing a handle releases its unread frames and stops observing", async () => { + const producer = new Producer(); + const closed = producer.consume(); + const open = producer.consume(); + producer.write(frame(0)); + closed.close(); + producer.write(frame(1)); + expect(await closed.next()).toBeUndefined(); + expect(await open.next()).toEqual(frame(0)); + expect(await open.next()).toEqual(frame(1)); + open.close(); +}); + +test("a closed decoder ends its handles once they drain", async () => { + const producer = new Producer(); + const handle = producer.consume(); + producer.write(frame(0)); + producer.close(); + expect(await handle.next()).toEqual(frame(0)); + expect(await handle.next()).toBeUndefined(); + expect(await producer.consume().next()).toBeUndefined(); +}); + +test("a pending read refuses a concurrent one", async () => { + const producer = new Producer(); + const handle = producer.consume(); + const waiting = handle.next(); + await expect(handle.next()).rejects.toThrow("multiple calls to next not supported"); + // Woken but not yet resumed: the queued frame still belongs to the pending read. + producer.write(frame(0)); + await expect(handle.next()).rejects.toThrow("multiple calls to next not supported"); + expect(await waiting).toEqual(frame(0)); + handle.close(); +}); + +test("leaving a for-await loop closes the handle", async () => { + const producer = new Producer(); + const handle = producer.consume(); + producer.write(frame(0)); + for await (const entry of handle) { + expect(entry).toEqual(frame(0)); + break; + } + producer.write(frame(1)); + expect(await handle.next()).toBeUndefined(); +}); diff --git a/js/watch/src/arrivals.ts b/js/watch/src/arrivals.ts new file mode 100644 index 00000000000..ecb1b75d236 --- /dev/null +++ b/js/watch/src/arrivals.ts @@ -0,0 +1,140 @@ +import type * as Container from "@moq/hang/container"; +import { Time } from "@moq/net"; + +/** One received frame, or a gap where media will never arrive, in delivery order. */ +export type Arrival = + | { + kind: "frame"; + /** Media presentation time in milliseconds. */ + timestamp: Time.Milli; + /** Monotonic transport ingress time in milliseconds. */ + arrival: Time.Milli; + /** Whether ingress missed the playback deadline. */ + late: boolean; + } + /** + * Media after an earlier frame never reached the decoder: a latency skip or a hole. The span is + * an upper bound, as on the container read's gap. + */ + | { + kind: "gap"; + /** The furthest presentation point received before the gap, in milliseconds. */ + timestamp: Time.Milli; + /** How long until media resumes, in milliseconds. */ + duration: Time.Milli; + }; + +/** The arrival entry for a container read's gap. */ +export function gap(read: Extract): Arrival { + return { + kind: "gap", + timestamp: Time.Milli.fromMicro(read.timestamp), + duration: Time.Milli.fromMicro(read.duration), + }; +} + +interface State { + queue: Arrival[]; + // The next unread index. Advancing it instead of shifting keeps a long backlog O(1) per read. + head: number; + closed: boolean; + wake?: () => void; +} + +/** + * Every frame a decoder receives while this handle is open, and every gap between them, in delivery order. + * + * Unread entries queue until {@link next} takes them, so read continuously or close the handle; + * closing releases the queue. Obtain one from a decoder's `arrivals()`. + */ +export class Arrivals implements AsyncIterable { + readonly #state: State; + readonly #open: Set; + #reading = false; + + /** @internal */ + constructor(state: State, open: Set) { + this.#state = state; + this.#open = open; + } + + /** Get the next frame or gap, or `undefined` once the handle or its decoder closes. */ + async next(): Promise { + if (this.#reading) throw new Error("multiple calls to next not supported"); + this.#reading = true; + const state = this.#state; + try { + for (;;) { + const entry = state.queue[state.head]; + if (entry) { + state.head++; + // Drop the read prefix once it outweighs the rest, which amortizes to O(1). + if (state.head * 2 >= state.queue.length) { + state.queue.splice(0, state.head); + state.head = 0; + } + return entry; + } + if (state.closed) return undefined; + await new Promise((resolve) => { + state.wake = resolve; + }); + state.wake = undefined; + } + } finally { + this.#reading = false; + } + } + + /** Iterate frames and gaps until the handle or its decoder closes; leaving the loop closes the handle. */ + async *[Symbol.asyncIterator](): AsyncIterator { + try { + for (;;) { + const entry = await this.next(); + if (entry === undefined) return; + yield entry; + } + } finally { + this.close(); + } + } + + /** Stop observing and release unread entries. */ + close(): void { + this.#open.delete(this.#state); + this.#state.queue.length = 0; + this.#state.head = 0; + this.#state.closed = true; + this.#state.wake?.(); + } +} + +/** Fans each frame and gap out to the open {@link Arrivals} handles. @internal */ +export class Producer { + readonly #open = new Set(); + #closed = false; + + /** Open a handle that receives every later frame. */ + consume(): Arrivals { + const state: State = { queue: [], head: 0, closed: this.#closed }; + if (!this.#closed) this.#open.add(state); + return new Arrivals(state, this.#open); + } + + write(arrival: Arrival): void { + for (const state of this.#open) { + state.queue.push(arrival); + state.wake?.(); + } + } + + /** End every handle once it drains what it already received. */ + close(): void { + this.#closed = true; + for (const state of this.#open) { + state.closed = true; + state.wake?.(); + } + this.#open.clear(); + } +} diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 15c8b468035..c4b42779e7b 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -75,7 +75,12 @@ function frame(): Extract { return { kind: "frame", group: 0, - frame: { timestamp: Time.Micro(frameTimestamp), payload: new Uint8Array([1]), keyframe: true }, + frame: { + arrival: Time.Milli.now(), + timestamp: Time.Micro(frameTimestamp), + payload: new Uint8Array([1]), + keyframe: true, + }, }; } @@ -246,6 +251,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { enabled, decoding, context: decoder.out.context, + arrivals: () => decoder.arrivals(), error: decoder.out.error, play, // Remove the rendition from the catalog, then restore it. @@ -294,12 +300,22 @@ it("decodes an audio packet inside a group after a gap", async () => { const playback = await play(Time.Milli(100)); try { await playback.play(); - playback.push({ kind: "gap" }); + // Opened late, so it observes only the gap and frame below. + const arrivals = playback.arrivals(); + playback.push({ kind: "gap", timestamp: Time.Micro(20_000), duration: Time.Micro(40_000) }); const next = frame(); next.frame.keyframe = false; playback.push(next); await microtasks(); expect(playback.inserted()).toContain(next.frame.timestamp); + expect(await arrivals.next()).toEqual({ kind: "gap", timestamp: Time.Milli(20), duration: Time.Milli(40) }); + expect(await arrivals.next()).toMatchObject({ + kind: "frame", + timestamp: Time.Milli.fromMicro(next.frame.timestamp), + arrival: next.frame.arrival, + }); + arrivals.close(); + expect(await arrivals.next()).toBeUndefined(); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 39a0451c090..fee44b1e310 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -13,6 +13,7 @@ import { readonlys, Signal, } from "@moq/signals"; +import { Producer as ArrivalProducer, type Arrivals, gap } from "../arrivals"; import { hostedAssets } from "../assets"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; @@ -137,6 +138,9 @@ export class Decoder { // The broadcast instance the ring's timeline belongs to. See #runDecoder. #instance?: Moq.Broadcast.Consumer["closed"]; + // Fans received frames out to open `arrivals()` handles; retains nothing without one. + #arrivals = new ArrivalProducer(); + #signals = new Effect(); #failed?: Failure; @@ -475,12 +479,14 @@ export class Decoder { decoder.reset(); decoder.configure(decoderConfig); } + if (next.kind === "gap") this.#arrivals.write(gap(next)); if (next.kind !== "frame") continue; const { frame } = next; - // Mark that we received this frame right now. + // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); - this.sync.received(timestamp, "audio"); + const late = this.sync.received(timestamp, "audio", frame.arrival); + this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); const duration = packetDuration(config.codec, frame); if (duration !== undefined) this.#frame.set(duration); @@ -581,11 +587,13 @@ export class Decoder { decoder.reset(); decoder.configure(decoderConfig); } + if (next.kind === "gap") this.#arrivals.write(gap(next)); if (next.kind !== "frame") continue; const { frame } = next; const timestamp = Time.Milli.fromMicro(frame.timestamp); - this.sync.received(timestamp, "audio"); + const late = this.sync.received(timestamp, "audio", frame.arrival); + this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); const duration = packetDuration(config.codec, frame); if (duration !== undefined) this.#frame.set(duration); @@ -736,7 +744,13 @@ export class Decoder { return true; } + /** Observe every frame and gap received from now until the handle or this decoder closes. */ + arrivals(): Arrivals { + return this.#arrivals.consume(); + } + close() { + this.#arrivals.close(); this.#signals.close(); } diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index e40ac71be16..1f5e70d2bed 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; import type * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; -import { type Arrival, type Options, QUANTUM, replay } from "./replay"; +import { type Arrival, type Options, QUANTUM, type Quantum, replay } from "./replay"; const RATE = 48000; const CONFIG = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; @@ -134,3 +134,33 @@ describe.each(["shared", "post"] as const)("%s ring at auto", (ring) => { expect(await settled(60, (i) => 30 + (4 - (i % 5)) * 20)).toBeGreaterThanOrEqual(100); }); }); + +describe.each(["shared", "post"] as const)("%s ingress diagnostics", (ring) => { + it("keeps recorded arrival and lateness when delivery is held", async () => { + const trace = [0, 20, 40].map((timestamp) => ({ timestamp, at: timestamp + 1000, group: 0 })); + const run = async (decodeAfter?: number) => { + let window: Quantum["arrivals"] = []; + for await (const quantum of replay(trace, { + ring, + rate: RATE, + config: CONFIG, + delay: Time.Milli(100), + duration: 300, + decodeAfter, + })) + window = quantum.arrivals; + return window; + }; + const direct = await run(); + const held = await run(1200); + expect(held).toEqual(direct); + expect(held).toEqual( + trace.map((entry) => ({ + kind: "frame" as const, + timestamp: Time.Milli(entry.timestamp), + arrival: Time.Milli(entry.at), + late: false, + })), + ); + }); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index e7ae9424f08..fff6d602d8a 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -19,6 +19,7 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { Group, Time, Track } from "@moq/net"; import { Effect } from "@moq/signals"; +import { type Arrival as Diagnostic, gap } from "../arrivals"; import { type Delay, Sync } from "../sync"; import { frameDuration } from "./config"; import { ringSamples, target } from "./latency"; @@ -50,6 +51,8 @@ export interface Arrival { /** What a replay plays through. */ export interface Options { + /** Hold decoder delivery until this simulated monotonic time. */ + decodeAfter?: number; /** `shared` is the SharedArrayBuffer ring a cross-origin isolated page runs; `post` is the rest. */ ring: "shared" | "post"; /** The sample rate, in Hz. */ @@ -67,6 +70,8 @@ export interface Options { /** One render quantum, as the worklet produced it. */ export interface Quantum { + /** Every frame and gap the decoder received so far, in delivery order. Grows between yields. */ + arrivals: readonly Diagnostic[]; /** When it finished rendering, on the trace's clock, in ms. */ at: number; /** The quantum: what the ring supplied at the front, zeros after it. Reused between yields. */ @@ -215,15 +220,23 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. const terminal = new Terminal(); + const arrivals: Diagnostic[] = []; + const delivery = Promise.withResolvers(); + if (options.decodeAfter === undefined) delivery.resolve(); let failure: { error: unknown } | undefined; const decode = (async () => { + await delivery.promise; for (;;) { const next = await consumer.next(); if (!next) return; if (terminal.update(next)) ring.reset(); + if (next.kind === "gap") arrivals.push(gap(next)); if (next.kind !== "frame") continue; - const { timestamp } = next.frame; + const { timestamp, arrival } = next.frame; + const media = Time.Milli.fromMicro(timestamp); + const late = sync.received(media, "audio", arrival); + arrivals.push({ kind: "frame", timestamp: media, arrival, late }); const length = samples.get(timestamp); if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); ring.insert(timestamp, [new Float32Array(length).fill(0.5)]); @@ -275,6 +288,11 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato next++; } if (next > stamped) await settle(); + clock = now; + if (options.decodeAfter !== undefined && now >= options.decodeAfter) { + delivery.resolve(); + await settle(); + } if (failure) throw failure.error; // The decoder resizes the ring as the resolved delay moves, which only an arrival does. @@ -286,9 +304,17 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato output.fill(0); ring.read([output]); - yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp), delay }; + yield { + arrivals, + at: now, + output, + stalled: ring.stalled, + timestamp: Time.Milli.fromMicro(ring.timestamp), + delay, + }; } } finally { + delivery.resolve(); consumer.close(); track.close(); await decode; diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 44eb0d06c33..f766ef247e0 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -39,7 +39,7 @@ test("Terminal preserves samples before the source endpoint", () => { expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); -test("the gap before the first frame ends no epoch", () => { +test("a gap before any frame ends no epoch", () => { const terminal = new Terminal(); expect(terminal.update(gap)).toBe(false); }); diff --git a/js/watch/src/index.ts b/js/watch/src/index.ts index 3b2eb13a433..b8a6a41c6b8 100644 --- a/js/watch/src/index.ts +++ b/js/watch/src/index.ts @@ -11,6 +11,7 @@ export * as Hang from "@moq/hang"; export type { BufferedRange, BufferedRanges } from "@moq/hang/container"; export * as Net from "@moq/net"; export * as Signals from "@moq/signals"; +export type { Arrival, Arrivals } from "./arrivals"; export { assets } from "./assets"; export * as Audio from "./audio"; export * from "./broadcast"; diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 457b2ec975d..d8b498827d9 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -100,7 +100,7 @@ test("a long subscription through the player path leaves nothing behind", async const play = async (count: number) => { for (let i = 0; i < count; i++) { producer.encode(new Uint8Array([i & 0xff]), Math.round(Time.Micro.now()) as Time.Micro, true); - // Pass over the gap before the first frame. + // Pass over any gap or end. let next = await nextMedia(consumer); while (next && next.kind !== "frame") next = await nextMedia(consumer); if (!next) throw new Error("the track ended"); diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 2236ee1a91c..45ee1f232e3 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -199,3 +199,21 @@ describe("wait", () => { await waiting; }); }); + +describe("recorded ingress", () => { + it("anchors and classifies using arrival rather than delayed delivery", async () => { + const sync = new Sync({ delay: Time.Milli(100) }); + const original = performance.now; + performance.now = () => 10000; + try { + for (let i = 0; i < 20; i++) await Promise.resolve(); + expect(sync.received(Time.Milli(0), "audio", Time.Milli(1000))).toBe(false); + expect(sync.out.reference.peek()).toBe(Time.Milli(1000)); + expect(sync.received(Time.Milli(20), "audio", Time.Milli(1020))).toBe(false); + expect(sync.received(Time.Milli(40), "audio", Time.Milli(1200))).toBe(true); + } finally { + performance.now = original; + sync.close(); + } + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 17f3cafbfe9..8f8d5978bff 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -149,18 +149,18 @@ export class Sync { this.#out.delay.set(delay); } + /** Anchor playback from ingress time and report whether the frame arrived late. */ // Fold a newly received frame into the reference. The reference anchors playback to the // wall clock; we lower it (skip ahead) only when keeping it would push the lookahead past `maxDelay`. - received(timestamp: Time.Milli, label = ""): void { + received(timestamp: Time.Milli, label = "", arrival = Time.Milli.now()): boolean { this.#out.timestamp.update((current) => (current === undefined || timestamp > current ? timestamp : current)); - const now = Time.Milli.now(); - const ref = Time.Milli.sub(now, timestamp); + const ref = Time.Milli.sub(arrival, timestamp); const currentRef = this.#out.reference.peek(); // First frame anchors the reference. if (currentRef === undefined) { this.#out.reference.set(ref); - return; + return false; } // Check if `wait()` would not sleep at all. @@ -187,15 +187,16 @@ export class Sync { } // Frame isn't earlier than the anchor: it can't add lookahead, so keep the reference. - if (ref >= currentRef) return; + if (ref >= currentRef) return sleep < 0; // Frame is earlier (more lookahead). `sleep` is how far ahead of the playhead keeping the // anchor would put it. const cap = this.#out.maxDelay.peek(); - if (sleep <= cap) return; // within budget: let the buffer grow instead of skipping ahead + if (sleep <= cap) return sleep < 0; // within budget: let the buffer grow instead of skipping ahead // Over the cap: re-anchor down so the resulting lookahead is exactly the cap. this.#out.reference.set(Time.Milli.add(ref, Time.Milli.sub(cap, delay))); + return sleep < 0; } // Re-anchor playback to the next frame received. Call this at an utterance boundary diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index e2acb831415..194901f5c47 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -178,10 +178,10 @@ function sample(group: number, timestamp: number, keyframe = true): Read { return { kind: "frame", group, - frame: { timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, + frame: { arrival: Time.Milli.now(), timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, }; } -const gap: Read = { kind: "gap" }; +const gap: Read = { kind: "gap", timestamp: Time.Micro(0), duration: Time.Micro(0) }; async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = []) { const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); @@ -274,9 +274,12 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea const sync = new Sync({ delay }); const enabled = new Signal(true); const decoder = new Decoder({ source, sync, enabled }); + // Opened before the reads below, so it observes every one of them. + const arrivals = decoder.arrivals(); await microtasks(); return { decoder, + arrivals, delay, fail: (error: DOMException, index = -1) => errors.at(index)?.(error), emit: (timestamp: number, index = -1) => frames.at(index)?.(new Picture(timestamp) as unknown as VideoFrame), @@ -437,6 +440,11 @@ for (const kind of ["legacy", "cmaf"] as const) { ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200]); + for (const timestamp of [1, 1.1, 1.2]) + expect(await playback.arrivals.next()).toMatchObject({ + kind: "frame", + timestamp: Time.Milli(timestamp), + }); // Rejected payloads were still received, but never counted as frames. expect(playback.decoder.out.stats.peek()).toEqual({ frameCount: 3, bytesReceived: 5 }); } finally { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 598a8a0509e..cc99c163c9f 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -14,6 +14,7 @@ import { Signal, } from "@moq/signals"; import { rotateVideoDimensions } from "@moq/video"; +import { Producer as ArrivalProducer, type Arrivals, gap } from "../arrivals"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; @@ -106,6 +107,9 @@ export class Decoder { #instance?: { path: Moq.Path.Valid; closed: unknown }; readonly #identity: Computed; + // Fans received frames out to open `arrivals()` handles; retains nothing without one. + #arrivals = new ArrivalProducer(); + #signals = new Effect(); #selected?: DecoderTrack; #failed?: Failure; @@ -270,6 +274,8 @@ export class Decoder { // Upgrade the pending track to active. // #runActive will be in charge of it now. + if (current) current.arrivals = undefined; + pending.arrivals = this.#arrivals; this.#active.set(pending); this.#pending.set(undefined); pending = undefined; @@ -366,8 +372,14 @@ export class Decoder { }, BUFFERING); } + /** Observe every frame and gap the displayed rendition receives until the handle or this decoder closes. */ + arrivals(): Arrivals { + return this.#arrivals.consume(); + } + close() { this.#clearCurrentFrame(); + this.#arrivals.close(); this.#signals.close(); } @@ -401,6 +413,8 @@ class DecoderTrack { identity: string; catalog: Getter; stats: Signal; + // Set once promoted, so only the rendition feeding the picture reports arrivals. + arrivals?: ArrivalProducer; timestamp = new Signal(undefined); frame = new Signal(undefined); @@ -548,6 +562,7 @@ class DecoderTrack { // A gap or a declared end breaks the timeline: re-anchor before decoding further, and // never bridge the buffer across it. if (next.kind !== "frame") { + if (next.kind === "gap") this.arrivals?.write(gap(next)); if (latest !== undefined) this.#onBreak(); previous = undefined; continue; @@ -564,9 +579,10 @@ class DecoderTrack { })); if (stale) continue; - // Mark that we received this frame right now. + // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); - this.sync.received(timestamp, "video"); + const late = this.sync.received(timestamp, "video", frame.arrival); + this.arrivals?.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); const chunk = new EncodedVideoChunk({ type: frame.keyframe ? "key" : "delta", @@ -630,6 +646,7 @@ class DecoderTrack { // A gap or a declared end breaks the timeline (see #runLegacy). if (next.kind !== "frame") { + if (next.kind === "gap") this.arrivals?.write(gap(next)); if (latest !== undefined) this.#onBreak(); previous = undefined; continue; @@ -646,9 +663,10 @@ class DecoderTrack { })); if (stale) continue; - // Mark that we received this frame right now. + // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp); - this.sync.received(timestamp, "video"); + const late = this.sync.received(timestamp, "video", frame.arrival); + this.arrivals?.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); // Track decode buffer (see #runLegacy: bridge only when no gap or end came between, // never on group adjacency, which proves nothing about the timeline). diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 1f8888b360a..4f334674b83 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -14,8 +14,9 @@ frames decode independently and set nothing; HE-AAC is out of scope. - `rs/moq-audio/src/encode/producer.rs` and `rs/moq-mux/src/codec/opus` publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; - the warmup trim is the same mechanism keyed on the container read's gap - item (`container::Read::Gap` in Rust, `{ kind: "gap" }` in JS), never on a + the warmup trim is the same mechanism keyed on the first frame read and the + container read's gap item (`container::Read::Gap` in Rust, `{ kind: "gap" }` + in JS; a join yields no gap), never on a declared end, which would cut 80 ms at every pause, and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. diff --git a/quest/m1/export-source-read.md b/quest/m1/export-source-read.md index 5cdea5890f2..0195d0ad9b4 100644 --- a/quest/m1/export-source-read.md +++ b/quest/m1/export-source-read.md @@ -14,7 +14,9 @@ counters, and those counters are deleted. the raw H.264/H.265 exports) still learn of a gap after the fact. - Map `Gap` to the exporter's discontinuity handling and `End` to its - declared-end handling, in delivery order. + declared-end handling, in delivery order. A join reads no `Gap` (#5311), + though `discontinuity()` still counts losses before the first item; a fresh + exporter has nothing to cut there. - Keep the events `Read` filters reaching the exporters. fMP4 cuts a fragment on `Event::GroupEnd` while the track stays live, and sizes a durationless final sample from a trailing `Event::FrameEnd { opens: false }`; losing diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index fa317d9f3c6..7c2f8526cc5 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -38,16 +38,16 @@ decode, and the JS test should drive the software decoder. - In the JS consumer (`js/hang/src/container/consumer.ts` forces the first sample of a group to `keyframe`, and `js/watch/src/video/decoder.ts` submits - it as `"key"`): for the first group after a gap item, skip delta frames - stamped before that group's keyframe. `next()` yields `{ kind: "gap" }` - before the first frame of a subscribe and after a latency skip, eviction, or - hole; a declared end is not a gap, since a fresh encoder has no earlier GOP + it as `"key"`): for the first group read and the first group after a gap + item, skip delta frames stamped before that group's keyframe. `next()` + yields `{ kind: "gap" }` after a latency skip, eviction, or hole, but not + before the first frame of a subscribe; a declared end is not a gap, since a fresh encoder has no earlier GOP to reference. A latency skip does not flush the decoder, and a viewer that skipped into a later GOP lacks its references just like a cold join. Every group without a gap before it is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - container read's `Read::Gap` item. + first frame read and the container read's `Read::Gap` item. - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present @@ -71,7 +71,8 @@ decode, and the JS test should drive the software decoder. the initial group fails. Both cases in both languages. Decided 2026-10-12: both container reads yield the gap item, so nothing -here waits on another quest. +here waits on another quest. Decided 2026-10-13 (#5311): a join is not a gap, +so the first frame read is its own trigger. ## Related diff --git a/quest/m2/README.md b/quest/m2/README.md index 7728406b578..4e8235bf70a 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -26,7 +26,6 @@ ranked just below deterministic TS legs, which it requires. - [Time test sweep](/quest/m2/time-test-sweep.md) - unrelated test conversion and the workspace lint ratchet - [Binding audio delay](/quest/m2/binding-surface.md) - moq-ffi and every wrapper configure and observe audio playout delay -- [Per-frame arrivals in @moq/watch](/quest/m2/watch-arrivals.md) - the audio and video decoders expose a window of recent frame arrivals, late and skipped marks included, as a signal - [Drill mutation gate](/quest/m2/drill-mutation-gate.md) - a PR touching a drill mutation's target runs that mutation's sensitivity check, advisory until the drills run on controlled time - [Pipelined requests](/quest/m2/pipeline-requests/README.md) - SUBSCRIBE and the first FETCH go out with the track-info request at every hop, so first data arrives a round trip sooner per hop - [OBS native codecs](/quest/m2/obs-moq-video/README.md) - replace FFmpeg video and audio decoding with moq-video and moq-audio, deliver GPU frames, and use native audio/video encoders diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 0ad9d0e0ffa..22fe2f67e3d 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -19,12 +19,12 @@ This absorbs the old gradual-recovery quest, which planned to carry `recovery_frame_cnt` on the wire and count `frame_num`. The catalog duration replaces both: the rule is timestamp arithmetic on the group start. -- Withhold rule, keyed on the gap item both container reads yield before the - first frame and after any skip: `{ kind: "gap" }` from +- Withhold rule, keyed on the first frame a reader reads and on the gap item + both container reads yield after any skip (a join yields no gap): `{ kind: "gap" }` from `js/hang/src/container/consumer.ts` `next()` and `Read::Gap` from the Rust `moq_mux::container::Consumer`, which `rs/moq-video/src/decode/consumer.rs` reads directly. A declared end is not a gap and keeps everything. For the - first group after a gap, every frame is decoded (the decoder needs them to + first group read and the first group after a gap, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery picture itself is never withheld. `js/watch/src/video/decoder.ts` and diff --git a/quest/m2/watch-arrivals.md b/quest/m2/watch-arrivals.md deleted file mode 100644 index fee768cb8e0..00000000000 --- a/quest/m2/watch-arrivals.md +++ /dev/null @@ -1,44 +0,0 @@ -# [S] Per-frame arrivals in @moq/watch - -## Goal - -The audio and video decoders in `@moq/watch` each expose an `arrivals` -signal: a bounded window of recent frames as `{timestamp, arrival, late, -skipped}`, where `arrival` is when the frame came off the transport, `late` -means it arrived after its sync deadline (not that it rendered late), and -`skipped` marks content the consumer dropped to stay within the latency -budget. A page can draw per-frame arrival against playout without patching -`Sync.received`. - -## Plan - -Decided 2026-10-09: - -- **A signal, not a callback.** A signal holding only the newest arrival - would coalesce and lose frames between reads, so it holds a window that - updates per frame. It must reach back at least as far as the span the - first consumer draws (a transcript word); time or count bound is chosen - when implementing. -- **Feed it from where the data already exists.** `Sync.received()` - computes lateness per frame and only logs it today; the container consumer - knows which groups it skipped and only warns. Neither has a signal. -- **`arrival` is the ingress time.** `Container.Consumer` stamps each frame - as `readFrame()` returns, but the `Frame` it returns drops that stamp, and - `Sync.received()` samples the clock later, after reordering and any - decoder backpressure. Carry the ingress stamp through to the decoder and - compute lateness from it. -- **Skipped content never reaches the decoder.** A latency-skipped group's - frames are not decoded, so they cannot appear as ordinary entries. Decide - how the window represents the gap (a mark on the first frame after it, or - a separate entry), keeping it distinct from other `continuous: false` - causes such as markers. -- The first consumer is moq.dev's Voice AI demo, which draws each word's - frames and marks the ones a WebRTC jitter buffer would have dropped - ([moq.dev's condition quest](https://github.com/moq-dev/moq.dev/blob/main/quest/m1/watch-arrivals-release.md)). -- Test with recorded arrival traces through `js/watch/src/audio/replay.ts`, - including a case where delivery is delayed but the recorded arrival is - unchanged. - -## Related - -- [Browser stats](/quest/m2/stats/js.md) - its late-frame and skip counters share these sources, but count at the producer since a reader of an evicting window can miss entries diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index 2e1226bdbf9..8b74ba453d8 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -123,7 +123,9 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: bytes = frame.payload.len(), "received frame" ), - moq_mux::container::Read::Gap => tracing::info!("media skipped"), + moq_mux::container::Read::Gap { timestamp, duration } => { + tracing::info!(?timestamp, ?duration, "media skipped") + } moq_mux::container::Read::End { end, .. } => tracing::info!(?end, "media stopped"), } } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 0e34a54c344..ee247471059 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -278,7 +278,7 @@ impl Consumer { } frame } - Some(Read::Gap) => { + Some(Read::Gap { .. }) => { self.restart()?; continue; } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index e17afe2f62d..a2717454037 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -542,7 +542,7 @@ impl Consume { read = track.read() => match read? { Some(moq_mux::container::Read::Frame { frame, .. }) => frame, // The C ABI carries frames only. - Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }) => continue, None => return Ok(()), }, }; diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 35a0cc608e5..6bc0487bcba 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -98,7 +98,7 @@ impl Video { false } // Applied in delivery order, so the restart lands before the frame after it. - Some(Read::Gap) => { + Some(Read::Gap { .. }) => { end = None; buffer.clear(); self.presentation.lock().unwrap().video_restarted(); diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 6782c05fc62..de6ca3af1c4 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1115,7 +1115,7 @@ impl Media { match self.inner.read().await? { Some(moq_mux::container::Read::Frame { frame, .. }) => return media_frame(frame).map(Some), // The bindings carry frames only. - Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }) => continue, None => return Ok(None), } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 4ae77a68b82..7f01e145c21 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1099,7 +1099,7 @@ impl Pump { // This rendition is being torn down (its run ended, or a catalog update replaced it). _ = cancel.changed() => return None, frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { // PTS restarts at zero with every pump, so the segment places that zero at the // current running time, or a synced sink would drop a restarted run as late. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index a83066c24af..b10bd4aba86 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -56,9 +56,10 @@ pub(crate) enum Event { /// /// ## Gaps and ends /// -/// [`read`](Self::read) yields [`Read::Gap`] before the first frame and wherever media +/// [`read`](Self::read) yields [`Read::Gap`] wherever media after an item already read /// will never arrive: a latency skip, an evicted current group, or a sequence hole whose -/// boundary is not contiguous within 1 ms. A marker group (no decodable frames, one empty +/// boundary is not contiguous within 1 ms. Whatever is lost before the first item is no +/// gap, since nothing was read before it. A marker group (no decodable frames, one empty /// payload) is the publisher declaring a stop, read as [`Read::End`]; nothing is missing. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. @@ -80,9 +81,9 @@ pub struct Consumer { // How far we may drift from the live edge before skipping a group. max_delay: std::time::Duration, - // The live edge of playback: the largest timestamp delivered so far and the group that - // carried it. `None` until the first frame is delivered. - live_edge: Option<(u64, Timestamp)>, + // The furthest presentation point delivered so far: max(timestamp + duration), endpoints + // included. `None` until the first item, so a loss before it is no gap. + edge: Option, // The first frame delivered from the latest group, and that group. Group starts never go // backwards, so a later group with a frame below this is malformed. @@ -104,7 +105,6 @@ pub struct Consumer { markers: u64, // Media went missing since the last item read, so the next one is preceded by a gap. - // Starts set: nothing before the first frame was read. gap: bool, // The item a gap was returned ahead of. @@ -124,6 +124,22 @@ fn pts_contiguous(end: Option, next_start: std::time::Durat end.is_some_and(|end| next_start <= end.saturating_add(CONTIGUITY_TOLERANCE)) } +/// A gap from the delivered `edge` to `resume`, in the resume's scale. +fn gap(edge: std::time::Duration, resume: Timestamp) -> Read { + let scale = resume.scale(); + // A next group may start before the edge (an overlapping endpoint), so the span bottoms out at zero. + let start = edge + .as_nanos() + .saturating_mul(scale.as_u64() as u128) + .div_euclid(1_000_000_000) + .min(resume.value() as u128) as u64; + let timestamp = Timestamp::new(start, scale).expect("at or below a valid timestamp"); + let duration = resume + .checked_sub(timestamp) + .expect("at or below the resume, in its scale"); + Read::Gap { timestamp, duration } +} + impl Consumer { /// Create a Consumer wrapping the given moq-lite consumer, decoding `format`. /// @@ -150,13 +166,13 @@ impl Consumer { pending: VecDeque::new(), startup: start.is_none(), max_delay, - live_edge: None, + edge: None, start: None, floor: None, presented_end: None, discontinuity: 0, markers: 0, - gap: true, + gap: false, next: None, group: 0, } @@ -194,7 +210,7 @@ impl Consumer { loop { match self.read().await? { Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), - Some(Read::Gap | Read::End { .. }) => continue, + Some(Read::Gap { .. } | Read::End { .. }) => continue, None => return Ok(None), } } @@ -209,6 +225,8 @@ impl Consumer { return Poll::Ready(Ok(Some(read))); } loop { + // Where delivered media reached before this item, which moves the edge. + let edge = self.edge; let read = match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => Read::Frame { group: self.group, @@ -219,9 +237,16 @@ impl Consumer { Some(Event::FrameEnd { opens: false, .. } | Event::GroupEnd) => continue, None => return Poll::Ready(Ok(None)), }; - if std::mem::take(&mut self.gap) { + let resume = match &read { + Read::Frame { frame, .. } => frame.timestamp, + Read::End { end, .. } => *end, + Read::Gap { .. } => unreachable!("events never read as a gap"), + }; + if std::mem::take(&mut self.gap) + && let Some(edge) = edge + { self.next = Some(read); - return Poll::Ready(Ok(Some(Read::Gap))); + return Poll::Ready(Ok(Some(gap(edge, resume)))); } return Poll::Ready(Ok(Some(read))); } @@ -282,21 +307,15 @@ impl Consumer { if self.start.is_none_or(|(start, _)| start != seq) { self.start = Some((seq, ts)); } - if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { - self.live_edge = Some((seq, ts)); - } + let end = std::time::Duration::from(ts) + + frame.duration.map(std::time::Duration::from).unwrap_or_default(); + self.advance(end); self.group = seq; return Poll::Ready(Ok(Some(Event::Frame(frame)))); } Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))) => { - let seq = group.group.sequence; - if self - .live_edge - .is_none_or(|(_, high)| end.as_micros() > high.as_micros()) - { - self.live_edge = Some((seq, end)); - } - self.group = seq; + self.group = group.group.sequence; + self.advance(end.into()); return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))); } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), @@ -489,6 +508,12 @@ impl Consumer { self.gap = true; } + fn advance(&mut self, end: std::time::Duration) { + if self.edge.is_none_or(|edge| end > edge) { + self.edge = Some(end); + } + } + fn note_group_edge(&mut self) { if let Some((_, ts)) = self.start { self.floor = Some(ts); @@ -951,13 +976,14 @@ mod tests { group.finish().unwrap(); } - /// Every item until the track ends: a frame as its timestamp in µs, a gap, or an end. + /// Every item until the track ends: a frame as its timestamp in µs, a gap as its start and + /// span in µs, or an end. async fn items(consumer: &mut Consumer) -> Vec { let mut items = Vec::new(); while let Some(read) = consumer.read().await.unwrap() { items.push(match read { Read::Frame { frame, .. } => frame.timestamp.as_micros().to_string(), - Read::Gap => "gap".to_string(), + Read::Gap { timestamp, duration } => format!("gap {}+{}", timestamp.as_micros(), duration.as_micros()), Read::End { end, .. } => format!("end {}", end.as_micros()), }); } @@ -980,12 +1006,31 @@ mod tests { .expect("the end is read while the track stays open") .unwrap() }; - assert!(matches!(read().await, Some(Read::Gap))); assert!(matches!(read().await, Some(Read::End { group: 16, end }) if end == ts(1_000_000))); } + /// A gap runs from the delivered edge to the resume point, in the resume's scale, and + /// never past it. + #[test] + fn a_gap_spans_the_edge_to_the_resume_in_its_scale() { + let scale = moq_net::Timescale::new(90_000).unwrap(); + let resume = Timestamp::new(90_000, scale).unwrap(); + let Read::Gap { timestamp, duration } = gap(Duration::from_millis(500), resume) else { + unreachable!() + }; + assert_eq!(timestamp, Timestamp::new(45_000, scale).unwrap()); + assert_eq!(duration, Timestamp::new(45_000, scale).unwrap()); + + // A next group overlapping the edge leaves an empty span at the resume point. + let Read::Gap { timestamp, duration } = gap(Duration::from_secs(2), resume) else { + unreachable!() + }; + assert_eq!(timestamp, resume); + assert_eq!(duration, Timestamp::new(0, scale).unwrap()); + } + #[tokio::test] - async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { + async fn gaps_mark_holes_while_a_marker_reads_as_an_end() { let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); @@ -998,10 +1043,11 @@ mod tests { assert_eq!( items(&mut reader).await, [ - "gap", + // Nothing was read before the join, so it is no gap. Legacy carries no duration, so + // the hole starts at the last frame read. "0", "10000", - "gap", + "gap 10000+990000", "1000000", "1010000", "end 1020000", @@ -1021,7 +1067,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(items(&mut reader).await, ["gap", "0", "gap", "1000000"]); + assert_eq!(items(&mut reader).await, ["0", "gap 0+1000000", "1000000"]); } /// A video group's trailing duration marker is that frame's end, not a declared stop. @@ -1051,7 +1097,7 @@ mod tests { group.finish().unwrap(); } track.finish().unwrap(); - assert_eq!(items(&mut reader).await, ["gap", "0", "33000"]); + assert_eq!(items(&mut reader).await, ["0", "33000"]); } /// An audio terminal opens its group with the endpoint, then the codec's final packets. @@ -1077,7 +1123,7 @@ mod tests { .unwrap(); group.finish().unwrap(); track.finish().unwrap(); - assert_eq!(items(&mut reader).await, ["gap", "0", "20000", "end 50000", "40000"]); + assert_eq!(items(&mut reader).await, ["0", "20000", "end 50000", "40000"]); } #[tokio::test] @@ -1351,7 +1397,7 @@ mod tests { let read = |consumer: &mut Consumer| loop { match consumer.poll_read(&waiter) { Poll::Ready(Ok(Some(Read::Frame { frame, .. }))) => break Some(frame.timestamp), - Poll::Ready(Ok(Some(Read::Gap))) => continue, + Poll::Ready(Ok(Some(Read::Gap { .. }))) => continue, Poll::Pending => break None, other => panic!("unexpected read: {other:?}"), } @@ -2001,7 +2047,10 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); - assert!(matches!(next, Read::Gap), "the evicted media is a gap"); + assert!( + matches!(next, Read::Gap { timestamp, duration } if timestamp == ts(0) && duration == ts(150_000)), + "the evicted media is a gap from the last frame to the live group: {next:?}" + ); let next = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 26602015cdc..f033ec5622f 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -55,10 +55,18 @@ pub enum Read { group: u64, frame: Frame, }, - /// Media before the next item will never reach this reader: it joined mid-track, or a - /// group was skipped, evicted, or never arrived. A decoder resumes from here without - /// the references or state that media built. - Gap, + /// Media after an item already read will never reach this reader: a group was skipped, + /// evicted, or never arrived. A decoder resumes from here without the references or + /// state that media built. Joining mid-track is not a gap: nothing was read before it. + /// + /// The span is an upper bound, in the scale of the item that follows. + Gap { + /// The furthest presentation point already read. Without a per-frame duration this + /// is the last frame's timestamp, one frame early. + timestamp: moq_net::Timestamp, + /// How long until the next item resumes, so it also covers any declared pause. + duration: moq_net::Timestamp, + }, /// The publisher declared that its media stops at `end`: it paused, switched source, or /// finished. Frames that follow from the same `group` are the stopped encoder's final /// packets (an audio codec's drain), whose output at or past `end` is padding. The next diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index 20d24b7a712..875b942defb 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -233,7 +233,7 @@ impl Track { let frame = match self.consumer.read().await? { Some(moq_mux::container::Read::Frame { frame, .. }) => frame, // RTP sequence numbers and timestamps already show the hole to a receiver. - Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }) => continue, None => return Ok(None), }; let payload = match &self.convert { diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index 2766366623c..c1f38d69d29 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -711,7 +711,7 @@ mod tests { loop { match reader.read().await.unwrap()? { moq_mux::container::Read::Frame { frame, .. } => return Some(frame), - moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. } => continue, + moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. } => continue, } } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 58f1bdf0604..f47c1443d41 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -168,7 +168,7 @@ impl Consumer { Some(Read::Frame { frame, .. }) => Some(frame), // The next group starts on a keyframe with parameter sets, so neither a // gap nor a declared end flushes the decoder. - Some(Read::Gap | Read::End { .. }) => continue, + Some(Read::Gap { .. } | Read::End { .. }) => continue, None => None, };