From 7eb5ecf0bfbd833b31b636a2cd387952fdb9cfb4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 17:12:29 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m2/groups-never-overlap.md Quest-Claim: 2fab16df5cb8d985 From d6ddabd9f1f790d5101861d318689e4a8c80e281 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 18:57:45 -0800 Subject: [PATCH 2/6] fix(mux): groups never overlap: every frame of a group presents before every frame of a higher group A keyframe at or before the newest frame written continues the open group (a TS content join), and a later frame presenting at or before an earlier group's newest frame (a leading picture after a splice) is dropped and counted as skipped. The Rust producer, the fMP4 passthrough, and the JS legacy producer keep it; the hang draft states it. Co-Authored-By: Claude Opus 5.5 --- doc/bin/hls.md | 2 +- drafts/draft-lcurley-moq-hang.md | 10 +- js/hang/src/container/consumer.rewind.test.ts | 22 +- js/hang/src/container/consumer.test.ts | 43 +++ js/hang/src/container/legacy.ts | 33 +- quest/m2/groups-never-overlap.md | 92 ------ quest/m2/lite07-finalize.md | 3 +- quest/m2/time-addressing/group-times.md | 8 +- quest/m3/expire-by-higher-min.md | 7 +- quest/m3/fmp4-group-start-tfdt.md | 12 +- rs/moq-hls/src/export/grid.rs | 8 +- rs/moq-mux/src/container/consumer.rs | 29 +- rs/moq-mux/src/container/fmp4/import.rs | 66 +++- rs/moq-mux/src/container/fmp4/import_test.rs | 65 ++++ rs/moq-mux/src/container/producer.rs | 299 +++++++++++++++--- rs/moq-mux/src/container/ts/import.rs | 63 +++- 16 files changed, 520 insertions(+), 242 deletions(-) delete mode 100644 quest/m2/groups-never-overlap.md diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 17f7166af67..76c9a65f1e8 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -59,7 +59,7 @@ What that means for you: - **Every rendition shares the grid.** A group spans from its start timestamp to the next group's and belongs to the first grid line it contains, or, containing none, the line it starts after. Segment N therefore starts at the group containing N times the segment length, every rendition numbers its segments alike, and each cuts its boundaries at its own keyframes. A grid line no group starts a segment at, because a GOP ran past it or the content paused, is listed as an `EXT-X-GAP` so the numbering never skips: on a 2 s grid, a GOP from 0 to 7 s is segment 0, segments 1 and 2 are gaps, and the next GOP starts segment 3. Timestamps that go back start a discontinuity on that rendition alone; `EXT-X-PROGRAM-DATE-TIME`, written after every discontinuity when the catalog has a clock, lets players resync. - **Include media starts on selection.** The HTTP multivariant playlist reads each root `include` entry's catalog and lists its audio and video beside the broadcast's own. An include shares the broadcast's timeline: its renditions are segmented on the broadcast's join interval and clock, whatever its own catalog declares, so publish it on the same timestamps. One not yet routed is listed once its broadcast is announced. Reading its catalog starts no encoding: only a request to one of its rendition URLs subscribes to that rendition's media. Demand lasts through active responses and, after the last one finishes, for three target durations or the playlist window plus one target duration, whichever is longer (20 s by default); a later request starts it again. The broadcast's own renditions stay subscribed. A segment is servable for as long as the relay's [cache](/bin/relay/config#cache) retains it. Keep `--window` (the playlist length, default 16 s) within that retention and within the media's max age (30 s by default), after which a superseded segment's groups are stale. - **Live joins cost the window.** The exporter subscribes 30 s behind the live edge and lists at most 256 segments. One joining partway through a segment lists nothing before the first segment it sees start. -- **Segments are timed by the grid.** A segment is listed once a later one starts or the track ends. A GOP that starts off a line is placed as soon as one of its frames passes the next line, without waiting for the next keyframe, so the segment before it lists then. A keyframe may overlap the previous GOP's last frames (a TS content join does), so when that overlap straddles a line, an edge that saw the earlier GOP lists it in the line's segment, while an edge that joined at the overlapping keyframe lists that segment without it. This one segment differs across edges, for that broadcast only (accepted, 2026-11-15). It starts at its first group's timestamp, or at its line when that group starts after it (as one resuming from a pause does), or a gap at its line, and lasts until the next segment starts, so a segment's bytes never change with a catalog estimate. A GOP longer than the segment is listed at its first line, so its duration understates its media. +- **Segments are timed by the grid.** A segment is listed once a later one starts or the track ends. A GOP that starts off a line is placed as soon as one of its frames passes the next line, without waiting for the next keyframe, so the segment before it lists then. It starts at its first group's timestamp, or at its line when that group starts after it (as one resuming from a pause does), or a gap at its line, and lasts until the next segment starts, so a segment's bytes never change with a catalog estimate. A GOP longer than the segment is listed at its first line, so its duration understates its media. - **The target duration is twice the segment.** Every media playlist keeps one `EXT-X-TARGETDURATION` while its rendition stays in the catalog, even across a reconfigure: twice the segment length, rounded up to whole seconds (twice the interval for a GOP of a second or more, 2 s for 20 ms audio groups), since the GOP containing a grid line starts at most one interval before it, and a gap ends under two segment lengths after its line. Only a final GOP longer than that is listed past it, against RFC 8216's rule: hls.js plays it, but Apple's `mediastreamvalidator` flags it, so keep keyframe intervals within the declared interval. - **A reconfigure keeps the playlist.** A rendition whose decoder config changes (its resolution, say) keeps its media playlist: the segments already listed stay, under the init they were muxed for, and the new config's segments follow behind an `EXT-X-DISCONTINUITY` and a new `EXT-X-MAP`, as RFC 8216 requires. The old config's segments end at the keyframe declaring the new size, even when that group reaches the export before the catalog update. A resize that opens a segment lists that segment under the new init. One partway through a segment cuts the old config's last segment short there, and the new config lists from the next segment, so the media between them is never listed. The DASH manifest lists only the new config's segments. - **Disabled renditions are hidden.** A rendition the catalog marks `enabled: false` leaves the multivariant playlist and DASH manifest until it re-enables. Its media playlist keeps serving, so a player already on it is not ended with `EXT-X-ENDLIST`. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 54ce7a86032..3e51505aa80 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -905,7 +905,7 @@ Every container shares the same group rules: Each group MUST start with a keyframe, except a group that contains only a discontinuity marker. If the codec does not support delta frames (e.g. audio), a group MAY consist of multiple keyframes. -Otherwise, a group MUST consist of a single keyframe followed by zero or more delta frames. +Otherwise, a group MUST consist of a keyframe followed by zero or more frames that decode from the frames before them in the group. A group with no decodable frames is a walk-now discontinuity: one empty codec payload and no media. An empty payload is data on a data track, so in every container a data track's marker is one object with no bytes, which is no frame; it ends at the furthest point the consumer has read. @@ -914,9 +914,9 @@ A consumer MUST NOT submit the marker to a decoder. Empty groups (zero objects) are permitted and mean nothing. After a discontinuity the timeline continues forward. A publisher that stops producing and may resume on the same track (e.g. an encoder idle for lack of demand) SHOULD publish a discontinuity marker when it stops, so the group before the pause does not reach across the gap and read as live. -The first timestamp of a group MUST NOT be lower than the first timestamp of any group with a lower start, and no frame of a group may be lower than the first timestamp of the group before it; a group that breaks either is malformed. -A group MAY start at the same timestamp as the previous group. -Frames MAY present below the previous group's content (B-frames, or a keyframe overlapping the previous group's last frame). +Every frame of a group MUST present after every frame of any group with a lower start; a group that breaks this is malformed. +Frames within a group MAY present out of order (B-frames, open-GOP leading pictures), so a consumer bounds a group that isn't whole by the lowest timestamp it knows in a higher group. +A publisher splicing in new content keeps this by continuing the current group at a keyframe that doesn't present after the group's newest frame, and by dropping a later frame that presents at or before the previous group's newest frame, such as a leading picture that references the old content. A publisher whose source restarts below the previous group's start begins a new broadcast instead. A hole between delivered groups is a playhead event unless the publisher declared the starts between them skipped or the boundary is contiguous within 1 ms. A consumer re-applies startup delay and skip at a playhead event; it does not reset codec state. @@ -1082,7 +1082,7 @@ This document has no IANA actions. - A `ttml` payload is one complete TTML document in the `media` time base, clipped to its frame: its `body` begin matches the frame timestamp, and its `body` end bounds what it presents. - Specified MSF text metadata preservation and refusal of unrepresentable text packaging during gateway conversion. - Added optional `bitrate`, `jitter`, and `delay` fields to `json` and `binary` track entries, `delay` measured only from payloads stamped with their capture time on the broadcast clock. -- Replaced the live-edge floor with monotonic group starts: a group may overlap the previous group's content but not start before it. +- Replaced the live-edge floor with groups that never overlap: every frame of a group presents after every frame of a lower group, while frames within a group may reorder. A spliced keyframe that doesn't present after its group's newest frame continues that group. - A data track marks a discontinuity with a group of one object with no bytes, since its empty payload is data. An object with no bytes on an audio or video track, or sharing its group, is malformed. - Timed CMAF samples from the frame timestamp, which is the fragment's earliest presentation time; `tfdt` is only relative within the fragment, and places the samples of an untimed frame. A timed CMAF track counts in its `mdhd` ticks, and a fragment's runs lie back to back in its `mdat`, each `data_offset` counting from its `moof`. - A `loc` frame's codec payload uses the `legacy` framing. diff --git a/js/hang/src/container/consumer.rewind.test.ts b/js/hang/src/container/consumer.rewind.test.ts index 5f95f1ad55c..065ecb56aa2 100644 --- a/js/hang/src/container/consumer.rewind.test.ts +++ b/js/hang/src/container/consumer.rewind.test.ts @@ -71,7 +71,7 @@ test("Consumer accepts a resuming keyframe before the estimated group end", asyn }); for (const start of [16_000, 0]) { - test(`Producer and Consumer accept the next group starting at ${start}us`, async () => { + test(`Producer continues the open group at a keyframe at ${start}us`, async () => { const { track, consumer } = setup(); const producer = new Producer(track, new Format("video")); try { @@ -80,8 +80,8 @@ for (const start of [16_000, 0]) { expect((await frame(consumer))?.group).toBe(0n); await frame(consumer); producer.encode(new Uint8Array([1]), Duration.micros(start), true); - // A keyframe that doesn't start past the open group continues it. - expect((await frame(consumer))?.group).toBe(BigInt(start)); + // A keyframe that doesn't start past the newest frame continues the open group. + expect((await frame(consumer))?.group).toBe(0n); } finally { consumer.close(); producer.finish(); @@ -89,22 +89,6 @@ for (const start of [16_000, 0]) { }); } -test("Consumer accepts leading pictures below the previous group's content", async () => { - const { track, consumer } = setup(); - try { - group(track, [100_000, 180_000]); - await frame(consumer); - await frame(consumer); - group(track, [200_000, 150_000]); - expect((await frame(consumer))?.frame?.timestamp.asMicros()).toBe(200_000); - expect((await frame(consumer))?.frame?.timestamp.asMicros()).toBe(150_000); - } finally { - consumer.close(); - track.finish(); - track.close(); - } -}); - test("Consumer rejects a later frame below the previous group start", async () => { const { track, consumer } = setup(); try { diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index d62b3354f62..50bb3b03ee4 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -198,6 +198,49 @@ test("Legacy Producer accepts open-GOP leading pictures above the previous group producer.finish(); }); +test("Legacy Producer continues a group at a keyframe overlapping it, and drops a leading picture before it", async () => { + const track = micro("test"); + const subscriber = track.subscribe({ delay: { max: Duration.millis(30_000) } }); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + for (const [timestamp, keyframe] of [ + [100_000, true], + [166_000, false], + [133_000, false], + // A splice dipping below the newest frame continues the group. + [150_000, true], + [183_000, false], + // Past the newest frame: a new group, whose leading picture at 183_000 presents with the last. + [200_000, true], + [183_000, false], + [190_000, false], + ] as const) { + producer.encode(new Uint8Array([1]), Duration.micros(timestamp), keyframe); + } + producer.finish(); + const groups = (await readGroups(subscriber, 200_000n)).map(([, frames]) => + frames.filter(([, size]) => size > 0).map(([timestamp]) => timestamp), + ); + expect(groups).toEqual([ + [100_000, 166_000, 133_000, 150_000, 183_000], + [200_000, 190_000], + ]); +}); + +test("Legacy Producer drops a keyframe overlapping a closed group", () => { + const track = micro("test"); + const producer = new LegacyProducer(track, new LegacyFormat("video")); + producer.encode(new Uint8Array([1]), Duration.micros(100_000), true); + producer.encode(new Uint8Array([1]), Duration.micros(166_000), false); + producer.discontinuity(Duration.micros(200_000)); + const largest = track.largest()?.group; + producer.encode(new Uint8Array([1]), Duration.micros(150_000), true); + expect(track.largest()?.group).toBe(largest); + expect(() => producer.encode(new Uint8Array([1]), Duration.micros(183_000), false)).toThrow( + "must start with a keyframe", + ); + producer.finish(); +}); + test("Legacy Producer writes a duration marker at the next keyframe", async () => { const track = micro("test"); const subscriber = track.subscribe({ delay: { max: Duration.millis(30_000) } }); diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index 64d8e2faed9..e6bd00b7bf0 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -147,9 +147,12 @@ export class Producer { #end?: number; // Furthest timestamp in finished groups, used only for discontinuity markers. #liveEdge?: number; - // The current group start and the previous group start, which bounds its frames. + // The current group start and the previous group start: a frame below it is a restart. #start?: number; #floor?: number; + // The newest frame of every group before the current one: every frame of the current group + // presents after it. + #ceiling?: number; // Gap between consecutive timestamps, used to close the last group when no successor exists. #interval?: number; // A discontinuity's marker is the newest group, so another one would say nothing new. @@ -163,24 +166,35 @@ export class Producer { this.#track = track; } - /** Encode and append a frame; a keyframe starts a new group. Throws if the first frame is not a keyframe, a group start goes backwards, or a frame is below the previous group start. */ + /** + * Encode and append a frame; a keyframe starts a new group. Every frame of a group presents after + * every frame of the groups before it, so a keyframe that doesn't start after the newest frame + * continues the open group (a splice), and a later frame presenting at or before the groups before + * its own (a leading picture after a splice) is dropped, as is a keyframe at or before the newest + * frame with no group open. Throws if the first frame is not a keyframe, a group start goes + * backwards, or a frame is below the previous group start. + */ encode(data: Uint8Array | Source, pts: Duration, keyframe: boolean) { const timestamp = micros(pts); const floor = keyframe ? this.#start : this.#floor; if (floor !== undefined && timestamp < floor) { throw new Error("frame timestamp is below the previous group start"); } - // A keyframe opens a group starting at its timestamp, which has to start after every group the - // track holds. One that doesn't, such as parameter sets and the picture they precede written - // as two keyframes at one timestamp, continues the open group instead. With none open it is + // A keyframe opens a group starting at its timestamp, which has to start after every frame + // written and every group the track holds. One that doesn't, such as parameter sets and the + // picture they precede written as two keyframes at one timestamp, or a splice dipping below + // the old content's last frames, continues the open group instead. With none open it is // refused, before anything changes. - const opens = keyframe && startsPast(this.#track, timestamp); + const behind = this.#newest !== undefined && timestamp <= this.#newest; + const opens = keyframe && !behind && startsPast(this.#track, timestamp); + // Dropped: a keyframe with no group to continue, and a frame presenting with the groups + // before its own, such as a leading picture after a splice. + if (keyframe && behind && !this.#group) return; if (keyframe && !opens && !this.#group) throw new Error("a keyframe must start after the previous group"); + if (!opens && this.#group && this.#ceiling !== undefined && timestamp <= this.#ceiling) return; this.#marked = false; if (opens) { - const rewound = this.#previous !== undefined && timestamp < this.#previous; - this.#close(rewound ? undefined : timestamp); - if (rewound) this.#interval = undefined; + this.#close(timestamp); // A track's first group says nothing came before it, so a reader can tell it is the first. if (this.#track.largest() === undefined) { const start = keyOf(this.#track, timestamp); @@ -188,6 +202,7 @@ export class Producer { } this.#group = this.#track.appendGroup(Time.Timestamp.fromMicros(timestamp)); this.#floor = this.#start; + this.#ceiling = this.#newest; this.#start = timestamp; } else if (!this.#group) { throw new Error("must start with a keyframe"); diff --git a/quest/m2/groups-never-overlap.md b/quest/m2/groups-never-overlap.md deleted file mode 100644 index b36a043b737..00000000000 --- a/quest/m2/groups-never-overlap.md +++ /dev/null @@ -1,92 +0,0 @@ -# [L] Every frame of a group presents before every frame of a higher group - -## Goal - -Groups never go back in time: every frame of a group presents strictly before -every frame of any higher group, so a reader bounds a group that isn't whole -by the lowest timestamp it knows in a higher group. Frames within a group may -still reorder (B-frames, open-GOP leading pictures). Sources that overlap -today, such as a TS content join, publish without ending the broadcast and -without rewriting a timestamp. The hang draft states the rule, and the -producers and every importer keep it, in Rust and JS. -[Group times](/quest/m2/time-addressing/group-times.md) puts it in the moq-lite -draft, as spans from a group's lowest frame to its End that never overlap. - -## Plan - -[moq-dev/moq#4543](https://github.com/moq-dev/moq/pull/4543) replaced the -importers' live-edge floor and TS `Reanchor` shift with monotonic group starts. -With nothing shifted, a TS content join -([#3798](https://github.com/moq-dev/moq/issues/3798)) publishes an IDR below -the old content's last frame, so the draft now lets frames present below the -previous group's content (`drafts/draft-lcurley-moq-hang.md`, the malformed-group -rules), and even lets a group start at the previous group's start. -`container::Producer::prepare` opens a group for a keyframe above the open -group's start, even one below its newest frame -(`a_keyframe_overlapping_the_previous_group_publishes`). The moq-lite draft -only orders frame 0s ("a publisher MUST NOT start a Group's frame 0 at or -before that of any Group with a lower start", in its Frame section), and -`@moq/net` accepted a frame at 250 in group 100 after group 200 opened. - -That overlap is why -[moq-dev/moq#8874](https://github.com/moq-dev/moq/pull/8874) cannot place an -HLS group early with certainty: when the overlap straddles a grid line, an -edge that saw the earlier group and one that joined at the overlapping group -list one segment differently. #8874 documents that case in -`rs/moq-hls/src/export/grid.rs` and `doc/bin/hls.md`; delete both notes here. - -Start by finding every source that overlaps and why: the TS content join, -other importers (fMP4, FLV, MKV, RTMP, SRT, HLS), encoders, and the JS -publisher. - -Decided 2026-11-15 (maintainer, /quest-plan): - -- ✅ Enforce the rule end to end: draft, producer, importers (rejected: - investigate only and plan the fix later; end the import at an overlapping - keyframe, which would stop a TS feed at every content join). -- ✅ A keyframe that doesn't start after the open group's newest frame continues - that group, as one at or below its start already does (rejected: shift the - broadcast forward at each join, which brings back the rewriting #4543 - removed and drifts the catalog clock; leave it to the implementer). The - group at a join then runs about two GOPs: a viewer can't join exactly there, - and HLS lists one over-long segment through its overrun path. -- ✅ Readers assume the rule and don't detect it, per AGENTS.md (rejected: - `container::Consumer` refusing an overlapping group). A publisher that - breaks it hurts only its own broadcast. Delete any reader-side overlap - handling the rule makes dead. moq-net's frame 0 check (`check_start`) stays - as it is. - -Decided 2026-11-20 (maintainer, /quest-plan auditing the subscription rules -before the lite-07 cut): - -- ✅ The strong form: every frame of a lower group presents before every frame - of a higher one (rejected: the weak form, every frame before the next - group's frame 0, which bounds an incomplete group only by a higher group's - frame 0). Conforming HEVC and H.264 already keep it, since a leading picture - follows every picture of the previous GOP in output order. Only a splice can - break it. -- ✅ The cut waits on this quest (moved from m3): lite-07 publishes the rule - as a publisher MUST through group times, and first-party producers break - it today. - -Recommended for a splice: a frame after the keyframe that would present at or -before the previous group's newest frame, such as a RASL picture, is dropped. -It references pictures from before the splice, so it can't decode there -anyway. Check RADL pictures, which do decode, before settling on dropping -them too. - -The hang draft's "a group MAY start at the same timestamp as the previous -group" goes too. - -Tests: the producer test above becomes one where the overlapping keyframe -continues the group, and a TS import across a content join, including leading -pictures, publishes groups whose frames all precede every frame of the next -group. - -Public API: none expected. Wire: none; the hang draft's group rule tightens. - -## Related - -- [Group Streams carry each time once](/quest/m2/time-addressing/group-times.md) - states the rule in the moq-lite draft, with GROUP_END's End closing each span -- [An incomplete group expires by the lowest timestamp of a higher group](/quest/m3/expire-by-higher-min.md) - the expiry bound this rule makes safe -- [An fMP4 group start never rewinds the decode timeline](/quest/m3/fmp4-group-start-tfdt.md) - a keyframe repeating its predecessor's timestamp would now continue the group; check which of its cases remain diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index b8082677334..5133965d3e6 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -220,7 +220,7 @@ the version that ships them. Decided 2026-11-20 (maintainer, /quest-plan auditing the cache, timestamp, and switching rules in `rs/moq-net/AGENTS.md` before the cut): -- ✅ The cut waits for [groups never overlap](/quest/m2/groups-never-overlap.md) +- ✅ The cut waits for groups never overlap, [moq-dev/moq#9264](https://github.com/moq-dev/moq/pull/9264) (rejected: after the cut). [Group times](/quest/m2/time-addressing/group-times.md) puts the strong rule in the draft as a publisher MUST: a group's span, from its lowest frame to its End, never overlaps another's. Expiry relies on it, @@ -238,7 +238,6 @@ and switching rules in `rs/moq-net/AGENTS.md` before the cut): - [No relative anchors](/quest/m2/time-addressing/no-relative-anchors.md) - SUBSCRIBE's anchors lose Relative, and Min Age places a live join - [Group Streams carry each time once](/quest/m2/time-addressing/group-times.md) - GROUP loses Previous Start, FRAME's first delta counts from the start, and GROUP_END carries an end -- [Groups never overlap](/quest/m2/groups-never-overlap.md) - first-party producers and importers keep the no-overlap MUST the cut publishes - [A lite-07 claim is shown no broader than the grants it crossed](/quest/m2/lite07-claim-reach-routes.md) - a peer is never shown a claim at a path no route of it can serve, through any number of relays (maintainer, 2026-11-20) ## Related diff --git a/quest/m2/time-addressing/group-times.md b/quest/m2/time-addressing/group-times.md index d713ca4d9e8..99766fa90c6 100644 --- a/quest/m2/time-addressing/group-times.md +++ b/quest/m2/time-addressing/group-times.md @@ -77,8 +77,11 @@ Recommended, for review: - Aging: a whole group's reach is its own End, and any other group's is bounded by its successor's start, as today. - With open-GOP leading pictures, a group's lowest frame sits below its - start. Check the Time Anchor rule against spans there, alongside - [groups never overlap](/quest/m2/groups-never-overlap.md). + start. Check the Time Anchor rule against spans there. Since [moq-dev/moq#9264](https://github.com/moq-dev/moq/pull/9264), + moq-mux keeps every frame of a group after every frame of a lower one, but + `container::Producer` still finishes a group whole at the next keyframe's + timestamp, above that keyframe's leading pictures, so the two spans + overlap. - `group::Producer::finish_at(end)` and `group::Consumer::end()` stay, since the end is what GROUP_END encodes. @@ -113,4 +116,3 @@ which is unpublished. - [lite-07 has no relative anchors](/quest/m2/time-addressing/no-relative-anchors.md) - its live join resolves as a time start, which this changes inside a skipped span - [A scale-less track ages by arrival](/quest/m2/time-addressing/scale-less-arrival.md) - lite-05/06 still stamp a scale-less frame, with its group's start -- [Groups never overlap](/quest/m2/groups-never-overlap.md) - enforces the non-overlap rule in hang's producers and importers diff --git a/quest/m3/expire-by-higher-min.md b/quest/m3/expire-by-higher-min.md index 5e5c4605a0b..97cc20cb51a 100644 --- a/quest/m3/expire-by-higher-min.md +++ b/quest/m3/expire-by-higher-min.md @@ -31,9 +31,9 @@ do today. This quest tightens the second half once producers keep the strong rule. - A higher group's lowest timestamp bounds a lower group only under the strong - no-overlap rule, so this waits on it. Until then moq-mux's overlapping groups - would expire early. With open-GOP leading pictures, that lowest frame sits - below the higher group's start. + no-overlap rule, which moq-mux's producers keep since [moq-dev/moq#9264](https://github.com/moq-dev/moq/pull/9264). With + open-GOP leading pictures, that lowest frame sits below the higher group's + start. - Use any known frame of any higher group, a tail copy's included. Under the rule, every one of them is above the lower group's frames. - Drop `ended_at`'s wall-clock aging of the latest group (JS `#endedAt`), if @@ -46,5 +46,4 @@ which only lets a publisher expire sooner. ## Required -- [Every frame of a group presents before every frame of a higher group](/quest/m2/groups-never-overlap.md) - first-party producers keep the rule that makes a higher group's lowest timestamp an upper bound - [Group Streams carry each time once](/quest/m2/time-addressing/group-times.md) - GROUP_END's End, the whole group's bound, and the draft's no-overlap MUST diff --git a/quest/m3/fmp4-group-start-tfdt.md b/quest/m3/fmp4-group-start-tfdt.md index 1906a2d18e2..101417f9e62 100644 --- a/quest/m3/fmp4-group-start-tfdt.md +++ b/quest/m3/fmp4-group-start-tfdt.md @@ -26,10 +26,11 @@ Suppose the previous group's last sample took the catalog cadence because its successor was on the same or the next output tick. The two cases fail differently today: -- An exact repeat never reaches the encoder: `emit_fragment` refuses the - keyframe as `TrackRewound` because its timestamp is at or before - `track.written`. That guard must accept a keyframe equal to `written` while - still refusing one strictly before it, a genuine rewind. +- An exact repeat no longer reaches an exporter: groups never overlap + ([moq-dev/moq#9264](https://github.com/moq-dev/moq/pull/9264)), so a + keyframe at or before the previous group's newest frame continues that + group, and `emit_fragment` refusing one as `TrackRewound` only refuses a + malformed track. - A sub-tick near-repeat (since #8470) exports with a `tfdt` after the previous `tfdt` but before the previous fragment's decode end. `fmp4::Import` only compares `tfdt`s, so it accepts the overlap, but the two fragments @@ -59,7 +60,7 @@ Decided 2026-11-11 (maintainer, follow-up planning in /quest-complete): sample already. Test, for video and audio and on each site above: exports whose last frame -before a keyframe repeats or nearly repeats the keyframe's timestamp succeed, +before a keyframe nearly repeats the keyframe's timestamp succeed, re-import through `fmp4::Import`, and each fragment's `tfdt` is at least the previous fragment's `tfdt` plus its sample durations (the check `export_test.rs` already uses). A keyframe strictly before `written` still @@ -70,4 +71,3 @@ Public API: none. Wire: none. ## Related - [fMP4 decode times from the declared depth](/quest/m3/fmp4-declared-clock.md) - replaces how reordered video's decode times are authored; this quest's rule must hold under it too -- [A group's frames present before the next group starts](/quest/m2/groups-never-overlap.md) - an overlapping or repeating keyframe continues its group there, which may remove some cases here diff --git a/rs/moq-hls/src/export/grid.rs b/rs/moq-hls/src/export/grid.rs index e8c7a013817..dcb26c8bf4e 100644 --- a/rs/moq-hls/src/export/grid.rs +++ b/rs/moq-hls/src/export/grid.rs @@ -12,12 +12,8 @@ //! contains, so segment N starts at the group containing pts N × length; or, if it contains none, //! the line it starts after, `floor(start / length)`. A group off the grid is placed //! once its successor starts, or sooner once one of its own frames passes the next line, since it -//! then contains that line unless its successor overlaps it; the segment before it completes then, -//! not a group later. A keyframe may overlap the previous group's last frames (a TS content join -//! does), so when that overlap straddles a line, an edge that saw the earlier group lists it in the -//! line's segment while one that joined at the overlapping group lists that segment without it. -//! That one segment differs across edges, for that broadcast alone (accepted by the maintainer, -//! 2026-11-15, for listing every off-grid segment up to a group sooner). Groups are never split, +//! then contains that line: every frame of its successor presents after all of its own. The +//! segment before it completes then, not a group later. Groups are never split, //! since a segment starting mid-GOP is not independent. A grid slot no group starts a //! segment in, because an overrun or a pause skipped it, is listed as a gap, so the numbering never //! skips: on a 2s grid, a group A = [0s, 7s) is segment 0, slots 1 and 2 are gaps, and a contiguous diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 5e587200567..33e467a7f00 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -168,9 +168,8 @@ pub struct Consumer { // backwards, so a later group with a frame below this is malformed. start: Option<(u64, Timestamp)>, - // The start of the group the cursor most recently left. B-frames, open-GOP pictures, and a - // keyframe overlapping the previous group's last frame may dip below that group's content, - // but a frame below its start is a rewind. + // The start of the group the cursor most recently left. A frame below it is a rewind. Every + // frame also presents after that group's content, which a reader assumes rather than checks. floor: Option, // Presentation end of the group we most recently advanced past, for the 1 ms @@ -3405,30 +3404,6 @@ mod tests { ); } - /// A keyframe one frame below the previous group's last frame, but above its start, is an - /// overlap rather than a rewind. - #[kio::test] - async fn a_keyframe_overlapping_the_previous_group_is_accepted() { - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); - write_group(&mut track, 0, &[ts(0), ts(33_000), ts(66_000)]); - // Read the first group before the second arrives: a blocked group that the next one - // already covers is skipped for latency, which is not what this checks. - for expected in [0, 33_000, 66_000] { - assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(expected)); - } - write_group(&mut track, 1, &[ts(50_000), ts(83_000)]); - let _track = track.finish().unwrap(); - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!( - frames - .iter() - .map(|f| f.timestamp.as_micros().unwrap()) - .collect::>(), - vec![50_000, 83_000] - ); - } - #[kio::test] async fn open_gop_leading_pictures_above_the_previous_group_are_accepted() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index b93d302745c..4f17b52aed4 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -161,6 +161,11 @@ struct Passthrough { group: Option, /// The latest group's start, which the next group may not start below. start: Option, + /// The newest sample written, which the next group starts after. + newest: Option, + /// The newest sample of every group before the latest one: every sample of the latest group + /// presents after it. + ceiling: Option, // This track's place in the broadcast's join interval. Passthrough writes groups by hand (no // `container::Producer`), so it reports what its estimator measured itself. @@ -262,6 +267,8 @@ struct Run { samples: Vec, /// The earliest presentation time, which a passthrough fragment is stamped with. earliest: Timestamp, + /// The latest presentation time of any sample. + latest: Timestamp, /// The furthest presentation end any sample reaches. end: Timestamp, /// The edit list dropped some of the fragment's samples. @@ -638,6 +645,8 @@ impl Import { track, group: None, start: None, + newest: None, + ceiling: None, member, pending_start: None, estimator: self.catalog.estimator(), @@ -1230,7 +1239,7 @@ impl Import { let prefix = pending.prefix.take().expect("one prefix per fragment"); let (writer, opened) = output.start(prefix, &pending.run, track.kind, pending.placement)?; - (Writer::Fragment(writer), opened) + (writer.map_or(Writer::Dropped, Writer::Fragment), opened) } output => { let (sample, opened) = open_sample(output, pending, *sample_index, track.timescale)?; @@ -1247,12 +1256,11 @@ impl Import { if complete { let last = *sample_index + 1 == pending.run.samples.len(); match &mut track.output { - Output::Passthrough(output) if last => { - let Some(Writer::Fragment(writer)) = pending.writer.take() else { - unreachable!("passthrough tracks write whole fragments"); - }; - output.complete(writer, pending, track.kind)?; - } + Output::Passthrough(output) if last => match pending.writer.take() { + Some(Writer::Fragment(writer)) => output.complete(writer, pending, track.kind)?, + Some(Writer::Dropped) => {} + _ => unreachable!("passthrough tracks write whole fragments"), + }, Output::Passthrough(_) => {} output => media!(output, producer => { let Some(Writer::Sample(sample)) = pending.writer.take() else { @@ -1312,7 +1320,7 @@ impl Mdat { let _ = writer.abort(err.clone()); } Some(Writer::Sample(sample)) => sample.abort(err.clone()), - None => {} + Some(Writer::Dropped) | None => {} } } } @@ -1332,6 +1340,8 @@ struct PendingTrack { enum Writer { /// A passthrough track's whole per-track fragment. Fragment(moq_net::frame::Producer), + /// A passthrough fragment the track dropped, whose payload is discarded. + Dropped, /// One unwrapped sample. Sample(Streamed), } @@ -1340,6 +1350,7 @@ impl Writer { fn write(&mut self, chunk: &[u8]) -> crate::Result<()> { match self { Self::Fragment(writer) => writer.write(chunk)?, + Self::Dropped => {} Self::Sample(sample) => sample.write(chunk)?, } Ok(()) @@ -1400,6 +1411,7 @@ impl Fmp4Track { self.last_decode_time = Some(dts); let mut earliest = None; + let mut latest = None; let mut end = None; let mut samples = Vec::with_capacity(sample::count(traf) as usize); let mut dropped = false; @@ -1431,6 +1443,7 @@ impl Fmp4Track { }; end = Some(end.map_or(sample_end, |max| crate::container::later(max, sample_end))); earliest = Some(earliest.map_or(timestamp, |min| crate::container::earlier(min, timestamp))); + latest = Some(latest.map_or(timestamp, |max| crate::container::later(max, timestamp))); samples.push(Sample { dts, @@ -1450,7 +1463,7 @@ impl Fmp4Track { } } - let (Some(earliest), Some(end)) = (earliest, end) else { + let (Some(earliest), Some(latest), Some(end)) = (earliest, latest, end) else { return match dropped { true => Ok(None), false => Err(Error::MissingTrun.into()), @@ -1459,6 +1472,7 @@ impl Fmp4Track { Ok(Some(Run { samples, earliest, + latest, end, dropped, })) @@ -1579,14 +1593,15 @@ fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { } impl Passthrough { - /// Open one per-track fragment's output frame, reporting whether it opened a group. + /// Open one per-track fragment's output frame, reporting whether it opened a group, or `None` + /// for a fragment dropped because it presents with the groups before its own. fn start( &mut self, prefix: Bytes, run: &Run, kind: Kind, placement: Placement, - ) -> Result<(moq_net::frame::Producer, bool)> { + ) -> Result<(Option, bool)> { // Carry the fragment's earliest presentation time as the frame timestamp, in the // track's native timescale and shifted onto the catalog clock. Consumers present the // fragment at it; `tfdt` only places the samples relative to each other. @@ -1604,6 +1619,27 @@ impl Passthrough { { return Err(crate::container::TimestampRewind { timestamp, floor }.into()); } + // Every sample of a group presents after every sample of the groups before it, so a segment + // that doesn't start after the newest sample written continues the open group, as + // `container::Producer` does for a keyframe; with none open it is dropped. So is a later + // fragment presenting with the groups before its own, such as a leading picture after a + // splice, its samples counted as skipped. + let behind = self.newest.is_some_and(|newest| timestamp <= newest); + if start_group && behind && self.group.is_some() { + start_group = false; + } + let dropped = match start_group { + true => behind, + false => self.ceiling.is_some_and(|ceiling| timestamp <= ceiling), + }; + if dropped { + tracing::debug!( + ?timestamp, + "dropping a fragment that presents before its group's predecessors end" + ); + self.sent.skip(run.samples.len() as u64); + return Ok((None, false)); + } // A group is keyed by its first fragment's timestamp, which has to start after every group // the track holds. A segment that doesn't continues the open group instead; with none open // it is refused. @@ -1647,8 +1683,14 @@ impl Passthrough { writer.write(prefix)?; if start_group { self.start = Some(timestamp); + self.ceiling = self.newest; } - Ok((writer, start_group)) + let latest = placement.offset.apply(run.latest)?; + self.newest = Some( + self.newest + .map_or(latest, |newest| crate::container::later(newest, latest)), + ); + Ok((Some(writer), start_group)) } fn complete(&mut self, writer: moq_net::frame::Producer, pending: &PendingTrack, kind: Kind) -> Result<()> { diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index ae90d4a8671..b191a650f2c 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1403,6 +1403,71 @@ async fn video_joined_mid_gop_starts_at_a_sync_sample() { } } +/// Every sample of a group presents after every sample of the groups before it, passthrough or +/// unwrapped: a sync sample that doesn't start after the newest sample continues the open group, +/// and a later one presenting at or before the groups before its own is dropped. +#[tokio::test] +async fn video_groups_never_overlap() { + let (_, (video_id, video_scale), _) = bbb_init(); + let tick = video_scale.as_u64() / 24; + let non_sync = 0x0001_0000; + + let init = bbb_init_with_edits(video_id, vec![media_edit(0)]); + // `(cts, flags)` per sample, each fragment decoding from `dts`, in ticks. + let fragments: [(u64, &[(u64, u32)]); 6] = [ + // Presents 2, 6, 3, 4. + (0, &[(2, 0), (5, non_sync), (1, non_sync), (1, non_sync)]), + // A join at 5, below the newest sample: it continues the group. Presents 5, 9, 7, 8. + (4, &[(1, 0), (4, non_sync), (1, non_sync), (1, non_sync)]), + // A sync sample past the newest, with a leading picture at it. Presents 10, 9, 15, 11. + (8, &[(2, 0), (0, non_sync), (5, non_sync), (0, non_sync)]), + // Past the newest again. Presents 16. + (12, &[(4, 0)]), + // Presents 13, with the group before: dropped. + (13, &[(0, non_sync)]), + // Presents 17. + (14, &[(3, non_sync)]), + ]; + let cases = [ + // Unwrapped, every sample is placed on its own: the leading picture at 9 is dropped. + ( + false, + vec![vec![2, 6, 3, 4, 5, 9, 7, 8], vec![10, 15, 11], vec![16, 17]], + ), + // Passthrough places whole fragments by their earliest sample, so the one with the leading + // picture continues the group instead. + (true, vec![vec![2, 6, 3, 4, 5, 9, 7, 8, 10, 9, 15, 11], vec![16, 17]]), + ]; + for (passthrough, expected) in cases { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_cmaf(passthrough); + fmp4.decode(&init).unwrap(); + for (dts, samples) in fragments { + let entries: Vec<_> = samples.iter().map(|&(cts, flags)| (cts * tick, flags, 0xA)).collect(); + fmp4.decode(&video_fragment(video_id, dts * tick, tick, &entries)) + .unwrap(); + } + let _finished = fmp4.finish().unwrap(); + let [video, _] = published(&consumer, &catalog).await; + let presented: Vec = video + .iter() + .map(|(timestamp, _, _)| (timestamp * u128::from(video_scale.as_u64())).div_ceil(1_000_000) as u64 / tick) + .collect(); + assert_eq!(presented, expected.concat(), "passthrough {passthrough}"); + // Each group starts at its first sample, the lowest of its sync samples. + let name = catalog.snapshot().video.renditions.keys().next().unwrap().clone(); + let starts: Vec = starts(&consumer, &name) + .await + .into_iter() + .map(|start| (u128::from(start) * u128::from(video_scale.as_u64())).div_ceil(1_000_000) as u64 / tick) + .collect(); + let firsts: Vec = expected.iter().map(|group| group[0]).collect(); + assert_eq!(starts, firsts, "passthrough {passthrough}"); + } +} + #[test] fn test_bbb_init_roundtrip() { let data = include_bytes!("test_data/bbb.mp4"); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index dd077aeee83..e62627333f1 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -85,8 +85,11 @@ pub struct Producer { /// The keyframe timestamp that opened the latest group. The next group may not start below it. start: Option, - /// The start of the group before the latest one. No frame of the latest group sits below it. + /// The start of the group before the latest one. A frame below it is a restart, not a splice. floor: Option, + /// The newest frame of every group before the latest one: every frame of the latest group + /// presents after it. + ceiling: Option, /// Duration of the frame that last raised [`end`](Self::end), if it had one. Distinguishes /// an exclusive presentation point from a max timestamp that still needs an estimate. @@ -120,28 +123,49 @@ pub struct Producer { /// It holds the track's open group until [`Producer::commit`] returns it, so nothing else writes to /// the track meanwhile. Dropping it unfinished aborts that group. pub(crate) struct Streamed { - writer: moq_net::frame::Producer, + payload: Payload, timestamp: moq_net::Timestamp, duration: Option, keyframe: bool, size: usize, } +/// Where a [`Streamed`] frame's payload goes. +#[expect( + clippy::large_enum_variant, + reason = "nearly every frame is written; boxing the writer would allocate for each" +)] +enum Payload { + Written(moq_net::frame::Producer), + /// The producer dropped the frame (see [`Producer::write`]); this many bytes are still to come. + Dropped(usize), +} + impl Streamed { /// Append payload bytes, readable by subscribers at once. pub fn write(&mut self, chunk: &[u8]) -> crate::Result<()> { - self.writer.write(chunk)?; + match &mut self.payload { + Payload::Written(writer) => writer.write(chunk)?, + Payload::Dropped(remaining) => { + *remaining = remaining.checked_sub(chunk.len()).ok_or(moq_net::Error::WrongSize)?; + } + } Ok(()) } /// Payload bytes still to write. pub fn remaining(&self) -> usize { - self.writer.remaining() + match &self.payload { + Payload::Written(writer) => writer.remaining(), + Payload::Dropped(remaining) => *remaining, + } } /// Abort the frame and its group with `err`. pub fn abort(self, err: moq_net::Error) { - let _ = self.writer.abort(err); + if let Payload::Written(writer) = self.payload { + let _ = writer.abort(err); + } } } @@ -165,7 +189,15 @@ impl Producer { _ => return Err(crate::Error::UnsupportedContainer("a streamed frame".into())), }; timed(timestamp, duration)?; - self.prepare(timestamp, keyframe)?; + if !self.prepare(timestamp, keyframe)? { + return Ok(Streamed { + payload: Payload::Dropped(size), + timestamp, + duration, + keyframe, + size, + }); + } let group = self.group.take().expect("prepare opens a group"); let total = prefix.len().checked_add(size).ok_or(moq_net::Error::FrameTooLarge)?; let mut writer = group.create_frame(moq_net::frame::Info { @@ -174,7 +206,7 @@ impl Producer { })?; writer.write(prefix)?; Ok(Streamed { - writer, + payload: Payload::Written(writer), timestamp, duration, keyframe, @@ -185,7 +217,12 @@ impl Producer { /// Finish a frame [`open`](Self::open) returned once its payload is written, handing its group /// back to the track. pub(crate) fn commit(&mut self, frame: Streamed) -> crate::Result<()> { - let group = frame.writer.finish()?; + let writer = match frame.payload { + Payload::Written(writer) => writer, + Payload::Dropped(0) => return Ok(()), + Payload::Dropped(_) => return Err(moq_net::Error::WrongSize.into()), + }; + let group = writer.finish()?; self.group = Some(group); self.written(frame.timestamp, frame.duration, frame.size, frame.keyframe); Ok(()) @@ -271,6 +308,7 @@ impl Producer { newest: None, start: None, floor: None, + ceiling: None, last_duration: None, previous_timestamp: None, cadence: None, @@ -310,6 +348,7 @@ where newest: None, start: None, floor: None, + ceiling: None, last_duration: None, previous_timestamp: None, cadence: None, @@ -601,18 +640,25 @@ where /// independently decodable (audio) marks only the first frame of each group a keyframe (see /// [`needs_keyframe`](Self::needs_keyframe)) so the group spans more than one frame. /// - /// A group starts at its keyframe's timestamp, in the track's timescale, and starts strictly - /// increase: a keyframe that doesn't start after the open group continues it. + /// A group starts at its keyframe's timestamp, in the track's timescale, and every frame of a + /// group presents after every frame of the groups before it. Frames within a group may still + /// reorder (B-frames, open-GOP leading pictures). A keyframe that doesn't start after the newest + /// frame written continues the open group, as at a splice whose keyframe dips below the old + /// content's last frames. A later frame presenting at or before the previous groups' newest, + /// such as a leading picture after a splice, is dropped and counted as + /// [skipped](Self::report_skipped), and so is a keyframe at or before the newest frame with no + /// group open, such as after a [`cut`](Self::cut). /// - /// Group starts never go backwards: a keyframe below the start of the previous group, or any + /// Group starts never go backwards: a keyframe below the start of the latest group, or any /// frame below the start of the group before its own, returns /// [`TimestampRewind`](super::TimestampRewind) with the refused timestamp and that start, /// without writing, the way an oversized frame is refused. That is a restart, which is a new - /// broadcast. Frames may still dip below the previous group's content: B-frames, open-GOP - /// leading pictures, and a keyframe that overlaps the previous group's last frame. + /// broadcast. pub fn write(&mut self, frame: Frame) -> crate::Result<()> { timed(frame.timestamp, frame.duration)?; - self.prepare(frame.timestamp, frame.keyframe)?; + if !self.prepare(frame.timestamp, frame.keyframe)? { + return Ok(()); + } // Buffer or write the frame. if self.buffer_duration.is_zero() { @@ -641,8 +687,9 @@ where Ok(()) } - /// Check `timestamp` against the group starts, and open a group for a keyframe. - fn prepare(&mut self, timestamp: moq_net::Timestamp, keyframe: bool) -> crate::Result<()> { + /// Check `timestamp` against the groups before it and open a group for a keyframe, returning + /// whether to write the frame. + fn prepare(&mut self, timestamp: moq_net::Timestamp, keyframe: bool) -> crate::Result { let floor = match keyframe { true => self.start, false => self.floor, @@ -653,6 +700,21 @@ where return Err(super::TimestampRewind { timestamp, floor }.into()); } + // Every frame of a group presents after every frame of the groups before it, so a keyframe + // that doesn't start after the newest frame written continues the open group: a splice whose + // keyframe dips below the old content's last frames, as a TS content join does, publishes + // one longer group. With no group open, as after bytes were lost, it has nothing to continue + // and is dropped like the frames that follow it, until a keyframe past the newest frame. + if keyframe + && let Some(newest) = self.newest + && !timestamp_lt(newest, timestamp) + { + return match self.group { + Some(_) => Ok(true), + None => Ok(self.drop_frame(timestamp)), + }; + } + // A keyframe opens a group keyed by its timestamp, which has to start after every group the // track holds. One that doesn't, such as parameter sets and the picture they precede written // as two keyframes at one timestamp, continues the open group instead. With none open it is @@ -660,22 +722,16 @@ where let start = match keyframe && self.pending_start.is_none() { true => match (crate::start::opens(&self.inner, timestamp)?, &self.group) { (Some(start), _) => Some(start), - (None, Some(_)) => return Ok(()), + (None, Some(_)) => return Ok(true), (None, None) => Some(crate::start::after(&self.inner, timestamp)?), }, false => None, }; - // A keyframe cuts the previous group, using its timestamp as the boundary where the - // previous group's content ends, unless it overlaps that content. + // A keyframe cuts the previous group, its timestamp the boundary where that group's content + // ends. if keyframe { - let overlaps = self - .previous_timestamp - .is_some_and(|previous| timestamp_lt(timestamp, previous)); - self.cut((!overlaps).then_some(timestamp))?; - if overlaps { - self.cadence = None; - } + self.cut(Some(timestamp))?; } // Start a new group if needed; the first frame of a group must be a keyframe. @@ -699,11 +755,28 @@ where }; self.floor = self.start; + self.ceiling = self.newest; self.start = Some(timestamp); self.group = Some(group); + } else if self.ceiling.is_some_and(|ceiling| !timestamp_lt(ceiling, timestamp)) { + // A frame presenting with the groups before its own, such as a leading picture after a + // splice, is dropped: it references pictures from before the splice, or shows new + // content while the old still plays. + return Ok(self.drop_frame(timestamp)); } - Ok(()) + Ok(true) + } + + /// Count a frame that presents with the groups before it as skipped rather than writing it, + /// returning false for [`prepare`](Self::prepare) to report. + fn drop_frame(&self, timestamp: moq_net::Timestamp) -> bool { + tracing::debug!( + ?timestamp, + "dropping a frame that presents before an earlier group ends" + ); + self.report_skipped(1); + false } /// Measure and report a frame the group accepted unbuffered. @@ -2144,10 +2217,24 @@ mod tests { assert!(matches!(err, crate::Error::TimestampRewind(_))); } - /// #3798's join: an IDR one frame below the previous group's last frame, but above its - /// start, overlaps that group rather than rewinding it. + /// The media frames of each group, without duration markers. + fn media(groups: Vec>) -> Vec> { + groups + .into_iter() + .map(|group| { + group + .into_iter() + .filter(|(_, len)| *len > 0) + .map(|(ts, _)| ts) + .collect() + }) + .collect() + } + + /// #3798's join: an IDR one frame below the open group's newest frame, but above its start, + /// continues that group, so every frame of the next group presents after it. #[tokio::test] - async fn a_keyframe_overlapping_the_previous_group_publishes() { + async fn a_keyframe_overlapping_the_open_group_continues_it() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); @@ -2156,9 +2243,56 @@ mod tests { producer.write(frame(166_000, false)).unwrap(); producer.write(frame(150_000, true)).unwrap(); producer.write(frame(183_000, false)).unwrap(); + producer.write(frame(216_000, true)).unwrap(); let _finished = producer.finish().unwrap(); - let groups = collect_payloads(consumer).await; - assert_eq!(groups[1][..2], [(150_000, 2), (183_000, 2)]); + assert_eq!( + media(collect_payloads(consumer).await), + [vec![100_000, 133_000, 166_000, 150_000, 183_000], vec![216_000]] + ); + } + + /// A splice whose keyframe lands just past the old content: a leading picture that would + /// present at or before the old content's newest frame is dropped, and the rest publish. + #[tokio::test] + async fn a_leading_picture_at_or_before_the_previous_group_is_dropped() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); + producer.write(frame(100_000, true)).unwrap(); + producer.write(frame(166_000, false)).unwrap(); + producer.write(frame(133_000, false)).unwrap(); + producer.write(frame(200_000, true)).unwrap(); + producer.write(frame(150_000, false)).unwrap(); + producer.write(frame(166_000, false)).unwrap(); + producer.write(frame(183_000, false)).unwrap(); + producer.write(frame(233_000, false)).unwrap(); + let _finished = producer.finish().unwrap(); + assert_eq!( + media(collect_payloads(consumer).await), + [vec![100_000, 166_000, 133_000], vec![200_000, 183_000, 233_000]] + ); + } + + /// After a cut there is no open group for a keyframe at or before the newest frame to continue, + /// so it is dropped, and the track waits for a keyframe past the newest frame. + #[tokio::test] + async fn a_keyframe_overlapping_a_closed_group_is_dropped() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); + producer.write(frame(100_000, true)).unwrap(); + producer.write(frame(166_000, false)).unwrap(); + producer.cut(None).unwrap(); + producer.write(frame(150_000, true)).unwrap(); + assert!(producer.needs_keyframe()); + let err = producer.write(frame(183_000, false)).unwrap_err(); + assert!(matches!(err, crate::Error::MissingKeyframe(_)), "{err:?}"); + producer.write(frame(200_000, true)).unwrap(); + let _finished = producer.finish().unwrap(); + assert_eq!( + media(collect_payloads(consumer).await), + [vec![100_000, 166_000], vec![200_000]] + ); } /// Group starts strictly increase, so a keyframe at the open group's start continues that group @@ -2175,17 +2309,6 @@ mod tests { assert_eq!(collect_starts(consumer).await, vec![100_000, 120_000]); } - /// With no group open, a keyframe at the previous group's start can't open one. - #[tokio::test] - async fn a_keyframe_at_a_closed_group_start_is_refused() { - let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); - let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Audio)); - producer.write(frame(100_000, true)).unwrap(); - producer.cut(None).unwrap(); - let err = producer.write(frame(100_000, true)).unwrap_err(); - assert!(matches!(err, crate::Error::TimestampRewind(_)), "{err:?}"); - } - /// A frame below the start of the group before its own reaches back past a whole group. #[tokio::test] async fn a_frame_below_the_previous_group_start_is_refused() { @@ -2226,6 +2349,96 @@ mod tests { assert_eq!(groups[1][1], (50_000, 2)); } + /// Random reordering, splices, and cuts: every frame of a group presents after every frame of + /// the groups before it, and a frame is dropped only when it presents at or before a frame + /// published ahead of it. + #[test] + fn groups_never_overlap_on_random_streams() { + use proptest::prelude::*; + use proptest::test_runner::{Config, TestRunner}; + + let mut runner = TestRunner::new(Config { + cases: 256, + failure_persistence: None, + ..Default::default() + }); + // Per frame: its offset from a 33ms cadence, whether it is a keyframe, and whether a cut + // precedes it. + let stream = proptest::collection::vec( + ( + -150_000i64..150_000, + proptest::bool::weighted(0.2), + proptest::bool::weighted(0.03), + ), + 1..120, + ); + runner + .run(&stream, |frames| { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)); + let mut accepted = Vec::new(); + for (index, (offset, keyframe, cut)) in frames.into_iter().enumerate() { + if cut { + producer.cut(None).unwrap(); + } + let timestamp = (1_000_000 + 33_000 * index as i64 + offset) as u64; + let written = producer.write(Frame { + timestamp: Timestamp::from_micros(timestamp).unwrap(), + payload: Bytes::copy_from_slice(&(index as u32).to_be_bytes()), + keyframe, + duration: None, + }); + if written.is_ok() { + accepted.push((index as u32, timestamp as u128)); + } + } + let _finished = producer.finish().unwrap(); + + let mut groups: Vec> = Vec::new(); + futures::executor::block_on(async { + while let Some(mut group) = consumer.recv_group().await.unwrap() { + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await.unwrap() { + let decoded = hang::container::Frame::decode(frame.payload).unwrap(); + if let Ok(index) = <[u8; 4]>::try_from(decoded.payload.as_ref()) { + frames.push((u32::from_be_bytes(index), decoded.timestamp.as_micros().unwrap())); + } + } + groups.push(frames); + } + }); + + let mut ceiling = None; + for group in &groups { + for &(_, timestamp) in group { + prop_assert!(ceiling.is_none_or(|ceiling| timestamp > ceiling), "{groups:?}"); + } + ceiling = group.iter().map(|&(_, timestamp)| timestamp).max().max(ceiling); + } + + // Published frames are the accepted ones in order, less those at or before a frame + // published ahead of them. + let published: Vec<(u32, u128)> = groups.concat(); + let mut next = published.iter().peekable(); + let mut newest = None; + for &(index, timestamp) in &accepted { + if next.peek().is_some_and(|&&(published, _)| published == index) { + next.next(); + newest = newest.max(Some(timestamp)); + } else { + prop_assert!( + newest.is_some_and(|newest| timestamp <= newest), + "frame {index} at {timestamp} was dropped above {newest:?}" + ); + } + } + prop_assert!(next.peek().is_none(), "published a frame never accepted"); + Ok(()) + }) + .unwrap(); + } + #[tokio::test] async fn duration_marker_resets_after_discontinuity() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 5507336b7c4..3c3f359a919 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -9456,22 +9456,21 @@ pub(super) mod test { assert!(is_rewind(&err), "{err:?}"); } - /// Group starts never go backwards, and no frame sits below the start of the group before - /// its own, which is what the producer demands. Frames may still present below the previous - /// group's content (B-frames, an overlapping keyframe). + /// Every frame of a group presents after every frame of the groups before it, which is what + /// the producer demands. Frames within a group may still reorder (B-frames, a join that + /// continues the group). fn assert_forward(frames: &[crate::container::Frame]) { - let (mut start, mut floor) = (None, None); + let (mut ceiling, mut newest) = (None, None); for frame in frames { let ts = frame.timestamp.as_micros().unwrap(); if frame.keyframe { - assert!( - start.is_none_or(|start| ts >= start), - "a group started at {ts} before {start:?}" - ); - floor = start; - start = Some(ts); + ceiling = newest; } - assert!(floor.is_none_or(|floor| ts >= floor), "rewound to {ts} below {floor:?}"); + assert!( + ceiling.is_none_or(|ceiling| ts > ceiling), + "{ts} presents at or before {ceiling:?}, an earlier group's newest frame" + ); + newest = newest.max(Some(ts)); } } @@ -10206,9 +10205,9 @@ pub(super) mod test { /// #3798's join: a new IDR presents below the last P-frame on a continuous clock, since /// the B-frames before it presented earlier than the P-frame decoded ahead of them. It - /// still starts after the previous group did, so it overlaps rather than rewinds. + /// continues the old content's group, so the next group still presents after all of it. #[tokio::test(start_paused = true)] - async fn h264_join_below_a_p_frame_publishes() { + async fn h264_join_below_a_p_frame_continues_the_group() { let mut mux = Mux { out: synth_pmt(&[(StreamType::H264, VIDEO)], false), ..Default::default() @@ -10218,6 +10217,44 @@ pub(super) mod test { mux.gops(VIDEO, 90_000 + 6 * FRAME, 2); let frames = video_frames(&mux.out).await; assert_eq!(frames.len(), 16, "every picture either side of the join"); + let groups = frames.iter().filter(|frame| frame.keyframe).count(); + assert_eq!(groups, 3, "the join's IDR continues the group it overlaps"); + assert_forward(&frames); + } + + /// A join whose IDR lands past the old content, followed by leading pictures: the one that + /// would present at or before the old content's newest frame is dropped, the rest publish. + #[tokio::test(start_paused = true)] + async fn h264_join_drops_a_leading_picture_before_the_old_content_ends() { + let mut mux = Mux { + out: synth_pmt(&[(StreamType::H264, VIDEO)], false), + ..Default::default() + }; + // The old content's newest frame presents at `base + 4 * FRAME` and decodes last at + // `base + 3 * FRAME`. + let base = 90_000 + 4 * FRAME; + mux.gops(VIDEO, 90_000, 2); + for (keyframe, pts, dts) in [ + (true, base + 7 * FRAME, base + 3 * FRAME + 1_000), + (false, base + 4 * FRAME, base + 3 * FRAME + 2_000), + (false, base + 6 * FRAME, base + 4 * FRAME), + (false, base + 9 * FRAME, base + 5 * FRAME), + (false, base + 8 * FRAME, base + 6 * FRAME), + ] { + let cc = mux.cc(VIDEO); + mux.out + .extend_from_slice(&video_pes(VIDEO, cc, pts, Some(dts), &annexb_au(keyframe))); + } + let frames = video_frames(&mux.out).await; + let micros = |ticks: u64| u128::from(ticks) * 1_000_000 / 90_000; + let joined: Vec = frames[8..] + .iter() + .map(|frame| frame.timestamp.as_micros().unwrap()) + .collect(); + assert_eq!( + joined, + [base + 7 * FRAME, base + 6 * FRAME, base + 9 * FRAME, base + 8 * FRAME].map(micros) + ); assert_forward(&frames); } From 1022a9a6af89c9d09ad44e58ed61bb0463ab9765 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 19:21:54 -0800 Subject: [PATCH 3/6] fix(mux): trim a passthrough fragment's overlapping samples, and drop what decodes from a dropped keyframe The fMP4 passthrough drops only the samples of a fragment that present with an earlier group, rewriting it as the edit list does, instead of the whole fragment. A keyframe dropped with no group open takes the frames decoding from it along until a keyframe opens a group, in Rust and JS, rather than refusing them with MissingKeyframe. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 4 +- js/hang/src/container/legacy.ts | 15 ++- rs/moq-mux/src/container/fmp4/import.rs | 118 +++++++++++++------ rs/moq-mux/src/container/fmp4/import_test.rs | 42 ++++--- rs/moq-mux/src/container/producer.rs | 34 ++++-- 5 files changed, 149 insertions(+), 64 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 50bb3b03ee4..b61ef678926 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -234,10 +234,8 @@ test("Legacy Producer drops a keyframe overlapping a closed group", () => { producer.discontinuity(Duration.micros(200_000)); const largest = track.largest()?.group; producer.encode(new Uint8Array([1]), Duration.micros(150_000), true); + producer.encode(new Uint8Array([1]), Duration.micros(183_000), false); expect(track.largest()?.group).toBe(largest); - expect(() => producer.encode(new Uint8Array([1]), Duration.micros(183_000), false)).toThrow( - "must start with a keyframe", - ); producer.finish(); }); diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index e6bd00b7bf0..3d3ab5757cf 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -153,6 +153,9 @@ export class Producer { // The newest frame of every group before the current one: every frame of the current group // presents after it. #ceiling?: number; + // A keyframe was dropped with no group open, so the frames decoding from it go too, until a + // keyframe opens a group. + #skipping = false; // Gap between consecutive timestamps, used to close the last group when no successor exists. #interval?: number; // A discontinuity's marker is the newest group, so another one would say nothing new. @@ -171,8 +174,9 @@ export class Producer { * every frame of the groups before it, so a keyframe that doesn't start after the newest frame * continues the open group (a splice), and a later frame presenting at or before the groups before * its own (a leading picture after a splice) is dropped, as is a keyframe at or before the newest - * frame with no group open. Throws if the first frame is not a keyframe, a group start goes - * backwards, or a frame is below the previous group start. + * frame with no group open, with the frames after it until a keyframe opens a group. Throws if the + * first frame is not a keyframe, a group start goes backwards, or a frame is below the previous + * group start. */ encode(data: Uint8Array | Source, pts: Duration, keyframe: boolean) { const timestamp = micros(pts); @@ -189,7 +193,11 @@ export class Producer { const opens = keyframe && !behind && startsPast(this.#track, timestamp); // Dropped: a keyframe with no group to continue, and a frame presenting with the groups // before its own, such as a leading picture after a splice. - if (keyframe && behind && !this.#group) return; + if (keyframe && behind && !this.#group) { + this.#skipping = true; + return; + } + if (!keyframe && !this.#group && this.#skipping) return; if (keyframe && !opens && !this.#group) throw new Error("a keyframe must start after the previous group"); if (!opens && this.#group && this.#ceiling !== undefined && timestamp <= this.#ceiling) return; this.#marked = false; @@ -203,6 +211,7 @@ export class Producer { this.#group = this.#track.appendGroup(Time.Timestamp.fromMicros(timestamp)); this.#floor = this.#start; this.#ceiling = this.#newest; + this.#skipping = false; this.#start = timestamp; } else if (!this.#group) { throw new Error("must start with a keyframe"); diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 4f17b52aed4..cf9f40155be 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -1156,7 +1156,7 @@ impl Import { None => return Err(Error::UnknownTrack(id).into()), }; // A fragment wholly before the edit or the first sync sample publishes nothing. - let Some(run) = track.run(traf, &mut resolver)? else { + let Some(mut run) = track.run(traf, &mut resolver)? else { continue; }; // The moov declared every track, so any track's anchored fragment releases the catalog. @@ -1166,6 +1166,15 @@ impl Import { offset, new_segment: track.segment != Some(self.segment), }; + if let Output::Passthrough(output) = &mut track.output + && !output.trim(&mut run, placement, track.kind)? + { + // With no group open, video waits for a sync sample past what was published. + if output.group.is_none() && track.kind == Kind::Video { + track.orphaned = true; + } + continue; + } let prefix = match track.output { Output::Passthrough(_) => Some(fragment(traf, &moof, &run, &track.defaults)?), _ => None, @@ -1239,7 +1248,7 @@ impl Import { let prefix = pending.prefix.take().expect("one prefix per fragment"); let (writer, opened) = output.start(prefix, &pending.run, track.kind, pending.placement)?; - (writer.map_or(Writer::Dropped, Writer::Fragment), opened) + (Writer::Fragment(writer), opened) } output => { let (sample, opened) = open_sample(output, pending, *sample_index, track.timescale)?; @@ -1256,11 +1265,12 @@ impl Import { if complete { let last = *sample_index + 1 == pending.run.samples.len(); match &mut track.output { - Output::Passthrough(output) if last => match pending.writer.take() { - Some(Writer::Fragment(writer)) => output.complete(writer, pending, track.kind)?, - Some(Writer::Dropped) => {} - _ => unreachable!("passthrough tracks write whole fragments"), - }, + Output::Passthrough(output) if last => { + let Some(Writer::Fragment(writer)) = pending.writer.take() else { + unreachable!("passthrough tracks write whole fragments"); + }; + output.complete(writer, pending, track.kind)?; + } Output::Passthrough(_) => {} output => media!(output, producer => { let Some(Writer::Sample(sample)) = pending.writer.take() else { @@ -1320,7 +1330,7 @@ impl Mdat { let _ = writer.abort(err.clone()); } Some(Writer::Sample(sample)) => sample.abort(err.clone()), - Some(Writer::Dropped) | None => {} + None => {} } } } @@ -1340,8 +1350,6 @@ struct PendingTrack { enum Writer { /// A passthrough track's whole per-track fragment. Fragment(moq_net::frame::Producer), - /// A passthrough fragment the track dropped, whose payload is discarded. - Dropped, /// One unwrapped sample. Sample(Streamed), } @@ -1350,7 +1358,6 @@ impl Writer { fn write(&mut self, chunk: &[u8]) -> crate::Result<()> { match self { Self::Fragment(writer) => writer.write(chunk)?, - Self::Dropped => {} Self::Sample(sample) => sample.write(chunk)?, } Ok(()) @@ -1593,15 +1600,74 @@ fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { } impl Passthrough { - /// Open one per-track fragment's output frame, reporting whether it opened a group, or `None` - /// for a fragment dropped because it presents with the groups before its own. + /// Drop the samples of `run` that would present at or before the groups before the one its + /// fragment lands in, such as leading pictures after a splice, counting them as skipped: every + /// sample of a group presents after every sample of the groups before it. A fragment that would + /// start a group at or before the newest sample continues the open group, as + /// `container::Producer` does for a keyframe; with none open it is dropped whole. Returns whether + /// any sample is left to publish. + fn trim(&mut self, run: &mut Run, placement: Placement, kind: Kind) -> Result { + let first = run.samples.first().ok_or(Error::MissingTrun)?; + let keyframe = placement.keyframe(kind, first); + let at = placement.offset.apply(first.timestamp)?; + let start_group = self.group.is_none() || (placement.new_segment && keyframe); + // A group below the start of the one before it is a restart, which `start` refuses at its + // first sample, as the unwrapped track does. + let earliest = placement.offset.apply(run.earliest)?; + if start_group && self.start.is_some_and(|floor| earliest < floor) { + return Ok(true); + } + let behind = self.newest.is_some_and(|newest| at <= newest); + let ceiling = match (start_group, behind) { + (true, false) => self.newest, + (true, true) if self.group.is_none() => { + tracing::debug!(timestamp = ?at, "dropping a fragment with no group to continue"); + self.sent.skip(run.samples.len() as u64); + return Ok(false); + } + _ => self.ceiling, + }; + let Some(ceiling) = ceiling else { + return Ok(true); + }; + let before = run.samples.len(); + let mut kept = Vec::with_capacity(before); + for sample in std::mem::take(&mut run.samples) { + if placement.offset.apply(sample.timestamp)? > ceiling { + kept.push(sample); + } + } + run.samples = kept; + let dropped = before - run.samples.len(); + if dropped == 0 { + return Ok(true); + } + tracing::debug!(dropped, "dropping samples that present before an earlier group ends"); + self.sent.skip(dropped as u64); + run.dropped = true; + let mut timestamps = run.samples.iter().map(|sample| sample.timestamp); + let Some(first) = timestamps.next() else { + return Ok(false); + }; + let (earliest, latest) = timestamps.fold((first, first), |(min, max), timestamp| { + ( + crate::container::earlier(min, timestamp), + crate::container::later(max, timestamp), + ) + }); + run.earliest = earliest; + run.latest = latest; + Ok(true) + } + + /// Open one per-track fragment's output frame, reporting whether it opened a group. fn start( &mut self, prefix: Bytes, run: &Run, kind: Kind, placement: Placement, - ) -> Result<(Option, bool)> { + ) -> Result<(moq_net::frame::Producer, bool)> { // Carry the fragment's earliest presentation time as the frame timestamp, in the // track's native timescale and shifted onto the catalog clock. Consumers present the // fragment at it; `tfdt` only places the samples relative to each other. @@ -1619,27 +1685,11 @@ impl Passthrough { { return Err(crate::container::TimestampRewind { timestamp, floor }.into()); } - // Every sample of a group presents after every sample of the groups before it, so a segment - // that doesn't start after the newest sample written continues the open group, as - // `container::Producer` does for a keyframe; with none open it is dropped. So is a later - // fragment presenting with the groups before its own, such as a leading picture after a - // splice, its samples counted as skipped. - let behind = self.newest.is_some_and(|newest| timestamp <= newest); - if start_group && behind && self.group.is_some() { + // A segment that doesn't start after the newest sample written continues the open group; + // `trim` dropped one with none open. + if start_group && self.newest.is_some_and(|newest| timestamp <= newest) { start_group = false; } - let dropped = match start_group { - true => behind, - false => self.ceiling.is_some_and(|ceiling| timestamp <= ceiling), - }; - if dropped { - tracing::debug!( - ?timestamp, - "dropping a fragment that presents before its group's predecessors end" - ); - self.sent.skip(run.samples.len() as u64); - return Ok((None, false)); - } // A group is keyed by its first fragment's timestamp, which has to start after every group // the track holds. A segment that doesn't continues the open group instead; with none open // it is refused. @@ -1690,7 +1740,7 @@ impl Passthrough { self.newest .map_or(latest, |newest| crate::container::later(newest, latest)), ); - Ok((Some(writer), start_group)) + Ok((writer, start_group)) } fn complete(&mut self, writer: moq_net::frame::Producer, pending: &PendingTrack, kind: Kind) -> Result<()> { diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index b191a650f2c..d338f745279 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1405,7 +1405,8 @@ async fn video_joined_mid_gop_starts_at_a_sync_sample() { /// Every sample of a group presents after every sample of the groups before it, passthrough or /// unwrapped: a sync sample that doesn't start after the newest sample continues the open group, -/// and a later one presenting at or before the groups before its own is dropped. +/// a later one presenting at or before the groups before its own is dropped, and with no group open +/// to continue, a sync sample at or before the newest is dropped with the samples decoding from it. #[tokio::test] async fn video_groups_never_overlap() { let (_, (video_id, video_scale), _) = bbb_init(); @@ -1419,7 +1420,8 @@ async fn video_groups_never_overlap() { (0, &[(2, 0), (5, non_sync), (1, non_sync), (1, non_sync)]), // A join at 5, below the newest sample: it continues the group. Presents 5, 9, 7, 8. (4, &[(1, 0), (4, non_sync), (1, non_sync), (1, non_sync)]), - // A sync sample past the newest, with a leading picture at it. Presents 10, 9, 15, 11. + // A sync sample past the newest, with a leading picture at it beside pictures that are + // kept. Presents 10, 9, 15, 11. (8, &[(2, 0), (0, non_sync), (5, non_sync), (0, non_sync)]), // Past the newest again. Presents 16. (12, &[(4, 0)]), @@ -1428,26 +1430,38 @@ async fn video_groups_never_overlap() { // Presents 17. (14, &[(3, non_sync)]), ]; - let cases = [ - // Unwrapped, every sample is placed on its own: the leading picture at 9 is dropped. - ( - false, - vec![vec![2, 6, 3, 4, 5, 9, 7, 8], vec![10, 15, 11], vec![16, 17]], - ), - // Passthrough places whole fragments by their earliest sample, so the one with the leading - // picture continues the group instead. - (true, vec![vec![2, 6, 3, 4, 5, 9, 7, 8, 10, 9, 15, 11], vec![16, 17]]), + // After a rejoin closes the group, `(dts, samples)` the same way. + let rejoined: [(u64, &[(u64, u32)]); 3] = [ + // A sync sample at 17, the newest, with no group to continue: dropped. + (15, &[(2, 0)]), + // Presents 18, decoding from the dropped sync sample: dropped. + (16, &[(2, non_sync)]), + // Presents 20, 21. + (17, &[(3, 0), (3, non_sync)]), ]; - for (passthrough, expected) in cases { + let expected = [ + vec![2, 6, 3, 4, 5, 9, 7, 8], + vec![10, 15, 11], + vec![16, 17], + vec![20, 21], + ]; + for passthrough in [false, true] { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_cmaf(passthrough); fmp4.decode(&init).unwrap(); - for (dts, samples) in fragments { + let decode = |fmp4: &mut crate::container::fmp4::Import, (dts, samples): (u64, &[(u64, u32)])| { let entries: Vec<_> = samples.iter().map(|&(cts, flags)| (cts * tick, flags, 0xA)).collect(); fmp4.decode(&video_fragment(video_id, dts * tick, tick, &entries)) .unwrap(); + }; + for fragment in fragments { + decode(&mut fmp4, fragment); + } + fmp4.rejoin().unwrap(); + for fragment in rejoined { + decode(&mut fmp4, fragment); } let _finished = fmp4.finish().unwrap(); let [video, _] = published(&consumer, &catalog).await; @@ -1456,7 +1470,7 @@ async fn video_groups_never_overlap() { .map(|(timestamp, _, _)| (timestamp * u128::from(video_scale.as_u64())).div_ceil(1_000_000) as u64 / tick) .collect(); assert_eq!(presented, expected.concat(), "passthrough {passthrough}"); - // Each group starts at its first sample, the lowest of its sync samples. + // Each group starts at its first sample. let name = catalog.snapshot().video.renditions.keys().next().unwrap().clone(); let starts: Vec = starts(&consumer, &name) .await diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index e62627333f1..630644df644 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -90,6 +90,9 @@ pub struct Producer { /// The newest frame of every group before the latest one: every frame of the latest group /// presents after it. ceiling: Option, + /// A keyframe was dropped with no group open, so the frames decoding from it are dropped too, + /// until a keyframe opens a group. + skipping: bool, /// Duration of the frame that last raised [`end`](Self::end), if it had one. Distinguishes /// an exclusive presentation point from a max timestamp that still needs an estimate. @@ -309,6 +312,7 @@ impl Producer { start: None, floor: None, ceiling: None, + skipping: false, last_duration: None, previous_timestamp: None, cadence: None, @@ -349,6 +353,7 @@ where start: None, floor: None, ceiling: None, + skipping: false, last_duration: None, previous_timestamp: None, cadence: None, @@ -647,7 +652,8 @@ where /// content's last frames. A later frame presenting at or before the previous groups' newest, /// such as a leading picture after a splice, is dropped and counted as /// [skipped](Self::report_skipped), and so is a keyframe at or before the newest frame with no - /// group open, such as after a [`cut`](Self::cut). + /// group open, such as after a [`cut`](Self::cut), with the frames after it until a keyframe + /// opens a group. /// /// Group starts never go backwards: a keyframe below the start of the latest group, or any /// frame below the start of the group before its own, returns @@ -711,7 +717,10 @@ where { return match self.group { Some(_) => Ok(true), - None => Ok(self.drop_frame(timestamp)), + None => { + self.skipping = true; + Ok(self.drop_frame(timestamp)) + } }; } @@ -737,6 +746,9 @@ where // Start a new group if needed; the first frame of a group must be a keyframe. if self.group.is_none() { if !keyframe { + if self.skipping { + return Ok(self.drop_frame(timestamp)); + } // No group yet and this delta can't anchor one. The caller (e.g. a // mid-stream join) decides whether to skip until the first keyframe. return Err(super::MissingKeyframe.into()); @@ -756,6 +768,7 @@ where self.floor = self.start; self.ceiling = self.newest; + self.skipping = false; self.start = Some(timestamp); self.group = Some(group); } else if self.ceiling.is_some_and(|ceiling| !timestamp_lt(ceiling, timestamp)) { @@ -2274,7 +2287,7 @@ mod tests { } /// After a cut there is no open group for a keyframe at or before the newest frame to continue, - /// so it is dropped, and the track waits for a keyframe past the newest frame. + /// so it is dropped with the frames decoding from it, until a keyframe past the newest frame. #[tokio::test] async fn a_keyframe_overlapping_a_closed_group_is_dropped() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); @@ -2285,8 +2298,7 @@ mod tests { producer.cut(None).unwrap(); producer.write(frame(150_000, true)).unwrap(); assert!(producer.needs_keyframe()); - let err = producer.write(frame(183_000, false)).unwrap_err(); - assert!(matches!(err, crate::Error::MissingKeyframe(_)), "{err:?}"); + producer.write(frame(183_000, false)).unwrap(); producer.write(frame(200_000, true)).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!( @@ -2390,7 +2402,7 @@ mod tests { duration: None, }); if written.is_ok() { - accepted.push((index as u32, timestamp as u128)); + accepted.push((index as u32, timestamp as u128, keyframe)); } } let _finished = producer.finish().unwrap(); @@ -2418,19 +2430,21 @@ mod tests { } // Published frames are the accepted ones in order, less those at or before a frame - // published ahead of them. + // published ahead of them, and those after a dropped keyframe until one is published. let published: Vec<(u32, u128)> = groups.concat(); let mut next = published.iter().peekable(); - let mut newest = None; - for &(index, timestamp) in &accepted { + let (mut newest, mut skipping) = (None, false); + for &(index, timestamp, keyframe) in &accepted { if next.peek().is_some_and(|&&(published, _)| published == index) { next.next(); newest = newest.max(Some(timestamp)); + skipping &= !keyframe; } else { prop_assert!( - newest.is_some_and(|newest| timestamp <= newest), + newest.is_some_and(|newest| timestamp <= newest) || (skipping && !keyframe), "frame {index} at {timestamp} was dropped above {newest:?}" ); + skipping |= keyframe; } } prop_assert!(next.peek().is_none(), "published a frame never accepted"); From e7aca32f2a44a527f66698802e4e1f3f5c3fd8d0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 19:41:43 -0800 Subject: [PATCH 4/6] fix(mux): plan where each passthrough fragment lands before any is written Passthrough::plan advances the track's planned group state (open, start, key, newest, ceiling) per traf, so a moof with several runs of one track trims each against the group it actually enters. start() only executes the plan. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import.rs | 286 +++++++++++-------- rs/moq-mux/src/container/fmp4/import_test.rs | 91 ++++-- rs/moq-mux/src/start.rs | 5 + 3 files changed, 236 insertions(+), 146 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index cf9f40155be..e67212edcb7 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -159,9 +159,16 @@ struct Passthrough { track: moq_net::track::Producer, group: Option, + + // Where the fragments planned so far leave the track, which [`Passthrough::plan`] advances + // before they are written, so each fragment of a moof is planned after the ones before it. + /// A group is open, or a planned fragment opens one. + open: bool, /// The latest group's start, which the next group may not start below. start: Option, - /// The newest sample written, which the next group starts after. + /// The latest group's key, which the next group's has to pass. + key: Option, + /// The newest sample, which the next group starts after. newest: Option, /// The newest sample of every group before the latest one: every sample of the latest group /// presents after it. @@ -199,6 +206,7 @@ impl Passthrough { /// Close the open group, so the next fragment opens one at `pending_start` or its own timestamp. fn close(&mut self) -> crate::Result<()> { + self.open = false; self.estimator.cut(None); self.publish_estimate()?; if let Some(group) = self.group.take() { @@ -644,7 +652,9 @@ impl Import { rendition, track, group: None, + open: false, start: None, + key: None, newest: None, ceiling: None, member, @@ -1166,15 +1176,19 @@ impl Import { offset, new_segment: track.segment != Some(self.segment), }; - if let Output::Passthrough(output) = &mut track.output - && !output.trim(&mut run, placement, track.kind)? - { - // With no group open, video waits for a sync sample past what was published. - if output.group.is_none() && track.kind == Kind::Video { - track.orphaned = true; - } - continue; - } + let land = match &mut track.output { + Output::Passthrough(output) => match output.plan(&mut run, placement, track.kind)? { + Some(land) => Some(land), + None => { + // With no group open, video waits for a sync sample past what was published. + if !output.open && track.kind == Kind::Video { + track.orphaned = true; + } + continue; + } + }, + _ => None, + }; let prefix = match track.output { Output::Passthrough(_) => Some(fragment(traf, &moof, &run, &track.defaults)?), _ => None, @@ -1193,6 +1207,7 @@ impl Import { id, run, placement, + land, prefix, writer: None, opened: false, @@ -1246,8 +1261,8 @@ impl Import { let (writer, opened) = match &mut track.output { Output::Passthrough(output) => { let prefix = pending.prefix.take().expect("one prefix per fragment"); - let (writer, opened) = - output.start(prefix, &pending.run, track.kind, pending.placement)?; + let land = pending.land.expect("a planned passthrough fragment"); + let (writer, opened) = output.start(prefix, &pending.run, pending.placement, land)?; (Writer::Fragment(writer), opened) } output => { @@ -1341,6 +1356,8 @@ struct PendingTrack { id: u32, run: Run, placement: Placement, + /// Where a passthrough fragment lands, as planned. + land: Option, prefix: Option, writer: Option, opened: bool, @@ -1376,6 +1393,15 @@ enum ChunkOutput { Text(usize), } +/// Where [`Passthrough::plan`] lands a fragment in its track's groups. +#[derive(Clone, Copy)] +enum Land { + /// In the open group. + Continues, + /// In a new group, keyed `start` when a seek named one, else by its own timestamp. + Opens { start: Option }, +} + /// Where a fragment lands on its track. #[derive(Clone, Copy)] struct Placement { @@ -1600,128 +1626,148 @@ fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { } impl Passthrough { - /// Drop the samples of `run` that would present at or before the groups before the one its - /// fragment lands in, such as leading pictures after a splice, counting them as skipped: every - /// sample of a group presents after every sample of the groups before it. A fragment that would - /// start a group at or before the newest sample continues the open group, as - /// `container::Producer` does for a keyframe; with none open it is dropped whole. Returns whether - /// any sample is left to publish. - fn trim(&mut self, run: &mut Run, placement: Placement, kind: Kind) -> Result { + /// Plan where `run`'s fragment lands, advancing the planned state past it. + /// + /// Every sample of a group presents after every sample of the groups before it. A fragment + /// that would start a group at or before the newest sample continues the open group, as + /// `container::Producer` does for a keyframe, and with none open it is dropped whole. The + /// samples presenting at or before the groups before the one the fragment lands in, such as + /// leading pictures after a splice, are dropped from `run`. Dropped samples count as skipped. + /// Returns `None` when nothing is left to publish. + fn plan(&mut self, run: &mut Run, placement: Placement, kind: Kind) -> Result> { let first = run.samples.first().ok_or(Error::MissingTrun)?; + // Audio follows segment boundaries; video also waits for a sync first sample so a new + // group's first fragment can be decoded independently. let keyframe = placement.keyframe(kind, first); let at = placement.offset.apply(first.timestamp)?; - let start_group = self.group.is_none() || (placement.new_segment && keyframe); - // A group below the start of the one before it is a restart, which `start` refuses at its - // first sample, as the unwrapped track does. let earliest = placement.offset.apply(run.earliest)?; - if start_group && self.start.is_some_and(|floor| earliest < floor) { - return Ok(true); + let mut opens = !self.open || (placement.new_segment && keyframe); + // A group may not start below the one before it: a restart, refused before anything of it + // publishes, as an unwrapped track does. + if opens + && let Some(floor) = self.start + && earliest < floor + { + return Err(crate::container::TimestampRewind { + timestamp: earliest, + floor, + } + .into()); } - let behind = self.newest.is_some_and(|newest| at <= newest); - let ceiling = match (start_group, behind) { - (true, false) => self.newest, - (true, true) if self.group.is_none() => { + if opens && self.newest.is_some_and(|newest| at <= newest) { + if !self.open { tracing::debug!(timestamp = ?at, "dropping a fragment with no group to continue"); self.sent.skip(run.samples.len() as u64); - return Ok(false); + return Ok(None); } - _ => self.ceiling, - }; - let Some(ceiling) = ceiling else { - return Ok(true); + opens = false; + } + + let ceiling = match opens { + true => self.newest, + false => self.ceiling, }; - let before = run.samples.len(); - let mut kept = Vec::with_capacity(before); - for sample in std::mem::take(&mut run.samples) { - if placement.offset.apply(sample.timestamp)? > ceiling { - kept.push(sample); + if let Some(ceiling) = ceiling { + let before = run.samples.len(); + let mut kept = Vec::with_capacity(before); + for sample in std::mem::take(&mut run.samples) { + if placement.offset.apply(sample.timestamp)? > ceiling { + kept.push(sample); + } + } + run.samples = kept; + let dropped = before - run.samples.len(); + if dropped > 0 { + tracing::debug!(dropped, "dropping samples that present before an earlier group ends"); + self.sent.skip(dropped as u64); + run.dropped = true; + let mut timestamps = run.samples.iter().map(|sample| sample.timestamp); + let Some(first) = timestamps.next() else { + return Ok(None); + }; + let (earliest, latest) = timestamps.fold((first, first), |(min, max), timestamp| { + ( + crate::container::earlier(min, timestamp), + crate::container::later(max, timestamp), + ) + }); + run.earliest = earliest; + run.latest = latest; } } - run.samples = kept; - let dropped = before - run.samples.len(); - if dropped == 0 { - return Ok(true); - } - tracing::debug!(dropped, "dropping samples that present before an earlier group ends"); - self.sent.skip(dropped as u64); - run.dropped = true; - let mut timestamps = run.samples.iter().map(|sample| sample.timestamp); - let Some(first) = timestamps.next() else { - return Ok(false); + + let earliest = placement.offset.apply(run.earliest)?; + let land = match opens { + // A group is keyed by its first fragment's timestamp, or a seek's start, which has to + // pass the latest group's. A segment that doesn't continues the open group instead; with + // none open it is refused. + true => match self.pending_start.take() { + Some(start) => { + self.key = Some(start); + Land::Opens { start: Some(start) } + } + None => { + let key = crate::start::key(&self.track, earliest)?; + match self.key { + Some(latest) if key <= latest && self.open => Land::Continues, + Some(latest) if key <= latest => { + let floor = self.start.unwrap_or(earliest); + return Err(crate::container::TimestampRewind { + timestamp: earliest, + floor, + } + .into()); + } + _ => { + self.key = Some(key); + Land::Opens { start: None } + } + } + } + }, + false => Land::Continues, }; - let (earliest, latest) = timestamps.fold((first, first), |(min, max), timestamp| { - ( - crate::container::earlier(min, timestamp), - crate::container::later(max, timestamp), - ) - }); - run.earliest = earliest; - run.latest = latest; - Ok(true) + if let Land::Opens { .. } = land { + self.start = Some(earliest); + self.ceiling = self.newest; + } + self.open = true; + let latest = placement.offset.apply(run.latest)?; + self.newest = Some( + self.newest + .map_or(latest, |newest| crate::container::later(newest, latest)), + ); + Ok(Some(land)) } - /// Open one per-track fragment's output frame, reporting whether it opened a group. + /// Open one per-track fragment's output frame where [`plan`](Self::plan) landed it, reporting + /// whether it opened a group. fn start( &mut self, prefix: Bytes, run: &Run, - kind: Kind, placement: Placement, + land: Land, ) -> Result<(moq_net::frame::Producer, bool)> { // Carry the fragment's earliest presentation time as the frame timestamp, in the // track's native timescale and shifted onto the catalog clock. Consumers present the // fragment at it; `tfdt` only places the samples relative to each other. let timestamp = placement.offset.apply(run.earliest)?; - - // Audio follows segment boundaries; video also waits for a sync first sample so a new - // group's first fragment can be decoded independently. - let keyframe = placement.keyframe(kind, run.samples.first().ok_or(Error::MissingTrun)?); - let mut start_group = self.group.is_none() || (placement.new_segment && keyframe); - // A group may not start below the one before it. Refused before the open group closes or - // a new one opens, so a rewound fragment publishes nothing, as an unwrapped track does. - if start_group - && let Some(floor) = self.start - && timestamp < floor - { - return Err(crate::container::TimestampRewind { timestamp, floor }.into()); - } - // A segment that doesn't start after the newest sample written continues the open group; - // `trim` dropped one with none open. - if start_group && self.newest.is_some_and(|newest| timestamp <= newest) { - start_group = false; - } - // A group is keyed by its first fragment's timestamp, which has to start after every group - // the track holds. A segment that doesn't continues the open group instead; with none open - // it is refused. - let start = match start_group && self.pending_start.is_none() { - true => match (crate::start::opens(&self.track, timestamp)?, &self.group) { - (Some(start), _) => Some(start), - (None, Some(_)) => { - start_group = false; - None + let group = match land { + Land::Opens { start } => { + if let Some(prev) = self.group.take() { + prev.finish()?; + } + // Close the previous group for the bitrate estimator (used only when the + // descriptor didn't declare a bitrate). + self.estimator.cut(Some(timestamp)); + match start { + Some(start) => self.track.create_group(moq_net::group::Info { start })?, + None => self.track.append_group(timestamp)?, } - (None, None) => Some(crate::start::after(&self.track, timestamp)?), - }, - false => None, - }; - let group = if start_group { - if let Some(prev) = self.group.take() { - prev.finish()?; - } - match (self.pending_start.take(), start) { - (Some(start), _) => self.track.create_group(moq_net::group::Info { start })?, - (None, Some(_)) => self.track.append_group(timestamp)?, - (None, None) => unreachable!("a new group without a pending start has one"), } - } else { - self.group.take().ok_or(Error::NoKeyframe)? + Land::Continues => self.group.take().ok_or(Error::NoKeyframe)?, }; - - if start_group { - // Close the previous group for the bitrate estimator (used only when the - // descriptor didn't declare a bitrate). - self.estimator.cut(Some(timestamp)); - } let size = prefix .len() .checked_add(run.samples.iter().map(|sample| sample.range.len()).sum()) @@ -1731,16 +1777,7 @@ impl Passthrough { size: size as u64, })?; writer.write(prefix)?; - if start_group { - self.start = Some(timestamp); - self.ceiling = self.newest; - } - let latest = placement.offset.apply(run.latest)?; - self.newest = Some( - self.newest - .map_or(latest, |newest| crate::container::later(newest, latest)), - ); - Ok((writer, start_group)) + Ok((writer, matches!(land, Land::Opens { .. }))) } fn complete(&mut self, writer: moq_net::frame::Producer, pending: &PendingTrack, kind: Kind) -> Result<()> { @@ -2604,12 +2641,17 @@ mod tests { .decode(&media_header(id, 48_000, &[0], SIZE as u32, Some(SIZE))) .unwrap(); import.decode(&[0xA5; SIZE]).unwrap(); - // A new timeline that restarts below the published one is refused at its first sample. + // A new timeline that restarts below the published one is refused at its first sample, + // or at its header by a passthrough track, which plans where its fragment lands. import.discontinuity().unwrap(); - import - .decode(&media_header(id, 0, &[0], SIZE as u32, Some(SIZE))) - .unwrap(); - let err = import.decode(&[0xA5; SIZE / 2]).unwrap_err(); + let header = import.decode(&media_header(id, 0, &[0], SIZE as u32, Some(SIZE))); + let err = match passthrough { + true => header.unwrap_err(), + false => { + header.unwrap(); + import.decode(&[0xA5; SIZE / 2]).unwrap_err() + } + }; assert_eq!( import.decode(&[0xA5; SIZE / 2]).unwrap_err().to_string(), err.to_string() diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index d338f745279..4ffbe9814b3 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1248,36 +1248,65 @@ async fn audio_presents_from_its_edit() { /// One video fragment decoding from `dts`, a sample per `(cts, flags, byte)` lasting `tick`. fn video_fragment(track_id: u32, dts: u64, tick: u64, entries: &[(u64, u32, u8)]) -> Vec { - let moof = mp4_atom::Moof { + video_moof(track_id, tick, &[(dts, entries)]) +} + +/// A video run decoding from its `dts`, a sample per `(cts, flags, byte)`. +type VideoRun<'a> = (u64, &'a [(u64, u32, u8)]); + +/// One moof of a traf per run on `track_id`, each sample lasting `tick`, and their samples back to +/// back in one mdat. +fn video_moof(track_id: u32, tick: u64, trafs: &[VideoRun]) -> Vec { + let mut moof = mp4_atom::Moof { mfhd: mp4_atom::Mfhd { sequence_number: 1 }, - traf: vec![mp4_atom::Traf { - tfhd: mp4_atom::Tfhd { - track_id, + traf: trafs + .iter() + .map(|&(dts, entries)| mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id, + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: dts, + }), + trun: vec![mp4_atom::Trun { + data_offset: None, + samples: entries + .iter() + .map(|&(cts, flags, _)| mp4_atom::TrunEntry { + duration: Some(tick as u32), + size: Some(2), + flags: Some(flags), + cts: Some(cts as i64), + }) + .collect::>() + .into(), + }], ..Default::default() - }, - tfdt: Some(mp4_atom::Tfdt { - base_media_decode_time: dts, - }), - trun: vec![mp4_atom::Trun { - data_offset: None, - samples: entries - .iter() - .map(|&(cts, flags, _)| mp4_atom::TrunEntry { - duration: Some(tick as u32), - size: Some(2), - flags: Some(flags), - cts: Some(cts as i64), - }) - .collect::>() - .into(), - }], - ..Default::default() - }], + }) + .collect(), }; + // A traf after the first starts at its own offset, counted from the moof's first byte. + if trafs.len() > 1 { + for traf in &mut moof.traf { + traf.trun[0].data_offset = Some(0); + } + let mut measured = Vec::new(); + moof.encode(&mut measured).unwrap(); + // The mdat's header is 8 bytes, and every sample 2. + let mut offset = measured.len() + 8; + for (traf, (_, entries)) in moof.traf.iter_mut().zip(trafs) { + traf.trun[0].data_offset = Some(offset as i32); + offset += 2 * entries.len(); + } + } let mut fragment = Vec::new(); moof.encode(&mut fragment).unwrap(); mp4_atom::Mdat { - data: entries.iter().flat_map(|&(_, _, byte)| [byte; 2]).collect(), + data: trafs + .iter() + .flat_map(|(_, entries)| entries.iter().flat_map(|&(_, _, byte)| [byte; 2])) + .collect(), } .encode(&mut fragment) .unwrap(); @@ -1439,11 +1468,15 @@ async fn video_groups_never_overlap() { // Presents 20, 21. (17, &[(3, 0), (3, non_sync)]), ]; + // One moof of two runs: a sync sample at 23 opening a group, then a leading picture at 21, + // presenting with the group before, and 25. + let split: [(u64, &[(u64, u32)]); 2] = [(19, &[(4, 0)]), (20, &[(1, non_sync), (4, non_sync)])]; let expected = [ vec![2, 6, 3, 4, 5, 9, 7, 8], vec![10, 15, 11], vec![16, 17], vec![20, 21], + vec![23, 25], ]; for passthrough in [false, true] { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -1463,6 +1496,16 @@ async fn video_groups_never_overlap() { for fragment in rejoined { decode(&mut fmp4, fragment); } + let entries: Vec> = split + .iter() + .map(|(_, samples)| samples.iter().map(|&(cts, flags)| (cts * tick, flags, 0xA)).collect()) + .collect(); + let trafs: Vec<_> = split + .iter() + .zip(&entries) + .map(|(&(dts, _), entries)| (dts * tick, entries.as_slice())) + .collect(); + fmp4.decode(&video_moof(video_id, tick, &trafs)).unwrap(); let _finished = fmp4.finish().unwrap(); let [video, _] = published(&consumer, &catalog).await; let presented: Vec = video diff --git a/rs/moq-mux/src/start.rs b/rs/moq-mux/src/start.rs index 2a274630a05..380568c2a69 100644 --- a/rs/moq-mux/src/start.rs +++ b/rs/moq-mux/src/start.rs @@ -62,6 +62,11 @@ pub(crate) fn opens(track: &moq_net::track::Producer, timestamp: moq_net::Timest .then_some(start.value())) } +/// The key of a group starting at `timestamp` on `track`: `timestamp` in the track's timescale. +pub(crate) fn key(track: &moq_net::track::Producer, timestamp: moq_net::Timestamp) -> crate::Result { + Ok(of(track, timestamp)?.0.value()) +} + /// [`opens`], refusing a start that doesn't pass the newest with /// [`TimestampRewind`](crate::container::TimestampRewind). pub(crate) fn after(track: &moq_net::track::Producer, timestamp: moq_net::Timestamp) -> crate::Result { From dfae86bce85faf50f9e920f593db51c23d810b84 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 20:37:08 -0800 Subject: [PATCH 5/6] fix(mux): a passthrough plan advances its track's segment when it opens a group Two audio runs of one track in a segment's first moof opened two groups, since the segment state advanced only while writing. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import.rs | 9 ++++- rs/moq-mux/src/container/fmp4/import_test.rs | 39 +++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index e67212edcb7..cd1f59f002b 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -1178,7 +1178,14 @@ impl Import { }; let land = match &mut track.output { Output::Passthrough(output) => match output.plan(&mut run, placement, track.kind)? { - Some(land) => Some(land), + Some(land) => { + // The plan opened the segment's group, so a later run of this track in the + // same moof continues it. + if let Land::Opens { .. } = land { + track.segment = Some(self.segment); + } + Some(land) + } None => { // With no group open, video waits for a sync sample past what was published. if !output.open && track.kind == Kind::Video { diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 4ffbe9814b3..b98ec7129b2 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1248,15 +1248,15 @@ async fn audio_presents_from_its_edit() { /// One video fragment decoding from `dts`, a sample per `(cts, flags, byte)` lasting `tick`. fn video_fragment(track_id: u32, dts: u64, tick: u64, entries: &[(u64, u32, u8)]) -> Vec { - video_moof(track_id, tick, &[(dts, entries)]) + track_moof(track_id, tick, &[(dts, entries)]) } -/// A video run decoding from its `dts`, a sample per `(cts, flags, byte)`. -type VideoRun<'a> = (u64, &'a [(u64, u32, u8)]); +/// A run decoding from its `dts`, a sample per `(cts, flags, byte)`. +type TrackRun<'a> = (u64, &'a [(u64, u32, u8)]); /// One moof of a traf per run on `track_id`, each sample lasting `tick`, and their samples back to /// back in one mdat. -fn video_moof(track_id: u32, tick: u64, trafs: &[VideoRun]) -> Vec { +fn track_moof(track_id: u32, tick: u64, trafs: &[TrackRun]) -> Vec { let mut moof = mp4_atom::Moof { mfhd: mp4_atom::Mfhd { sequence_number: 1 }, traf: trafs @@ -1505,7 +1505,7 @@ async fn video_groups_never_overlap() { .zip(&entries) .map(|(&(dts, _), entries)| (dts * tick, entries.as_slice())) .collect(); - fmp4.decode(&video_moof(video_id, tick, &trafs)).unwrap(); + fmp4.decode(&track_moof(video_id, tick, &trafs)).unwrap(); let _finished = fmp4.finish().unwrap(); let [video, _] = published(&consumer, &catalog).await; let presented: Vec = video @@ -1525,6 +1525,35 @@ async fn video_groups_never_overlap() { } } +/// A segment's first moof carrying two runs of one audio track opens one group, not one per run, +/// passthrough or unwrapped. +#[tokio::test] +async fn audio_runs_in_one_moof_share_a_group() { + let (init, _, (audio_id, _)) = bbb_init(); + let tick = 1024; + for passthrough in [false, true] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_cmaf(passthrough); + fmp4.decode(&init).unwrap(); + let name = catalog.snapshot().audio.renditions.keys().next().unwrap().clone(); + let mut track = consumer.track(&name).unwrap().subscribe(replay()).await.unwrap(); + for segment in 0..2u64 { + fmp4.decode(&styp()).unwrap(); + let dts = segment * 2 * tick; + let runs: [TrackRun; 2] = [(dts, &[(0, 0, 0xA)]), (dts + tick, &[(0, 0, 0xB)])]; + fmp4.decode(&track_moof(audio_id, tick, &runs)).unwrap(); + } + let _finished = fmp4.finish().unwrap(); + assert_eq!( + drain_group_frames(&mut track), + vec![2, 2], + "one group per segment, passthrough {passthrough}" + ); + } +} + #[test] fn test_bbb_init_roundtrip() { let data = include_bytes!("test_data/bbb.mp4"); From 4ea1ee0b2047d49758a0dc09e3c83567855b90df Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 20:57:02 -0800 Subject: [PATCH 6/6] fix(mux): a passthrough fragment that continues its group is trimmed against the group's ceiling The plan trimmed a fragment against the newest sample before deciding, from its key, that it continued the open group, so after a seek a leading picture the group could hold was dropped. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import.rs | 49 ++++++++++++-------- rs/moq-mux/src/container/fmp4/import_test.rs | 38 +++++++++++++++ 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index cd1f59f002b..189c0913ef3 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -1670,6 +1670,34 @@ impl Passthrough { opens = false; } + // A group is keyed by its earliest sample past the newest, which `at` already passes, or by + // a seek's start, and the key has to pass the latest group's. A fragment whose key doesn't + // continues the open group, decided before trimming so that it keeps the samples only the + // groups before it bound; with none open it is refused. + let mut key = None; + if opens && self.pending_start.is_none() { + let mut first = at; + for sample in &run.samples { + let timestamp = placement.offset.apply(sample.timestamp)?; + if self.newest.is_none_or(|newest| timestamp > newest) { + first = crate::container::earlier(first, timestamp); + } + } + let next = crate::start::key(&self.track, first)?; + match self.key { + Some(latest) if next <= latest && self.open => opens = false, + Some(latest) if next <= latest => { + let floor = self.start.unwrap_or(first); + return Err(crate::container::TimestampRewind { + timestamp: first, + floor, + } + .into()); + } + _ => key = Some(next), + } + } + let ceiling = match opens { true => self.newest, false => self.ceiling, @@ -1705,31 +1733,14 @@ impl Passthrough { let earliest = placement.offset.apply(run.earliest)?; let land = match opens { - // A group is keyed by its first fragment's timestamp, or a seek's start, which has to - // pass the latest group's. A segment that doesn't continues the open group instead; with - // none open it is refused. true => match self.pending_start.take() { Some(start) => { self.key = Some(start); Land::Opens { start: Some(start) } } None => { - let key = crate::start::key(&self.track, earliest)?; - match self.key { - Some(latest) if key <= latest && self.open => Land::Continues, - Some(latest) if key <= latest => { - let floor = self.start.unwrap_or(earliest); - return Err(crate::container::TimestampRewind { - timestamp: earliest, - floor, - } - .into()); - } - _ => { - self.key = Some(key); - Land::Opens { start: None } - } - } + self.key = key; + Land::Opens { start: None } } }, false => Land::Continues, diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index b98ec7129b2..f43a672c7ed 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1554,6 +1554,44 @@ async fn audio_runs_in_one_moof_share_a_group() { } } +/// A sync sample whose group key doesn't pass a seek's start continues the open group, keeping a +/// leading picture that presents at or before the newest sample, as only the groups before it +/// bound the group's samples, passthrough or unwrapped. +#[tokio::test] +async fn video_continuing_a_seeked_group_keeps_its_leading_pictures() { + let (_, (video_id, video_scale), _) = bbb_init(); + let tick = video_scale.as_u64() / 24; + let non_sync = 0x0001_0000; + let init = bbb_init_with_edits(video_id, vec![media_edit(0)]); + // Presents 0, 6, 2; then a sync sample at 7 and a leading picture at 4. + let fragments: [(u64, &[(u64, u32)]); 2] = [ + (0, &[(0, 0), (5, non_sync), (0, non_sync)]), + (3, &[(4, 0), (0, non_sync)]), + ]; + for passthrough in [false, true] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_cmaf(passthrough); + fmp4.decode(&init).unwrap(); + fmp4.seek(10_000_000).unwrap(); + for (dts, samples) in fragments { + let entries: Vec<_> = samples.iter().map(|&(cts, flags)| (cts * tick, flags, 0xA)).collect(); + fmp4.decode(&video_fragment(video_id, dts * tick, tick, &entries)) + .unwrap(); + } + let _finished = fmp4.finish().unwrap(); + let [video, _] = published(&consumer, &catalog).await; + let presented: Vec = video + .iter() + .map(|(timestamp, _, _)| (timestamp * u128::from(video_scale.as_u64())).div_ceil(1_000_000) as u64 / tick) + .collect(); + assert_eq!(presented, [0, 6, 2, 7, 4], "passthrough {passthrough}"); + let name = catalog.snapshot().video.renditions.keys().next().unwrap().clone(); + assert_eq!(starts(&consumer, &name).await.len(), 1, "passthrough {passthrough}"); + } +} + #[test] fn test_bbb_init_roundtrip() { let data = include_bytes!("test_data/bbb.mp4");