From aa09acfb7d18106887fa581d32a0ba35c99d7d7e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 08:32:42 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m2/hang04/video-warmup.md Quest-Claim: 4b5c0bc890ac43a1 From 59e48e1cbf515115ed43ea8324e66aac1e190a6a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 09:40:48 -0800 Subject: [PATCH 2/3] feat(hang)!: video warmup leaves hang-04, and the players stop reading it moq-hang-04 defines `warmup` for audio renditions only. A video rendition's `warmup` is an unknown field, every video group starts with a keyframe again, and the renamed `track` is the only version-4 trigger. The one-decoder video warmup path in @moq/watch, moq_video::decode, moq play, moq-gst, moq-ffi, and moq-c is removed; audio warmup and the container consumer's reach stay. Co-Authored-By: Claude Opus 5.5 --- dart/moq_ffi/lib/src/moq.dart | 18 +- doc/concept/hang.md | 2 +- doc/lib/c/index.md | 2 +- doc/lib/js/watch.md | 9 - doc/lib/rs/hang.md | 2 +- drafts/draft-lcurley-moq-hang.md | 39 ++- go/wrapper/internal/bridge/handles.go | 13 +- go/wrapper/media/media_test.go | 10 +- go/wrapper/video/video.go | 3 - go/wrapper/video/video_internal_test.go | 17 -- js/hang/api/catalog.api.md | 3 - js/hang/src/catalog/root.test.ts | 3 +- js/hang/src/catalog/root.ts | 6 +- js/hang/src/catalog/video.test.ts | 13 +- js/hang/src/catalog/video.ts | 4 - js/hang/src/container/consumer.ts | 3 +- js/watch/src/video/config.ts | 3 +- js/watch/src/video/decoder.test.ts | 327 +---------------------- js/watch/src/video/decoder.ts | 143 ++-------- js/watch/src/video/recovery.ts | 47 +--- quest/m2/hang04/README.md | 6 +- quest/m2/hang04/entry-version.md | 4 +- quest/m2/hang04/shape.md | 5 +- quest/m2/hang04/video-warmup.md | 102 ------- quest/m4/intra-refresh/README.md | 6 +- quest/m4/intra-refresh/catalog-warmup.md | 3 +- rs/hang/fixtures/catalog-parity.json | 72 +---- rs/hang/src/catalog/root.rs | 24 +- rs/hang/src/catalog/video/mod.rs | 32 --- rs/moq-c/src/api.rs | 17 +- rs/moq-c/src/consume.rs | 130 ++++----- rs/moq-c/src/test.rs | 31 --- rs/moq-cli/benches/play_buffer.rs | 6 +- rs/moq-cli/src/play/buffer.rs | 159 ++--------- rs/moq-cli/src/play/media.rs | 41 +-- rs/moq-cli/src/play/video.rs | 116 +------- rs/moq-ffi/src/media.rs | 112 +++----- rs/moq-ffi/src/test.rs | 2 - rs/moq-ffi/src/video.rs | 14 - rs/moq-gst/src/source/imp.rs | 171 ++++-------- rs/moq-mux/src/container/consumer.rs | 5 +- rs/moq-video/src/decode/consumer.rs | 276 ++----------------- 42 files changed, 292 insertions(+), 1709 deletions(-) delete mode 100644 go/wrapper/video/video_internal_test.go delete mode 100644 quest/m2/hang04/video-warmup.md diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 74bbb704978..260340b939b 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,9 +8,7 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; - import "package:ffi/ffi.dart"; - import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -2304,7 +2302,6 @@ class MoqVideo { final int preference; final double? framerate; final MoqContainer container; - final Duration? warmup; MoqVideo({ this.label = null, this.broadcast = null, @@ -2319,7 +2316,6 @@ class MoqVideo { this.preference = 0, this.framerate, required this.container, - this.warmup = null, }); } @@ -2395,11 +2391,6 @@ class FfiConverterMoqVideo { ); final container = container_lifted.value; new_offset += container_lifted.bytesRead; - final warmup_lifted = FfiConverterOptionalDuration.read( - Uint8List.view(buf.buffer, new_offset), - ); - final warmup = warmup_lifted.value; - new_offset += warmup_lifted.bytesRead; return LiftRetVal( MoqVideo( label: label, @@ -2415,7 +2406,6 @@ class FfiConverterMoqVideo { preference: preference, framerate: framerate, container: container, - warmup: warmup, ), new_offset - buf.offsetInBytes, ); @@ -2436,7 +2426,6 @@ class FfiConverterMoqVideo { FfiConverterInt32.allocationSize(value.preference) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + - FfiConverterOptionalDuration.allocationSize(value.warmup) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -2497,10 +2486,6 @@ class FfiConverterMoqVideo { value.container, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalDuration.write( - value.warmup, - Uint8List.view(buf.buffer, new_offset), - ); return new_offset - buf.offsetInBytes; } @@ -2518,7 +2503,6 @@ class FfiConverterMoqVideo { FfiConverterInt32.allocationSize(value.preference) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + - FfiConverterOptionalDuration.allocationSize(value.warmup) + 0; } } @@ -25898,7 +25882,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqmediatrackconsumer_subscribe() != - 16252) { + 25805) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqmediatrackproducer_audio() != diff --git a/doc/concept/hang.md b/doc/concept/hang.md index fe1b0afc9ba..90d754c22cb 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -74,7 +74,7 @@ A few things the catalog can express beyond decoder config: - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its encoders' `join`. A broadcast without it gets no HLS playlists. - **Clock.** The optional `media.clock` maps PTS zero to wall time so every media track shares one fixed epoch after timescale conversion. Publish independent sources as sibling broadcasts under one prefix, such as `robot/front.hang` and `robot/rear.hang`; each broadcast's audio and video entries are renditions of that source. In one Rust publisher, pass the same `moq_mux::Clock` to each `catalog::Config::with_clock`; JS publishers on one page already share the page clock. Consumers discover the siblings by prefix and align their timestamps through the advertised wall mapping; `@moq/watch` plays one broadcast at a time. -- **Version.** The root `version` is the oldest moq-hang draft that reads the catalog correctly, written only when a field needs one newer than 03: a `track` that differs from its ID, or a video `warmup`. Readers ignore fields they don't know, but refuse a `version` above their own with a typed "unsupported catalog version" error, so a player can say to upgrade instead of misreading the catalog. An additive field never raises it. +- **Version.** The root `version` is the oldest moq-hang draft that reads the catalog correctly, written only when a field needs one newer than 03: a `track` that differs from its ID. Readers ignore fields they don't know, but refuse a `version` above their own with a typed "unsupported catalog version" error, so a player can say to upgrade instead of misreading the catalog. An additive field never raises it. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. ## Media reports diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index 647f975a8d4..96b721cb197 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -45,7 +45,7 @@ same frame/gap/end tags as decoded audio and video. Release every kind with `moq_consume_read_free`; frame payload pointers borrow that handle. A gap breaks continuity. An encoder end is not callback EOF: frames before its endpoint are the old encoder's drain, and the first at or past it starts fresh. Callback 0 ends the stream. -These consumers include the rendition's catalog `warmup` in both the history fetched at +These consumers include an audio rendition's catalog `warmup` in both the history fetched at join and the subscription budget. Feed that history to your decoder before presenting output; the requested max delay still controls steady-state skips. diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index a0b6f8ee103..9de48606f89 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -63,15 +63,6 @@ The failing track closes its subscription and stops decoding. Select another source or rendition to clear the failure; the player does not resubscribe to broken media automatically. -A video rendition encoded with intra refresh has no keyframes, and its catalog -`warmup` says how long a decoder takes to clean up after joining. The player -joins such a rendition that much earlier and decodes from the group start, but -shows nothing until the warmup has passed, after a join and after each skip. -An H.264 group opens on a recovery point that is not an IDR, which Chromium's -WebCodecs refuses unless it counts zero frames, so the player inserts one that -does before the first slice. Firefox and Safari are untested, and Chromium -refuses an H.265 group that opens without an IDR. - `el.broadcast.out.status` is `connecting`, `offline`, `loading`, `live`, `error`, or `unsupported`. It reads `connecting` until a session could have announced the broadcast: while the connection dials, redials, or was refused (`el.connection.error` says why), and until the diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index 1662ba86bf0..22ef654566a 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -27,7 +27,7 @@ hang is the format, not the pipeline. [`moq-mux`](/lib/rs/moq-mux) produces the catalog for you and decodes any container in group order (`moq_mux::container::Consumer`). Its `Delay { min, max }` says how far behind the live edge to play: it skips groups that fall past `max`, and starts at the -group covering the live edge less `min` (your playout delay), plus a rendition's +group covering the live edge less `min` (your playout delay), plus an audio rendition's catalog `warmup` passed to a `Playout`, waiting up to that long for it when the newest group arrives first. Both default to zero, which starts at the newest group at once. A backlog that landed before the first read starts within `max` diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index c61e9acf4aa..3715b23541c 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -183,12 +183,9 @@ A consumer MUST refuse a catalog whose `version` is above the draft it implement A publisher writes the highest draft whose fields the catalog uses that an older consumer would misread, and omits `version` while that is 3. A change raises the version only when a consumer that ignored it would misread the catalog, never for a field it can safely ignore. -This draft needs `version` 4 for: +This draft needs `version` 4 only for a `track` ({{field-track}}) other than the entry's ID, which an older consumer subscribes to instead. -- a `track` ({{field-track}}) other than the entry's ID, which an older consumer subscribes to instead; -- a video `warmup` ({{field-warmup}}) above zero, since an older consumer starts decoding a group at a picture that is not a keyframe. - -Every other catalog change in this draft is safe to ignore: `enabled` ({{field-enabled}}) leaves an older consumer selecting a disabled rendition, which plays nothing; `media` ({{media}}) only offers the clock and join interval, which an older consumer goes without; `stats` and `echo` ({{media-stats}}) only offer optional tracks; `include` ({{include}}) only offers broadcasts to open on selection; video `reorder` leaves an older consumer inferring decode times from timestamps; `jitter`, `delay`, audio `warmup`, and the data track estimates are hints. +Every other catalog change in this draft is safe to ignore: `enabled` ({{field-enabled}}) leaves an older consumer selecting a disabled rendition, which plays nothing; `media` ({{media}}) only offers the clock and join interval, which an older consumer goes without; `stats` and `echo` ({{media-stats}}) only offer optional tracks; `include` ({{include}}) only offers broadcasts to open on selection; video `reorder` leaves an older consumer inferring decode times from timestamps; `jitter`, `delay`, audio `warmup` ({{field-warmup}}), and the data track estimates are hints. ## Media {#media} The catalog's root `media` field holds what applies to every audio and video track: @@ -479,6 +476,7 @@ In addition to the WebCodecs fields, each rendition MAY carry the common renditi ~~~ type AudioDecoderConfigExtensions = { "layout": string | undefined, + "warmup": number | undefined, } ~~~ @@ -489,6 +487,14 @@ type AudioDecoderConfigExtensions = { An Opus frame timestamp counts the samples the OpusHead pre-skip trims: a consumer presents each decoded sample the pre-skip, at 48 kHz, earlier than its frame timestamp places it, so the first sample kept plays at the first decoded frame's timestamp. +### warmup {#field-warmup} + +An audio rendition MAY declare `warmup`, an integer duration in milliseconds after a group start before its decoded output is presentable without decoding the previous group: the decoder's pre-roll, for example 80 milliseconds for Opus. +A publisher MUST round a fractional-millisecond warmup up to the next whole millisecond. +Absent or `0` means every group start is immediately presentable. +After a non-continuous join, a consumer MUST decode from the group start and MUST NOT present samples stamped before the group's start plus `warmup`. +It SHOULD join `warmup` earlier to obtain the requested presentation time. + ### PCM {#audio-pcm} Hang defines the `"pcm"` audio codec for uncompressed samples. @@ -800,18 +806,7 @@ For example: - A B-frame stream contributes its reorder delay, the presentation timestamp minus the decode timestamp. - A fragment or packet batch contributes the media span between its earliest and latest timestamps. -### warmup {#field-warmup} - -An audio or video rendition MAY declare `warmup`, an integer duration in milliseconds after a group start before its decoded output is presentable without decoding the previous group. -A publisher MUST round a fractional-millisecond warmup up to the next whole millisecond. -Absent or `0` means every group start is immediately presentable. -After a non-continuous join, a consumer MUST decode from the group start. -It SHOULD join `warmup` earlier to obtain the requested presentation time. -Unless it also decodes from an earlier group start, it MUST NOT present frames stamped before the group's start plus `warmup`. -A consumer that does MAY present those frames, which can show decoding artifacts, but MUST switch to the earlier decoder's frames once that decoder reaches the presented timestamp. - -For video, `warmup` can describe a refresh cycle during which decoding converges without an initial keyframe. -For audio, it describes decoder pre-roll, for example 80 milliseconds for Opus, and does not change the group-start rule. +>>> ### delay {#field-delay} The maximum amount, in milliseconds, by which a frame's presentation timestamp has trailed one that another rendition of the broadcast had already written. @@ -923,11 +918,9 @@ The `kind` field selects the framing; a consumer MUST ignore a rendition whose ` A container without a string `kind`, or with a recognized `kind` missing its fields, makes the rendition malformed ({{malformed}}). Every container shares the same group rules: -A video group MUST start at a random access point: a keyframe unless the rendition declares `warmup` ({{field-warmup}}), in which case it MAY start at a picture from which decoding converges within that duration. -Other groups MUST start with a keyframe. -A group containing only a discontinuity marker is exempt from these rules. +Each group MUST start with a keyframe, except a group that contains only a discontinuity marker. If the codec does not support delta frames (e.g. audio), a group MAY consist of multiple keyframes. -Otherwise, a group MUST consist of its random access picture followed by zero or more delta frames. +Otherwise, a group MUST consist of a single keyframe followed by zero or more delta frames. A group with no decodable frames is a walk-now discontinuity: one empty codec payload and no media. An empty payload is data on a data track, so in every container a data track's marker is one object with no bytes, which is no frame; it ends at the furthest point the consumer has read. @@ -1123,7 +1116,7 @@ This document has no IANA actions. - Replaced the root `archive` section and its per-track timelines with the root `media` section ({{media}}). Its `join.interval` is the longest group of any audio or video track, fixed for the broadcast, and the grid an edge numbers HLS and DASH segments on, each segment the smallest whole multiple of it at least a second long, from each group's first timestamp ({{hls}}); the root `clock` moved to `media.clock`. Segment indexes, frame positions, `end`, `keyframe`, `timescale`, `replay`, `replayEpoch`, `store`, and `version` are gone. - A reader refuses a JSON frame in which an object repeats a member name. - A malformed audio or video rendition ({{malformed}}) refuses the catalog: a known codec string without valid parameters, a zero or out-of-range dimension, sample rate, or channel count, a bitrate or duration above 2^53 - 1, or a container without a string `kind`. -- Added the root `version` ({{catalog-version}}): the oldest draft that reads the catalog, which a consumer refuses when above its own. A `track` other than the entry's ID or a video `warmup` needs 4. Consumers MUST ignore unknown fields. +- Added the root `version` ({{catalog-version}}): the oldest draft that reads the catalog, which a consumer refuses when above its own. A `track` other than the entry's ID needs 4. Consumers MUST ignore unknown fields. - A data track marks a discontinuity with a group of one object with no bytes, since its empty payload is data. An object with no bytes on an audio or video track, or sharing its group, is malformed. - Removed the Recording section: the store layout is a moq-archive format, documented with that crate, not part of this draft. - Removed the viewer feedback `latency` field. @@ -1140,7 +1133,7 @@ This document has no IANA actions. - Specified MSF text metadata preservation and refusal of unrepresentable text packaging during gateway conversion. - Specified conversion between relative broadcast references and MSF track namespaces without changing track identity, reading a namespace aligned with the catalog path's trailing segments as the catalog itself. -- Added optional audio and video `warmup` durations, rounded up to whole milliseconds, and allowed video groups to start at convergent random access pictures. A consumer also decoding from an earlier group start may present frames before `warmup`, switching to that decoder once it reaches the presented timestamp. +- Added the optional audio `warmup` ({{field-warmup}}): the decoder pre-roll after a group start, rounded up to whole milliseconds, that a consumer decodes but does not present after a join. - Defined `jitter` from media timestamps alone, the reorder delay and batch spans a publisher writes, replacing fixed frame-duration hints and wall-clock flush lateness. - Timed CMAF samples from the frame timestamp, which is the fragment's earliest presentation time; `tfdt` is only relative within the fragment, and places the samples of an untimed frame. A timed CMAF track counts in its `mdhd` ticks. - Replaced the live-edge floor with monotonic group starts: a group may overlap the previous group's content but not start before it. diff --git a/go/wrapper/internal/bridge/handles.go b/go/wrapper/internal/bridge/handles.go index e1f3b9dd0c5..1de15778a45 100644 --- a/go/wrapper/internal/bridge/handles.go +++ b/go/wrapper/internal/bridge/handles.go @@ -62,13 +62,11 @@ func NonNegative(value *time.Duration, name string) error { return nil } -// Rendition refuses a rendition's negative jitter or warmup, for the same reason. +// Rendition refuses an audio rendition's negative jitter or warmup, for the same reason. A video +// rendition carries no duration. func Rendition(rendition ffi.MoqRendition) error { - switch rendition := rendition.(type) { - case ffi.MoqRenditionAudio: + if rendition, ok := rendition.(ffi.MoqRenditionAudio); ok { return Audio(rendition.Config) - case ffi.MoqRenditionVideo: - return Video(rendition.Config) } return nil } @@ -80,8 +78,3 @@ func Audio(audio ffi.MoqAudio) error { } return NonNegative(audio.Warmup, "warmup") } - -// Video refuses a video rendition's negative warmup. -func Video(video ffi.MoqVideo) error { - return NonNegative(video.Warmup, "warmup") -} diff --git a/go/wrapper/media/media_test.go b/go/wrapper/media/media_test.go index dceb8f474c5..149d98e674b 100644 --- a/go/wrapper/media/media_test.go +++ b/go/wrapper/media/media_test.go @@ -62,13 +62,9 @@ func TestRefusesNegativeDurations(t *testing.T) { if _, err := (&moq.BroadcastConsumer{}).Resolve(ctx, rendition); err == nil { t.Fatal("a negative jitter was resolved, want an error") } - for _, warmup := range []media.Rendition{ - media.AudioRendition("audio", media.Audio{Warmup: &negative}), - media.VideoRendition("video", media.Video{Warmup: &negative}), - } { - if _, err := media.NewTrackConsumer(ctx, nil, media.TrackConfig{Rendition: warmup}); err == nil { - t.Fatal("a negative warmup was subscribed, want an error") - } + warmup := media.AudioRendition("audio", media.Audio{Warmup: &negative}) + if _, err := media.NewTrackConsumer(ctx, nil, media.TrackConfig{Rendition: warmup}); err == nil { + t.Fatal("a negative warmup was subscribed, want an error") } } diff --git a/go/wrapper/video/video.go b/go/wrapper/video/video.go index e431be7fe4c..bbce2291b3b 100644 --- a/go/wrapper/video/video.go +++ b/go/wrapper/video/video.go @@ -163,9 +163,6 @@ type Consumer struct{ inner *ffi.MoqVideoConsumer } // Subscribe subscribes to the catalog rendition id, following its broadcast and track. func Subscribe(ctx context.Context, broadcast *moq.BroadcastConsumer, id string, rendition media.Video, output DecoderOutput) (*Consumer, error) { - if err := bridge.Video(rendition); err != nil { - return nil, err - } out, err := output.ffi() if err != nil { return nil, err diff --git a/go/wrapper/video/video_internal_test.go b/go/wrapper/video/video_internal_test.go deleted file mode 100644 index ae52c5cf38e..00000000000 --- a/go/wrapper/video/video_internal_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package video - -import ( - "context" - "testing" - "time" - - "moq.dev/moq/media" -) - -// The generated converter panics on a negative duration, so it is refused first. -func TestSubscribeRefusesNegativeWarmup(t *testing.T) { - negative := -time.Millisecond - if _, err := Subscribe(context.Background(), nil, "video", media.Video{Warmup: &negative}, DecoderOutput{}); err == nil { - t.Fatal("a negative warmup was subscribed, want an error") - } -} diff --git a/js/hang/api/catalog.api.md b/js/hang/api/catalog.api.md index d0c8e83767a..184c4ef4dab 100644 --- a/js/hang/api/catalog.api.md +++ b/js/hang/api/catalog.api.md @@ -517,7 +517,6 @@ export const RootSchema: ZodMiniObject<{ reorder?: number & $brand<"u8"> | undefined; stereo?: string | undefined; track?: string | undefined; - warmup?: number & $brand<"u53"> | undefined; }>, Record>>; rotation: ZodMiniOptional>; }, $strip>>; @@ -728,7 +727,6 @@ export const VideoConfigSchema: ZodMiniObject<{ reorder: ZodMiniOptional<$ZodBranded, "u8", "out">>; stereo: ZodMiniOptional>; track: ZodMiniOptional>; - warmup: ZodMiniOptional<$ZodBranded, "u53", "out">>; }, $strip>; export const VideoSchema: ZodMiniObject<{ @@ -814,7 +812,6 @@ export const VideoSchema: ZodMiniObject<{ reorder?: number & $brand<"u8"> | undefined; stereo?: string | undefined; track?: string | undefined; - warmup?: number & $brand<"u53"> | undefined; }>, Record>>; rotation: ZodMiniOptional>; }, $strip>; diff --git a/js/hang/src/catalog/root.test.ts b/js/hang/src/catalog/root.test.ts index 1abfc205ad0..b710fe83f74 100644 --- a/js/hang/src/catalog/root.test.ts +++ b/js/hang/src/catalog/root.test.ts @@ -364,7 +364,8 @@ test("the catalog version is needed only for misread fields", () => { expect(encode(root(avc)).version).toBeUndefined(); expect(encode(root({ ...avc, track: "hd" })).version).toBeUndefined(); expect(encode(root({ ...avc, track: "video" })).version).toBe(4); - expect(encode(root({ ...avc, warmup: 500 })).version).toBe(4); + // moq-hang-04 defines no video `warmup`, so it is dropped as an unknown field. + expect(encode(root({ ...avc, warmup: 500 })).version).toBeUndefined(); const data = RootSchema.parse({ json: { tracks: { chat: { mode: "stream", track: "chat.json" } } } }); expect(encode(data).version).toBe(4); // A carried version is recomputed, never trusted. diff --git a/js/hang/src/catalog/root.ts b/js/hang/src/catalog/root.ts index 80f80eb11d4..006adac2e89 100644 --- a/js/hang/src/catalog/root.ts +++ b/js/hang/src/catalog/root.ts @@ -106,11 +106,7 @@ function minVersion(root: Root): number | undefined { root.binary?.tracks, ].some((entries) => Object.entries(entries ?? {}).some(([id, config]) => track(id, config) !== id)); - // moq-hang-04: a reader that ignores video `warmup` assumes every group starts at a keyframe, - // and hands its decoder one that starts at a converging picture instead. - const warmup = Object.values(root.video?.renditions ?? {}).some((config) => (config.warmup ?? 0) > 0); - - return renamed || warmup ? 4 : undefined; + return renamed ? 4 : undefined; } /** A catalog update used one rendition ID for both a video and an audio rendition. */ diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 145b20d783d..7d3a3c3e1e7 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -118,16 +118,11 @@ test("video config ignores a legacy stalled flag", () => { expect(parsed.enabled).toBeUndefined(); }); -test("video warmup round trips as optional integer milliseconds", () => { +// moq-hang-04 defines `warmup` for audio only, so a video rendition's is an unknown field. +test("a video warmup is ignored", () => { const base = { codec: "vp8", container: { kind: "legacy" } }; - expect(VideoConfigSchema.parse(base).warmup).toBeUndefined(); - for (const warmup of [0, 80, 1_000]) { - const config = VideoConfigSchema.parse({ ...base, warmup }); - expect(Number(config.warmup)).toBe(warmup); - expect(VideoConfigSchema.parse(JSON.parse(JSON.stringify(config)))).toEqual(config); - } - for (const warmup of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { - expect(() => VideoConfigSchema.parse({ ...base, warmup })).toThrow(); + for (const warmup of [0, 500, -1, 0.5]) { + expect("warmup" in VideoConfigSchema.parse({ ...base, warmup })).toBe(false); } }); diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index a04b81d3aad..8d30b2034e1 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -208,10 +208,6 @@ export const VideoConfigSchema = z.object({ ), ), - // After a non-continuous join, decode from the group start, present at start plus warmup, - // and join that many milliseconds earlier. - warmup: z.optional(u53Schema), - // How far this rendition's frames reach the transport behind the broadcast's earliest // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index f063129a06f..72ded97a25c 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -58,8 +58,7 @@ export interface ConsumerProps { /** * How much further than the min delay the start reaches, in milliseconds (default: 0), so a * decoder that needs that much media before its output is clean starts early enough, such as an - * Opus decoder or an intra-refresh video decoder. Pass the catalog - * rendition's `warmup`. Mirrors Rust's `container::Playout`. + * Opus decoder. Pass the catalog audio rendition's `warmup`. Mirrors Rust's `container::Playout`. * * Put the max delay plus the warmup on the subscription, since the publisher skips what falls * outside that budget. The start is no later than the group covering the live edge less diff --git a/js/watch/src/video/config.ts b/js/watch/src/video/config.ts index 68cf7afa9be..e7b58ee04f9 100644 --- a/js/watch/src/video/config.ts +++ b/js/watch/src/video/config.ts @@ -3,7 +3,7 @@ import type * as Catalog from "@moq/hang/catalog"; /** The catalog fields that determine the demuxer and WebCodecs decoder instance. */ export type DecoderConfig = Pick< Catalog.VideoConfig, - "codec" | "container" | "description" | "displayAspectWidth" | "displayAspectHeight" | "warmup" + "codec" | "container" | "description" | "displayAspectWidth" | "displayAspectHeight" > & { optimizeForLatency: boolean; interlaced: boolean; @@ -31,7 +31,6 @@ export function playbackIdentity(config: Catalog.VideoConfig): PlaybackIdentity displayAspectHeight: config.displayAspectHeight, optimizeForLatency: config.optimizeForLatency ?? true, interlaced: config.interlaced ?? false, - warmup: config.warmup, }, }; } diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 2c7e70b7183..c80c5e71274 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -9,8 +9,8 @@ import { Effect, Signal } from "@moq/signals"; import { ringSamples } from "../audio/latency"; import { AudioRingBuffer } from "../audio/ring-buffer"; import type { Broadcast } from "../broadcast"; -import { DELAY_MAX, type Delay, Sync, Track } from "../sync"; -import { Decoder, insertRecoveryPoint } from "./decoder"; +import { type Delay, Sync, Track } from "../sync"; +import { Decoder } from "./decoder"; import { Source } from "./source"; const flush = () => yieldNow(); @@ -190,14 +190,14 @@ function manualAt(ms: number) { clock.jump(Duration.millis(ms)); return { advance: (by: number) => clock.advance(Duration.millis(by)), installed: clock.install() }; } -function sample(group: bigint, timestamp: number, keyframe = true, payload: Uint8Array = new Uint8Array([1])): Read { +function sample(group: bigint, timestamp: number, keyframe = true): Read { return { kind: "frame", group, frame: { arrival: now(), timestamp: Duration.micros(timestamp), - payload, + payload: new Uint8Array([1]), keyframe, }, }; @@ -253,9 +253,8 @@ async function guardedPlayback( const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); const originalChunk = Object.getOwnPropertyDescriptor(globalThis, "EncodedVideoChunk"); const submitted: number[] = []; - // What each codec, in creation order, was given, and the bytes of each chunk. + // What each codec, in creation order, was given. const decoded: number[][] = []; - const payloads: Uint8Array[][] = []; const outputs: number[] = []; const pictures: Picture[] = []; const owed = new Set(hold); @@ -289,7 +288,6 @@ async function guardedPlayback( constructor(callbacks: VideoDecoderInit) { this.callbacks = callbacks; decoded.push([]); - payloads.push([]); errors.push(callbacks.error); frames.push(callbacks.output); } @@ -307,7 +305,6 @@ async function guardedPlayback( this.#keyRequired = false; submitted.push(chunk.timestamp); decoded[this.index].push(chunk.timestamp); - payloads[this.index].push((chunk as unknown as { data: Uint8Array }).data); const emit = () => { outputs.push(chunk.timestamp); const picture = new Picture(chunk.timestamp, pictures); @@ -414,7 +411,6 @@ async function guardedPlayback( track, submitted, decoded, - payloads, outputs, pictures, sync, @@ -1676,316 +1672,3 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); } - -describe("Decoder with a warmup rendition", () => { - // One refresh sweep, which is one group. - const WARMUP = Catalog.u53(300); - - // A frame of `group` at `ms`, a recovery point when `key`. - const at = (group: bigint, ms: number, key = false) => sample(group, ms * 1000, key); - - // A warmup rendition reading the container consumer's reads from a feed, presenting at once. - // `hold` keeps those timestamps in the codec until the next chunk. - async function refresh(fields: Partial = { warmup: WARMUP }, hold: number[] = []) { - const feeds = new Feeds(); - const playback = await guardedPlayback("legacy", [], hold, false, 0, { fields, feeds }); - // Every picture shown, in ms. - const shown: number[] = []; - const watch = new Effect(); - watch.subscribe(playback.decoder.out.frame, (frame) => { - const picture = frame as unknown as Picture | undefined; - if (picture) shown.push(picture.timestamp / 1000); - }); - return { - playback, - shown, - push: async (...reads: Read[]) => { - feeds.push(0, ...reads); - await microtasks(); - }, - async close() { - watch.close(); - playback.close(); - await microtasks(); - expect(playback.pictures.filter((frame) => !frame.closed)).toEqual([]); - }, - }; - } - - it("opens one decoder whose subscription tolerates the warmup", async () => { - for (const fields of [{}, { warmup: Catalog.u53(0) }, { warmup: WARMUP }, { warmup: Catalog.u53(3_600_000) }]) { - const subscribe = spyOn(Moq.Track.Consumer.prototype, "subscribe"); - const run = await refresh(fields); - try { - expect(run.playback.codecs).toBe(1); - // An hour of warmup reaches back only as far as the delay cap. - const warmup = Duration.millis(fields.warmup ?? 0).min(DELAY_MAX); - expect(subscribe.mock.calls.map(([options]) => options?.delay?.max)).toEqual([warmup]); - } finally { - subscribe.mockRestore(); - await run.close(); - } - } - }); - - it("hides the warmup after a join and after a gap, but not across groups or a fresh encoder", async () => { - const run = await refresh(); - try { - // The join's recovery point at 1000: the next 300ms are decoded but not shown. - await run.push(at(10n, 1000, true), at(10n, 1100), at(10n, 1200), at(10n, 1300)); - // The next group continues the codec, so it shows from its first picture. - await run.push(at(11n, 1400, true), at(11n, 1500)); - expect(run.playback.decoded[0]).toEqual([1000, 1100, 1200, 1300, 1400, 1500].map((ms) => ms * 1000)); - expect(run.shown).toEqual([1300, 1400, 1500]); - - // A skip resumes at group 13's recovery point, warming up again. - await run.push(gap, at(13n, 2000, true), at(13n, 2200), at(13n, 2300), at(14n, 2400, true)); - expect(run.shown).toEqual([1300, 1400, 1500, 2300, 2400]); - - // A declared end, then a fresh encoder on another timeline: its recovery point is clean. - await run.push(marker(14n, 2_500_000), at(15n, 0, true), at(15n, 100)); - expect(run.shown).toEqual([1300, 1400, 1500, 2300, 2400, 0, 100]); - expect(run.playback.resets).toBe(1); - } finally { - await run.close(); - } - }); - - it("keeps a warming picture the codec still holds hidden past a later gap", async () => { - const run = await refresh({ warmup: WARMUP }, [1_100_000]); - try { - // 1100 is still inside the codec when the skip moves the warmup to group 13. - await run.push(at(10n, 1000, true), at(10n, 1100), gap, at(13n, 2000, true), at(13n, 2300)); - expect(run.playback.outputs).toContain(1_100_000); - expect(run.shown).toEqual([2300]); - } finally { - await run.close(); - } - }); - - it("drops a picture whose clock wait resolved before a timeline break it resumes after", async () => { - // Hold 1100's wait open: resolving it once the break is read stands in for a wait that returned - // true but whose output callback resumed only after the break. - let release: ((present: boolean) => void) | undefined; - const original = Track.prototype.wait; - const wait = spyOn(Track.prototype, "wait").mockImplementation(function (this: Track, timestamp, signal) { - if (timestamp.asMicros() !== 1_100_000) return original.call(this, timestamp, signal); - return new Promise((resolve) => { - release = resolve; - }); - }); - const run = await refresh({}); - try { - await run.push(at(10n, 1000, true), at(10n, 1100)); - expect(run.shown).toEqual([1000]); - // A declared end, then a fresh encoder starting over at 0. - await run.push(marker(10n, 1_200_000), at(11n, 0, true)); - expect(run.shown).toEqual([1000, 0]); - release?.(true); - await microtasks(); - // The old timeline's picture neither shows nor late-rejects the new timeline's next one. - await run.push(at(11n, 100)); - expect(run.shown).toEqual([1000, 0, 100]); - } finally { - wait.mockRestore(); - await run.close(); - } - }); - - it("needs no warmup for a fresh encoder, even after a join or a gap", async () => { - const run = await refresh(); - try { - // Joined on a declared end: the next group is a fresh encoder's first. - await run.push(marker(9n, 900_000), at(10n, 1000, true), at(10n, 1100)); - expect(run.shown).toEqual([1000, 1100]); - // A skip, then a declared end before any recovery point: the fresh encoder is clean too. - await run.push(gap, marker(12n, 1_300_000), at(13n, 0, true), at(13n, 100)); - expect(run.shown).toEqual([1000, 1100, 0, 100]); - } finally { - await run.close(); - } - }); - - it("shows every picture of a rendition without a warmup", async () => { - const run = await refresh({}); - try { - await run.push(at(10n, 1000, true), at(10n, 1100), gap, at(13n, 2000, true), at(13n, 2100)); - expect(run.shown).toEqual([1000, 1100, 2000, 2100]); - } finally { - await run.close(); - } - }); - - // Through a real container consumer, which reaches back one more sweep for its start. - it("joins one group earlier and plays the live edge at the delay", async () => { - const clock = manualAt(5000); - const playback = await guardedPlayback("legacy", [], [], true, 0, { - fields: { warmup: WARMUP }, - initial: Duration.millis(100), - }); - const track = playback.track as Moq.Track.Producer; - const write = (group: Moq.Group.Producer, from: number, to: number) => { - for (let ms = from; ms <= to; ms += 30) group.writeFrame(legacyFrame(Duration.millis(ms))); - }; - const flush = async () => { - await clock.advance(0); - await microtasks(); - }; - try { - // A relay serves a cold join newest first: the live group, then the sweep before it. - write(track.createGroup(5n), 1500, 1620); - await flush(); - const earlier = track.createGroup(4n); - write(earlier, 1200, 1470); - earlier.finish(); - await flush(); - - // The start reaches 400ms back, to the group before the live one, and decodes it all. - expect(playback.decoded[0][0]).toBe(1_200_000); - expect(playback.decoded[0]).toContain(1_620_000); - - // The poster is the first picture past the warmup. The live group's arrivals hold the clock - // 100ms behind the live edge, not another 300ms behind for the warmup's. - const poster = playback.decoder.out.frame.peek() as unknown as Picture; - expect(poster.timestamp).toBe(1_500_000); - await clock.advance(10); - await microtasks(); - const due = playback.decoder.out.frame.peek() as unknown as Picture; - expect(due.timestamp).toBe(1_530_000); - } finally { - playback.close(); - await microtasks(); - clock.installed.close(); - } - }); -}); - -// H.264 NAL units, without their start code or length prefix. -const NAL = { - aud: [0x09, 0xf0], - sps: [0x67, 0x64, 0x00, 0x28], - pps: [0x68, 0xee], - // An SEI recovery point counting one 30-frame sweep, as an intra-refresh encoder writes it. - sweep: [0x06, 0x06, 0x02, 0x0f, 0x84, 0x80], - // The one the decoder inserts: recovery_frame_cnt = 0. - recovered: [0x06, 0x06, 0x01, 0x84, 0x80], - slice: [0x41, 0x9a, 0x02], - idr: [0x65, 0x88, 0x84], -}; - -// Length-prefixed with `size` bytes, as an avcC `description` declares. -function avcc(nals: number[][], size = 4): Uint8Array { - const out: number[] = []; - for (const nal of nals) { - for (let i = size - 1; i >= 0; i--) out.push((nal.length >> (8 * i)) & 0xff); - out.push(...nal); - } - return new Uint8Array(out); -} - -function annexB(nals: number[][], code = [0, 0, 0, 1]): Uint8Array { - return new Uint8Array(nals.flatMap((nal) => [...code, ...nal])); -} - -// avcC records whose lengthSizeMinusOne is 3 and 1. -const AVCC4 = new Uint8Array([0x01, 0x64, 0x00, 0x28, 0xff]); -const AVCC2 = new Uint8Array([0x01, 0x64, 0x00, 0x28, 0xfd]); - -describe("insertRecoveryPoint", () => { - it("inserts a zero-count recovery point before the first slice of a length-prefixed recovery point", () => { - const payload = avcc([NAL.sweep, NAL.slice, NAL.slice]); - expect(insertRecoveryPoint(payload, AVCC4)).toEqual(avcc([NAL.sweep, NAL.recovered, NAL.slice, NAL.slice])); - expect(insertRecoveryPoint(avcc([NAL.slice], 2), AVCC2)).toEqual(avcc([NAL.recovered, NAL.slice], 2)); - }); - - it("inserts it before the first slice of an Annex B recovery point", () => { - // The slice's 4-byte start code keeps its leading zero byte, which now leads the inserted unit. - expect(insertRecoveryPoint(annexB([NAL.aud, NAL.sweep, NAL.slice]))).toEqual( - new Uint8Array([...annexB([NAL.aud, NAL.sweep]), 0, ...annexB([NAL.recovered, NAL.slice], [0, 0, 1])]), - ); - const short = [0, 0, 1]; - expect(insertRecoveryPoint(annexB([NAL.sps, NAL.pps, NAL.slice], short))).toEqual( - annexB([NAL.sps, NAL.pps, NAL.recovered, NAL.slice], short), - ); - }); - - it("leaves an IDR, a payload without a slice, and a malformed payload unchanged", () => { - for (const [payload, description] of [ - [avcc([NAL.sps, NAL.pps, NAL.idr]), AVCC4], - [annexB([NAL.sps, NAL.pps, NAL.idr]), undefined], - [avcc([NAL.sps, NAL.pps]), AVCC4], - [annexB([NAL.sweep]), undefined], - // A length running past the payload, and a truncated description. - [avcc([NAL.slice]).subarray(0, 6), AVCC4], - [avcc([NAL.slice]), new Uint8Array([0x01])], - ] as const) { - expect(insertRecoveryPoint(payload, description)).toBe(payload); - } - }); -}); - -for (const kind of ["legacy", "cmaf"] as const) { - describe(`Decoder opening a ${kind} warmup rendition`, () => { - const recovery = avcc([NAL.sweep, NAL.slice]); - const patched = avcc([NAL.sweep, NAL.recovered, NAL.slice]); - const delta = avcc([NAL.slice]); - const idr = avcc([NAL.sps, NAL.pps, NAL.idr]); - const at = (group: bigint, ms: number, key: boolean, payload: Uint8Array) => - sample(group, ms * 1000, key, payload); - - async function open(fields: Partial) { - const feeds = new Feeds(); - const playback = await guardedPlayback(kind, [], [], false, 0, { fields, feeds }); - return { - playback, - push: async (...reads: Read[]) => { - feeds.push(0, ...reads); - await microtasks(); - }, - }; - } - - it("rewrites only the recovery point that opens each codec", async () => { - const { playback, push } = await open({ warmup: Catalog.u53(300) }); - try { - expect(playback.codecs).toBe(1); - await push(at(10n, 1000, true, recovery), at(10n, 1033, false, delta)); - // The next group continues the open codec, so Chromium does not check it. - await push(at(11n, 1300, true, recovery)); - expect(playback.payloads[0]).toEqual([patched, delta, recovery]); - - // A skip keeps the open codec, so the group it resumes on is left alone. - await push(gap, at(13n, 1900, true, recovery)); - expect(playback.resets).toBe(0); - expect(playback.payloads[0].at(-1)).toEqual(recovery); - } finally { - playback.close(); - await microtasks(); - } - }); - - it("leaves an opening IDR unchanged", async () => { - const { playback, push } = await open({ warmup: Catalog.u53(300) }); - try { - await push(at(10n, 1000, true, idr)); - expect(playback.payloads[0]).toEqual([idr]); - } finally { - playback.close(); - await microtasks(); - } - }); - - it("leaves a rendition without a warmup unchanged", async () => { - const { playback, push } = await open({}); - try { - expect(playback.codecs).toBe(1); - await push(at(10n, 1000, true, recovery), gap, at(13n, 1900, true, recovery)); - expect(playback.resets).toBe(0); - expect(playback.payloads[0]).toEqual([recovery, recovery]); - } finally { - playback.close(); - await microtasks(); - } - }); - }); -} diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index fdadbb35d57..1ab52756b51 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -19,7 +19,7 @@ import { rotateVideoDimensions } from "@moq/video"; import { base64ToBytes } from "../base64"; import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; -import { DELAY_MAX, type Sync, type Track } from "../sync"; +import type { Sync, Track } from "../sync"; import { type DecoderConfig, hardwareAcceleration, type PlaybackIdentity, playbackIdentity } from "./config"; import { caughtUp, renditionJitter, switchJitter } from "./playhead"; import { Recovery } from "./recovery"; @@ -575,14 +575,7 @@ interface RenditionProps { key: Key; } -/** - * One rendition's playback through one WebCodecs decoder. - * - * A rendition with a catalog `warmup` (intra refresh, so no keyframes) joins that much earlier, - * through the container consumer's reach, and decodes from the group start without presenting the - * warmup that follows each join or gap, so its first picture is clean. Rust players do the same in - * `moq_video::decode::Consumer`. - */ +/** One rendition's playback through one WebCodecs decoder. */ class Rendition { error = new Signal(undefined); sync: Sync; @@ -599,15 +592,9 @@ class Rendition { timestamp = new Signal(undefined); frame = new Signal(undefined); - // The catalog `warmup`, capped like the delay so a publisher cannot hold the viewer without bound. - #warmup: Duration; - // The playout delay is where a cold join starts, and the shared max delay how late it may fall. The - // consumer's warmup reaches the start further back; the subscription widens its max by the same - // warmup, since the publisher skips what falls outside it. + // The playout delay is where a cold join starts, and the shared max delay how late it may fall. + // Its subscription and its consumer take the same delay. #delay: Getter; - #subscription: Getter; - // The running codec's join and gap state: which leading pictures to refuse, and which still warm up. - #recovery = new Recovery(); // Network jitter + decode buffer. buffered = new Signal([]); @@ -624,8 +611,6 @@ class Rendition { #codec?: string; // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; - // Whether the next chunk opens the codec, after a configure or a reset. - #opening = false; // A timeline restart makes the next decoded frame decide whether the codec resets. #resetPending = false; @@ -666,12 +651,7 @@ class Rendition { this.stats = props.stats; this.tally = props.tally; this.key = props.key; - this.#warmup = Duration.millis(this.config.warmup ?? 0).min(DELAY_MAX); this.#delay = new Derived([this.sync.out.delay, this.sync.out.maxDelay], (min, max) => ({ min, max })); - this.#subscription = new Derived([this.sync.out.delay, this.sync.out.maxDelay], (min, max) => ({ - min, - max: max.add(this.#warmup), - })); this.#predecessor = props.predecessor; this.#signals.run(this.#run.bind(this)); @@ -685,8 +665,7 @@ class Rendition { // Count a frame read from the track: its bytes even when rejected, and whether an accepted one // arrived too late for playout. Classifies ingress before decoder backpressure changes the - // delivery clock. A warming frame anchors no later playhead: the live frames behind it skip the - // clock to within the max delay, and buffered startup anchors on the first picture shown. + // delivery clock. #receive(frame: Container.ReceivedFrame, accepted: boolean): void { const timestamp = frame.timestamp as Duration; const late = accepted && this.clock.received(timestamp, frame.arrival); @@ -707,7 +686,6 @@ class Rendition { const consumer = new Container.Consumer(sub, { format, delay: this.#delay, - warmup: this.#warmup, predecessor: this.consumer ?? this.#predecessor, // The one decoder drives the clock, so it skips a hole once the clock passed it. playhead: this.sync, @@ -727,14 +705,10 @@ class Rendition { broadcast: this.broadcast, track: this.name, priority: Catalog.PRIORITY.video, - delay: this.#subscription, + delay: this.#delay, }); if (!sub) return; - // Each codec joins afresh. - const recovery = new Recovery(this.#warmup); - this.#recovery = recovery; - // Keep output tied to this codec lifetime even if the effect opens another run. const signal = effect.abort; const decoder = new VideoDecoder({ @@ -746,8 +720,6 @@ class Rendition { this.#count({ decoded: 1 }); const timestamp = Duration.micros(frame.timestamp); const generation = this.#generation; - // Still warming up after a join or gap: decoded for its references alone. - if (recovery.warming(timestamp)) return; if (this.#late(timestamp)) { // Late frame, don't render it. return; @@ -840,7 +812,7 @@ class Rendition { let previous: Duration | undefined; let latest: bigint | undefined; - const recovery = this.#recovery; + const recovery = new Recovery(); this.#spawn(effect, async () => { for (;;) { @@ -918,7 +890,7 @@ class Rendition { let previous: Duration | undefined; let latest: bigint | undefined; - const recovery = this.#recovery; + const recovery = new Recovery(); this.#spawn(effect, async () => { for (;;) { @@ -1032,7 +1004,6 @@ class Rendition { this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); - this.#recovery.restart(); // Restarts video's timeline, leaving audio's. this.clock.reset(); this.#resetPending = true; @@ -1057,23 +1028,13 @@ class Rendition { // Feed `frame` to the codec, noting the newest timestamp it holds. #submit(decoder: VideoDecoder, frame: Container.ReceivedFrame): void { this.#submitted = this.#submitted?.max(frame.timestamp) ?? frame.timestamp; - decoder.decode(this.#chunk(frame)); - } - - // The chunk for `frame`, run through the opening rewrite when it opens the codec. - #chunk(frame: Container.ReceivedFrame): EncodedVideoChunk { - let data = frame.payload; - if (this.#opening) { - this.#opening = false; - if (frame.keyframe && !this.#warmup.isZero() && isH264(this.config.codec)) { - data = insertRecoveryPoint(data, this.#description); - } - } - return new EncodedVideoChunk({ - type: frame.keyframe ? "key" : "delta", - data, - timestamp: frame.timestamp.asMicros(), - }); + decoder.decode( + new EncodedVideoChunk({ + type: frame.keyframe ? "key" : "delta", + data: frame.payload, + timestamp: frame.timestamp.asMicros(), + }), + ); } // Whether a picture at `timestamp` is behind the one already rendered. @@ -1096,7 +1057,6 @@ class Rendition { #configure(decoder: VideoDecoder): void { if (this.#codec === undefined) throw new Error("video: configured before the codec was accepted"); - this.#opening = true; this.#submitted = undefined; decoder.configure({ codec: this.#codec, @@ -1151,76 +1111,3 @@ class Rendition { }); } } - -function isH264(codec: string): boolean { - return codec.startsWith("avc1.") || codec.startsWith("avc3."); -} - -// An SEI NAL unit (type 6) holding one recovery point message (payload type 6, one byte): -// recovery_frame_cnt = 0, exact_match_flag = 0, broken_link_flag = 0, changing_slice_group_idc = 0, -// then payload alignment and the RBSP trailing bits. It needs no emulation prevention. -const RECOVERY_SEI = new Uint8Array([0x06, 0x06, 0x01, 0x84, 0x80]); - -/** - * Insert an SEI recovery point with `recovery_frame_cnt = 0` before the first slice of an H.264 - * access unit that has no IDR, so a codec can open on it. `description` is the avcC record when the - * payload is length-prefixed, and absent for Annex B. An IDR, a payload without a slice, or one that - * does not parse comes back unchanged. - * - * Chromium's WebCodecs checks the chunk that opens a codec, after `configure()` or `reset()`: an - * H.264 chunk sent as `key` must hold an IDR, or an SEI recovery point with `recovery_frame_cnt == - * 0` (`AnalyzeAnnexB` in `media/formats/mp4/avc.cc`). Otherwise `decode()` throws "A key frame is - * required". An intra-refresh group opens on a recovery point that counts a whole sweep, so every - * join and skip of a `warmup` rendition would fail. The extra message declares that one picture - * recovered, which is what a `warmup` rendition already accepts: the decoder hides the pictures - * after a join or skip until the warmup passes. The slices are unchanged; the message goes right before the first one, after any SEI the stream carries. - * @internal - */ -export function insertRecoveryPoint(payload: Uint8Array, description?: Uint8Array): Uint8Array { - if (description) { - // lengthSizeMinusOne, in the low two bits of the avcC record's fifth byte. - if (description.byteLength < 5) return payload; - const size = (description[4] & 0x03) + 1; - for (let offset = 0; offset + size <= payload.byteLength; ) { - let length = 0; - for (let i = 0; i < size; i++) length = length * 256 + payload[offset + i]; - const start = offset + size; - if (length === 0 || start + length > payload.byteLength) return payload; - const type = payload[start] & 0x1f; - if (type === 5) return payload; - if (type >= 1 && type <= 4) { - const prefix = new Uint8Array(size); - for (let i = 0, n = RECOVERY_SEI.byteLength; i < size; i++, n = Math.floor(n / 256)) { - prefix[size - 1 - i] = n % 256; - } - return splice(payload, offset, prefix, RECOVERY_SEI); - } - offset = start + length; - } - return payload; - } - - // Annex B: each NAL unit follows a 00 00 01 start code, with an optional leading zero byte. - for (let i = 0; i + 3 < payload.byteLength; i++) { - if (payload[i] !== 0 || payload[i + 1] !== 0 || payload[i + 2] !== 1) continue; - const type = payload[i + 3] & 0x1f; - if (type === 5) return payload; - if (type >= 1 && type <= 4) return splice(payload, i, new Uint8Array([0, 0, 1]), RECOVERY_SEI); - i += 2; - } - return payload; -} - -// `payload` with `parts` inserted at `at`. -function splice(payload: Uint8Array, at: number, ...parts: Uint8Array[]): Uint8Array { - const length = parts.reduce((sum, part) => sum + part.byteLength, payload.byteLength); - const out = new Uint8Array(length); - out.set(payload.subarray(0, at), 0); - let offset = at; - for (const part of parts) { - out.set(part, offset); - offset += part.byteLength; - } - out.set(payload.subarray(at), offset); - return out; -} diff --git a/js/watch/src/video/recovery.ts b/js/watch/src/video/recovery.ts index 483d41ca50f..63134fa3b37 100644 --- a/js/watch/src/video/recovery.ts +++ b/js/watch/src/video/recovery.ts @@ -1,65 +1,22 @@ -import { Duration } from "@moq/clock"; +import type { Duration } from "@moq/clock"; import type { Container } from "@moq/hang"; -/** - * Refuse leading pictures whose previous GOP was not delivered to this decoder, and hold back the - * catalog `warmup` of pictures after each join, which a recovery point is not clean before. @internal - */ +/** Refuse leading pictures whose previous GOP was not delivered to this decoder. @internal */ export class Recovery { #needed = true; #floor?: Duration; - readonly #warmup: Duration; - // The pictures still warming up since the last join or gap: from its recovery point, for the - // warmup. `passed` once the codec output a picture past it, so none of it can still come out. - #warming?: { start: Duration; end: Duration; passed: boolean }; - - constructor(warmup: Duration = Duration.ZERO) { - this.#warmup = warmup; - } /** Arm recovery after media was lost, preserving continuous groups and declared ends. */ gap(): void { this.#needed = true; } - /** - * A fresh encoder starts another timeline after a declared end. Its first picture references - * nothing earlier and needs no warmup, even after a join or a gap. - */ - restart(): void { - this.#needed = false; - this.#floor = undefined; - this.#warming = undefined; - } - /** Whether this access unit has the references available at the current delivery boundary. */ accept(frame: Container.ReceivedFrame): boolean { if (frame.keyframe) { - if (this.#needed && !this.#warmup.isZero()) { - // The codec may still hold pictures of a warmup it has not output past, which stay - // hidden: the hidden span reaches back over them rather than growing a list. - const previous = this.#warming; - const start = previous && !previous.passed ? previous.start.min(frame.timestamp) : frame.timestamp; - this.#warming = { start, end: frame.timestamp.add(this.#warmup), passed: false }; - } this.#floor = this.#needed ? frame.timestamp : undefined; this.#needed = false; } return this.#floor === undefined || frame.timestamp.ge(this.#floor); } - - /** - * Whether the decoded picture at `timestamp` is still warming up after the last join or gap, - * so it is decoded but not shown. Called on each picture the codec outputs, in presentation - * order. - */ - warming(timestamp: Duration): boolean { - const warming = this.#warming; - if (warming === undefined) return false; - if (timestamp.ge(warming.end)) { - warming.passed = true; - return false; - } - return timestamp.ge(warming.start); - } } diff --git a/quest/m2/hang04/README.md b/quest/m2/hang04/README.md index e4d7b2aab49..6a50f847c41 100644 --- a/quest/m2/hang04/README.md +++ b/quest/m2/hang04/README.md @@ -24,13 +24,9 @@ read the catalog as a frozen format: hang-03 is what the datatracker and readers are the first that check `version` at all. Each child's Plan records its own decisions. -The children land in any order. They overlap in three places, so whichever +The children land in any order. They overlap in two places, so whichever lands second merges main first: -- [Entry versions](/quest/m2/hang04/entry-version.md) and - [video warmup](/quest/m2/hang04/video-warmup.md) both edit the version - computation (`min_version` in `rs/hang/src/catalog/root.rs`, `minVersion` - in `js/hang/src/catalog/root.ts`) and the draft's Version section. - [The sync section](/quest/m2/hang04/sync.md) is born strict, under the rule [one strict rule](/quest/m2/hang04/shape.md) applies to the sections that exist today. diff --git a/quest/m2/hang04/entry-version.md b/quest/m2/hang04/entry-version.md index c2c143549e6..af3d992e038 100644 --- a/quest/m2/hang04/entry-version.md +++ b/quest/m2/hang04/entry-version.md @@ -50,7 +50,8 @@ Recommendations for the PR: IDs were skipped and the highest version among them, the per-entry twin of `Error::UnsupportedVersion` and JS `UnsupportedVersion`, so a player can say "upgrade to see everything". -- With [video warmup](/quest/m2/hang04/video-warmup.md) out of hang-04, the +- With video warmup out of hang-04 + ([moq-dev/moq#8960](https://github.com/moq-dev/moq/pull/8960)), the renamed `track` is the only trigger, and nothing in hang-04 raises the root `version`. Keep the root check and its tests: it is what a reshaped root will use. @@ -72,5 +73,4 @@ Wire: the hang-04 catalog JSON, unpublished. ## Related -- [Video warmup leaves hang-04](/quest/m2/hang04/video-warmup.md) - removes the other trigger from the same lines - [Audio languages](/quest/m3/audio-lang.md) - a later rendition a 0.22 player should skip can be marked with this diff --git a/quest/m2/hang04/shape.md b/quest/m2/hang04/shape.md index 5b325a7272d..607472f7e5b 100644 --- a/quest/m2/hang04/shape.md +++ b/quest/m2/hang04/shape.md @@ -77,9 +77,8 @@ For the PR: every gateway shares; `JsonSchema` and `BinarySchema` gain the `instance` the code carries. - Not here: the duplicated MPEG-TS SI section goes with - [the MPEG-TS section](/quest/m2/hang04/m2ts.md), the dual-decoder - sentences with [video warmup](/quest/m2/hang04/video-warmup.md), and the - marker rules with + [the MPEG-TS section](/quest/m2/hang04/m2ts.md), and the marker rules + with [media consumers](/quest/m2/time-addressing/consumers.md). Tests: for each key, a wrong-shaped value refuses the catalog in both diff --git a/quest/m2/hang04/video-warmup.md b/quest/m2/hang04/video-warmup.md deleted file mode 100644 index 8ae3d5de379..00000000000 --- a/quest/m2/hang04/video-warmup.md +++ /dev/null @@ -1,102 +0,0 @@ -# [M] Video warmup leaves hang-04, and the players stop reading it - -## Goal - -moq-hang-04 defines `warmup` for audio renditions only. A video rendition -has no `warmup`, every video group starts with a keyframe again, and no -`version` rule mentions it. No player carries a video warmup path: `@moq/watch`, -`moq_video::decode`, `moq play`, moq-gst, and the bindings read a video -rendition as one whose groups each decode on their own. Audio `warmup` -(Opus pre-roll) and the container consumer's reach that serves it are -unchanged. Video warmup returns in a later catalog version with the -producer that writes it -([the intra-refresh epic](/quest/m4/intra-refresh/README.md)). - -## Plan - -Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): - -- ✅ Video `warmup` is dropped from hang-04 and audio `warmup` stays - (rejected: keep it and make a passing Safari and Firefox run a gate for - the cut; keep it as is). No producer writes it, two engines have never - run it, and a 0.22 player that claims catalog version 4 would accept a - later producer's rendition and mis-decode it instead of refusing it. -- ✅ The same quest deletes the players' video warmup path (rejected: keep - the path merged with no catalog that can carry the field; close the PR - that built it). - -A correction to the second decision. The question the maintainer answered -said [moq-dev/moq#8857](https://github.com/moq-dev/moq/pull/8857) ("a -warmup rendition plays through one decoder") was open, and the pick read -"delete the path, park #8857". It was not open: it merged 2026-11-15 -22:35Z, about ten hours before the question. So there is no PR to park and -no dual decoder left to delete (#8857 removed it). What this quest deletes -is the one-decoder path #8857 landed. The option the maintainer turned -down, "land #8857 anyway, so the one-decoder path exists and is tested -before any producer does", is the state main is in. The pick stands as -recorded, but it was made on a wrong fact, so the PR says this at the top -of its body for the maintainer to confirm or reverse before it merges. - -Facts found by the audit: - -- No producer sets a video `warmup`: the only writers are the Opus - importers and encoders. The import that would write it, - [H.264 import](/quest/m4/intra-refresh/h264-import.md), moved to m4 on - 2026-11-12 for want of a consumer. -- In browsers the path rests on inserting a recovery-point SEI for - Chromium (`insertRecoveryPoint`, `js/watch/src/video/decoder.ts`). - Firefox and Safari are unverified, and Chromium refuses a non-IRAP H.265 - frame sent as `key`, so an HEVC refresh rendition fails - ([second decoder](/quest/m4/intra-refresh/dual-decoder.md)). -- The draft makes video `warmup` above zero one of two triggers for - `version` 4 (`min_version`, `rs/hang/src/catalog/root.rs`; `minVersion`, - `js/hang/src/catalog/root.ts`), and lets a video group start at "a - picture from which decoding converges within that duration" (the - Container section's group rules). - -For the PR, what goes (find the rest with `grep -rn -i warmup rs js doc`): - -- Catalog: `warmup` on `hang::catalog::Video` - (`rs/hang/src/catalog/video/mod.rs`) and the video schema in - `js/hang/src/catalog/video.ts`; the warmup trigger in both version - computations; the MSF mapping for video - (`rs/moq-mux/src/catalog/msf/consumer.rs`, `js/watch/src/msf.ts`). A - video `warmup` in a catalog is then an unknown field, ignored like any - other. -- Browser: the warmup hold in `js/watch/src/video/recovery.ts` (its - leading-picture refusal stays), `insertRecoveryPoint` and the widened - subscription in `decoder.ts`, the `config.ts` field, - `js/watch/bench/warmup.ts` if it measures only video, and the paragraph - in `doc/lib/js/watch.md`. -- Rust: the warmup in `rs/moq-video/src/decode/consumer.rs`; the video - buffer's hold in `rs/moq-cli/src/play/{video,buffer,media}.rs`; the video - `warmup` in moq-ffi (`rs/moq-ffi/src/{video,media}.rs`) and every wrapper, - and `warmup_us` with `has_warmup` on moq-c's video config - (`rs/moq-c/src/api.rs`); moq-gst's video branch - (`rs/moq-gst/src/source/imp.rs`). -- What stays: `moq_mux::container::Playout { delay, warmup }`, the - consumer's reach in both languages, and every audio use. Check that no - audio test lost its coverage by leaning on a video case. -- Draft: the `warmup` field section describes audio pre-roll only, and - loses the sentences about a consumer that "also decodes from an earlier - group start" and "the earlier decoder's frames", which describe the dual - decoder #8857 removed. The group rule returns to "a video group MUST - start with a keyframe". The Version section and the hang-04 changelog - drop video `warmup`. -- Revert by hand rather than with `git revert`: #8857 also deleted the - dual decoder (`rs/moq-video/src/decode/select.rs`, 808 lines), which - must stay deleted. - -Tests: a catalog whose video rendition carries `warmup` parses with the -field ignored and no `version`; the players' join and skip tests pass with -the hold gone; the audio warmup tests are untouched. - -Public API: breaking. Rust `hang::catalog::Video::warmup`, the video -decoder's warmup options, the moq-ffi and moq-c video `warmup` fields; JS -the video schema's `warmup` and the watch config field. Wire: the hang-04 -catalog JSON and group rule, unpublished. - -## Related - -- [Video warmup returns with its producer](/quest/m4/intra-refresh/catalog-warmup.md) - restores this from history as the next catalog version -- [A player skips a catalog entry newer than it reads](/quest/m2/hang04/entry-version.md) - how a 0.22 player will refuse that later rendition; edits the same version lines diff --git a/quest/m4/intra-refresh/README.md b/quest/m4/intra-refresh/README.md index 61ed817d308..d526b28ba38 100644 --- a/quest/m4/intra-refresh/README.md +++ b/quest/m4/intra-refresh/README.md @@ -30,9 +30,9 @@ Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): ✅ video `warmup` is not in moq-hang-04 (rejected: freezing it with Chromium-only evidence; gating the cut on a Safari and Firefox run). No producer wrote it, and a 0.22 player claiming to read it would mis-decode a later producer's -rendition instead of refusing it. The m2 quest -`quest/m2/hang04/video-warmup.md` removed the field and the one-decoder -player path that +rendition instead of refusing it. +[moq-dev/moq#8960](https://github.com/moq-dev/moq/pull/8960) removed the +field and the one-decoder player path that [moq-dev/moq#8857](https://github.com/moq-dev/moq/pull/8857) had landed the day before. [Catalog warmup](/quest/m4/intra-refresh/catalog-warmup.md) brings both back, from that history, as the next catalog version, and every diff --git a/quest/m4/intra-refresh/catalog-warmup.md b/quest/m4/intra-refresh/catalog-warmup.md index c7e6c3948bd..324320eb69b 100644 --- a/quest/m4/intra-refresh/catalog-warmup.md +++ b/quest/m4/intra-refresh/catalog-warmup.md @@ -22,7 +22,8 @@ rendition. It returns here, with the producers this epic adds. - Restore from history, not from scratch: #8857 is the design (one decoder, the container consumer's warmup reach, `Recovery`'s hold, the recovery-point SEI Chromium needs), and the PR that removed it before the - cut is its inverse. Read both, then re-apply by hand against the tree as + cut, [moq-dev/moq#8960](https://github.com/moq-dev/moq/pull/8960), is its + inverse. Read both, then re-apply by hand against the tree as it is. - The field comes back in the hang draft's next version, with the group rule that lets a video group start at a picture from which decoding diff --git a/rs/hang/fixtures/catalog-parity.json b/rs/hang/fixtures/catalog-parity.json index c64baa31b66..8499221d256 100644 --- a/rs/hang/fixtures/catalog-parity.json +++ b/rs/hang/fixtures/catalog-parity.json @@ -2143,23 +2143,6 @@ }, "refused": true }, - { - "name": "video warmup 0", - "catalog": { - "video": { - "renditions": { - "ok": { - "codec": "avc1.64001f" - }, - "probe": { - "codec": "avc1.64001f", - "warmup": 0 - } - } - } - }, - "video": ["ok", "probe"] - }, { "name": "audio warmup 0", "catalog": { @@ -2181,23 +2164,6 @@ }, "audio": ["ok", "probe"] }, - { - "name": "video warmup 9007199254740991", - "catalog": { - "video": { - "renditions": { - "ok": { - "codec": "avc1.64001f" - }, - "probe": { - "codec": "avc1.64001f", - "warmup": 9007199254740991 - } - } - } - }, - "video": ["ok", "probe"] - }, { "name": "audio warmup 9007199254740991", "catalog": { @@ -2219,23 +2185,6 @@ }, "audio": ["ok", "probe"] }, - { - "name": "video warmup 9007199254740992", - "catalog": { - "video": { - "renditions": { - "ok": { - "codec": "avc1.64001f" - }, - "probe": { - "codec": "avc1.64001f", - "warmup": 9007199254740992 - } - } - } - }, - "refused": true - }, { "name": "audio warmup 9007199254740992", "catalog": { @@ -2258,7 +2207,7 @@ "refused": true }, { - "name": "video warmup -1", + "name": "video warmup is an unknown field", "catalog": { "video": { "renditions": { @@ -2272,7 +2221,7 @@ } } }, - "refused": true + "video": ["ok", "probe"] }, { "name": "audio warmup -1", @@ -2295,23 +2244,6 @@ }, "refused": true }, - { - "name": "video warmup 1.5", - "catalog": { - "video": { - "renditions": { - "ok": { - "codec": "avc1.64001f" - }, - "probe": { - "codec": "avc1.64001f", - "warmup": 1.5 - } - } - } - }, - "refused": true - }, { "name": "audio warmup 1.5", "catalog": { diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 155e2753cd0..da0b4f7fcae 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -313,11 +313,7 @@ impl Catalog { || renamed(&self.json.tracks) || renamed(&self.binary.tracks); - // moq-hang-04: a reader that ignores video `warmup` assumes every group starts at a - // keyframe, and hands its decoder one that starts at a converging picture instead. - let warmup = (self.video.renditions.values()).any(|config| config.warmup.is_some_and(|w| !w.is_zero())); - - (renamed || warmup).then_some(4) + renamed.then_some(4) } /// Copy the media sections into a catalog with no application extension. @@ -783,7 +779,6 @@ mod test { reorder: None, optimize_for_latency: None, container: Container::Legacy, - warmup: None, jitter: Some(std::time::Duration::from_millis(100)), delay: Some(std::time::Duration::from_millis(200)), }, @@ -1583,12 +1578,19 @@ mod test { chat.track = Some("chat.json".into()); data.json.insert("chat", chat).unwrap(); assert_eq!(version(&data), Some(4.into())); + } - let mut warmup = Catalog::<()>::default(); - let mut config = rendition(); - config.warmup = Some(std::time::Duration::from_millis(500)); - warmup.video.renditions.insert("hd".into(), config); - assert_eq!(version(&warmup), Some(4.into())); + /// moq-hang-04 defines `warmup` for audio only, so a video rendition's is an unknown field: read + /// past, never written back, and no reason for a `version`. + #[test] + fn a_video_warmup_is_ignored() { + let json = r#"{"video":{"renditions":{"hd":{"codec":"vp8","container":{"kind":"legacy"},"warmup":500}}}}"#; + let catalog = Catalog::<()>::from_str(json).unwrap(); + assert_eq!(catalog.video.renditions["hd"].codec, crate::catalog::VideoCodec::VP8); + assert_eq!( + catalog.to_json().unwrap(), + r#"{"video":{"renditions":{"hd":{"codec":"vp8","container":{"kind":"legacy"}}}}}"# + ); } /// The version is computed, never carried: republishing a catalog that claimed more than it diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index e2faf3e660a..02c79392416 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -335,12 +335,6 @@ pub struct VideoConfig { #[serde(default)] pub jitter: Option, - /// After a non-continuous join, decode from the group start, present at start plus warmup, and join that much earlier. - /// Serialized as whole milliseconds, rounded up so presentation never starts early. - #[serde_as(as = "Option")] - #[serde(default)] - pub warmup: Option, - /// How far this rendition's frames reach the transport behind the broadcast's other /// renditions: a constant offset, either supplied by the publisher or measured as the least /// its frames trailed the others' over a window of a second of media, keeping the largest @@ -394,7 +388,6 @@ impl VideoConfig { optimize_for_latency: None, container: Container::default(), jitter: None, - warmup: None, delay: None, } } @@ -445,19 +438,6 @@ mod test { use super::*; - #[test] - fn warmup_round_trips() { - let mut config = VideoConfig::new(VideoCodec::VP8); - assert!(serde_json::to_value(&config).unwrap().get("warmup").is_none()); - for millis in [0, 80, 1_000] { - config.warmup = Some(std::time::Duration::from_millis(millis)); - let encoded = serde_json::to_value(&config).unwrap(); - assert_eq!(encoded["warmup"], millis); - let decoded: VideoConfig = serde_json::from_value(encoded).unwrap(); - assert_eq!(decoded.warmup, config.warmup); - } - } - #[test] fn reorder_keeps_zero_distinct_from_absent() { let mut config = VideoConfig::new(VideoCodec::VP8); @@ -471,18 +451,6 @@ mod test { } } - #[test] - fn warmup_rounds_up() { - let mut config = VideoConfig::new(VideoCodec::VP8); - for (nanos, millis) in [(1, 1), (499_999, 1), (333_200_000, 334)] { - config.warmup = Some(std::time::Duration::from_nanos(nanos)); - let encoded = serde_json::to_value(&config).unwrap(); - assert_eq!(encoded["warmup"], millis); - let decoded: VideoConfig = serde_json::from_value(encoded).unwrap(); - assert_eq!(decoded.warmup, Some(std::time::Duration::from_millis(millis))); - } - } - #[test] fn ranked_orders_by_picture_then_bitrate() { fn rendition(size: Option<(u32, u32)>, bitrate: Option) -> VideoConfig { diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 8c7bb4d3d25..c2d0b199986 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -437,12 +437,6 @@ pub struct moq_video_config { /// Higher values are preferred; compatibility transcodes use -1 and the default is 0. pub preference: i32, - /// Microseconds of media a decoder needs before its output is clean: after a join, decode - /// from the group start and present at the start plus this. The media consumers already - /// start this much earlier. Meaningless unless `has_warmup`. - pub warmup_us: u64, - /// Whether `warmup_us` is present; absent means none. - pub has_warmup: bool, /// The broadcast holding the rendition's track, relative to the catalog's own broadcast /// (`./source`), or NULL when it is the catalog's broadcast. pub broadcast: *const c_char, @@ -523,7 +517,9 @@ pub struct moq_audio_config { pub jitter_us: u64, /// Whether `jitter_us` is present; absent means frames flush at once. pub has_jitter: bool, - /// As [moq_video_config]'s `warmup_us`. + /// Microseconds of media a decoder needs before its output is clean, such as Opus pre-roll: + /// after a join, decode from the group start and present at the start plus this. The media + /// consumers already start this much earlier. Meaningless unless `has_warmup`. pub warmup_us: u64, /// Whether `warmup_us` is present; absent means none. pub has_warmup: bool, @@ -3415,9 +3411,6 @@ pub unsafe extern "C" fn moq_publish_video_config(broadcast: u32, config: *const video.coded_width = (config.coded_width > 0).then_some(config.coded_width); video.coded_height = (config.coded_height > 0).then_some(config.coded_height); video.container = unsafe { parse_container(&config.container)? }; - video.warmup = config - .has_warmup - .then(|| std::time::Duration::from_micros(config.warmup_us)); (video.broadcast, video.instance, video.track) = unsafe { parse_reference( (config.broadcast, config.broadcast_len), @@ -4720,10 +4713,6 @@ pub unsafe extern "C" fn moq_consume_catalog_section( /// `user_data` is never touched again, so release `user_data` there. The /// terminal callback fires even after [moq_consume_video_cancel]. /// -/// The catalog rendition's `warmup` reaches that much further back for decoder history and -/// increases the subscription's max delay by the same amount. The steady-state skip budget -/// stays at `delay.max`; the caller's decoder must consume the warmup before presenting output. -/// /// Returns a non-zero handle to the track on success, or a negative code on failure. /// /// # Safety diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 0ba8738f4ef..9330461829b 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -88,8 +88,11 @@ pub(crate) trait Media: hang::catalog::Reference + Clone { /// The delivery priority its tracks subscribe with. const PRIORITY: i8; - /// The history needed to prime a decoder. - fn warmup(&self) -> Duration; + /// The history needed to prime a decoder: none unless the catalog declares a `warmup`, which + /// moq-hang-04 defines for audio alone. + fn warmup(&self) -> Duration { + Duration::ZERO + } /// Its renditions, keyed by ID. fn renditions(catalog: &moq_mux::catalog::hang::Catalog) -> &BTreeMap; @@ -101,10 +104,6 @@ pub(crate) trait Media: hang::catalog::Reference + Clone { impl Media for hang::catalog::VideoConfig { const PRIORITY: i8 = hang::catalog::PRIORITY.video; - fn warmup(&self) -> Duration { - self.warmup.unwrap_or_default() - } - fn renditions(catalog: &moq_mux::catalog::hang::Catalog) -> &BTreeMap { &catalog.video.renditions } @@ -503,8 +502,6 @@ impl Consume { .map(|label| label.as_ptr() as *const c_char) .unwrap_or(std::ptr::null()), label_len: config.label.as_ref().map_or(0, String::len), - warmup_us: micros(config.warmup), - has_warmup: config.warmup.is_some(), broadcast, broadcast_len, instance, @@ -1563,16 +1560,13 @@ mod tests { } impl MediaConsumer { - async fn new(video: bool, warmup: Option, delay: Delay) -> Self { + async fn new(warmup: Option, delay: Delay) -> Self { let broadcast = moq_net::broadcast::Info::new().produce(); let mut request = broadcast.reserve_track("media").unwrap(); let mut catalog = moq_mux::catalog::hang::Catalog::default(); let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); audio.warmup = warmup; catalog.audio.renditions.insert("media".into(), audio); - let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); - config.warmup = warmup; - catalog.video.renditions.insert("media".into(), config); let reads = Box::new(kio::Queue::new()); let callback = unsafe { OnStatus::new(std::ptr::from_ref(reads.as_ref()).cast_mut().cast(), Some(on_read)).unwrap() }; @@ -1593,25 +1587,15 @@ mod tests { }) .unwrap(); // Exercise the shared C media path on this runtime so time stays controlled. - let task = if video { - state - .consume - .media::(snapshot, 0, delay, callback) - } else { - state - .consume - .media::(snapshot, 0, delay, callback) - } - .unwrap(); + let task = state + .consume + .media::(snapshot, 0, delay, callback) + .unwrap(); state.consume.catalog_free(snapshot).unwrap(); task }; let initial_delay = request.subscription_changed().await.unwrap().delay; - let track = request.accept(hang::container::track_info(if video { - hang::catalog::PRIORITY.video - } else { - hang::catalog::PRIORITY.audio - })); + let track = request.accept(hang::container::track_info(hang::catalog::PRIORITY.audio)); Self { _broadcast: broadcast, track, @@ -1658,28 +1642,26 @@ mod tests { /// Warmup changes the first request too, before the container can update it. #[tokio::test(start_paused = true)] async fn media_subscription_includes_warmup() { - for video in [false, true] { - for warmup in [None, Some(Duration::ZERO), Some(Duration::from_millis(300))] { - for max in [ - None, - Some(Duration::from_millis(200)), - Some(Duration::MAX - Duration::from_millis(100)), - ] { - let delay = Delay { - min: Some(Duration::from_millis(200)), - max, - }; - let consumer = MediaConsumer::new(video, warmup, delay).await; - assert_eq!( - consumer.initial_delay, - // The request aggregate represents Duration::MAX as unlimited. - delay.with_max( - max.map(|max| max.saturating_add(warmup.unwrap_or_default())) - .filter(|max| *max != Duration::MAX) - ) - ); - consumer.cancel().await; - } + for warmup in [None, Some(Duration::ZERO), Some(Duration::from_millis(300))] { + for max in [ + None, + Some(Duration::from_millis(200)), + Some(Duration::MAX - Duration::from_millis(100)), + ] { + let delay = Delay { + min: Some(Duration::from_millis(200)), + max, + }; + let consumer = MediaConsumer::new(warmup, delay).await; + assert_eq!( + consumer.initial_delay, + // The request aggregate represents Duration::MAX as unlimited. + delay.with_max( + max.map(|max| max.saturating_add(warmup.unwrap_or_default())) + .filter(|max| *max != Duration::MAX) + ) + ); + consumer.cancel().await; } } } @@ -1687,33 +1669,31 @@ mod tests { /// Newest-first delivery must leave time for history covering min plus warmup. #[tokio::test(start_paused = true)] async fn media_warmup_reaches_history() { - for video in [false, true] { - for warmup in [None, Some(Duration::ZERO), Some(Duration::from_millis(300))] { - let delay = Delay { - min: Some(Duration::from_millis(200)), - max: Some(Duration::from_millis(200)), - }; - let mut consumer = MediaConsumer::new(video, warmup, delay).await; - consumer.write(5, 5_000_000); - consumer.write(4, 4_700_000); - let first = kio::clock::timeout(Duration::from_millis(50), consumer.frame()).await; - let warming = warmup.is_some_and(|warmup| !warmup.is_zero()); - assert_eq!(first.is_err(), warming, "video={video}, warmup={warmup:?}"); - consumer.write(3, 4_400_000); - drop(consumer.track.clone().finish().unwrap()); - let mut stamps: Vec<_> = first.ok().flatten().into_iter().collect(); - while let Some(stamp) = consumer.frame().await { - stamps.push(stamp); - } - assert_eq!( - stamps, - if warming { - vec![4_400_000, 4_700_000, 5_000_000] - } else { - vec![4_700_000, 5_000_000] - } - ); + for warmup in [None, Some(Duration::ZERO), Some(Duration::from_millis(300))] { + let delay = Delay { + min: Some(Duration::from_millis(200)), + max: Some(Duration::from_millis(200)), + }; + let mut consumer = MediaConsumer::new(warmup, delay).await; + consumer.write(5, 5_000_000); + consumer.write(4, 4_700_000); + let first = kio::clock::timeout(Duration::from_millis(50), consumer.frame()).await; + let warming = warmup.is_some_and(|warmup| !warmup.is_zero()); + assert_eq!(first.is_err(), warming, "warmup={warmup:?}"); + consumer.write(3, 4_400_000); + drop(consumer.track.clone().finish().unwrap()); + let mut stamps: Vec<_> = first.ok().flatten().into_iter().collect(); + while let Some(stamp) = consumer.frame().await { + stamps.push(stamp); } + assert_eq!( + stamps, + if warming { + vec![4_400_000, 4_700_000, 5_000_000] + } else { + vec![4_700_000, 5_000_000] + } + ); } } } diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index b790ff6215f..f0baf3b6927 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -837,8 +837,6 @@ fn publish_catalog_config_invalid_broadcast() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -911,8 +909,6 @@ fn publish_media_owns_its_rendition_before_the_first_keyframe() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -970,8 +966,6 @@ fn publish_video_config_replaces_its_own_rendition() { coded_width: 640, coded_height: 360, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1023,8 +1017,6 @@ fn manual_video_identity_change_is_refused() { coded_height: 360, container: moq_container::default(), preference: 0, - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1256,8 +1248,6 @@ fn publish_catalog_roundtrip() { coded_width: width, coded_height: height, container: moq_container::default(), - warmup_us: 250_000, - has_warmup: true, broadcast: reference.as_ptr().cast(), broadcast_len: reference.len(), instance: instance.as_ptr().cast(), @@ -1280,8 +1270,6 @@ fn publish_catalog_roundtrip() { coded_width: width, coded_height: height, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1361,8 +1349,6 @@ fn publish_catalog_roundtrip() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1385,7 +1371,6 @@ fn publish_catalog_roundtrip() { ); assert_eq!(video_cfg.coded_width, 1920); assert_eq!(video_cfg.coded_height, 1080); - assert_eq!((video_cfg.warmup_us, video_cfg.has_warmup), (250_000, true)); // Normalized: a reader sees the plain relative path. assert_eq!( borrowed_string(video_cfg.broadcast, video_cfg.broadcast_len).as_deref(), @@ -1520,8 +1505,6 @@ fn a_half_specified_coded_size_round_trips() { coded_width: 1920, coded_height: 0, // absent, not "zero pixels tall" container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1549,8 +1532,6 @@ fn a_half_specified_coded_size_round_trips() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1623,8 +1604,6 @@ fn raw_loc_video_uses_the_declared_catalog_container() { init: std::ptr::null(), init_len: 0, }, - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1653,8 +1632,6 @@ fn raw_loc_video_uses_the_declared_catalog_container() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -1817,8 +1794,6 @@ fn unpublishable_catalog_containers_are_rejected() { coded_width: 0, coded_height: 0, container, - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -3643,8 +3618,6 @@ fn media_consumers_cancel_while_subscribing() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -5202,8 +5175,6 @@ fn local_publish_consume() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), @@ -5958,8 +5929,6 @@ fn video_publish_consume() { coded_width: 0, coded_height: 0, container: moq_container::default(), - warmup_us: 0, - has_warmup: false, broadcast: std::ptr::null(), broadcast_len: 0, instance: std::ptr::null(), diff --git a/rs/moq-cli/benches/play_buffer.rs b/rs/moq-cli/benches/play_buffer.rs index aabc17d4038..bc1834faa4d 100644 --- a/rs/moq-cli/benches/play_buffer.rs +++ b/rs/moq-cli/benches/play_buffer.rs @@ -32,11 +32,7 @@ fn main() { for _ in 0..1_000 { let mut buffer = buffer::Buffer::default(); for frame in &frames { - buffer.push( - black_box(frame.clone()), - Duration::from_millis(delay_ms), - Duration::ZERO, - ); + buffer.push(black_box(frame.clone()), Duration::from_millis(delay_ms)); peak = peak.max(buffer.bytes); } black_box(buffer); diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 3052767f6ad..e9e603a9e44 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -15,13 +15,6 @@ pub(super) struct Buffer { /// The keyframe the buffer last resumed at. A picture presenting before it references /// media the buffer never had, so it is neither decoded nor shown. pub floor: Option, - /// Pictures presenting before this are still warming up after a join or a skip: - /// decoded for their references and never shown. They age out the warmup later - /// than other media, since they trail the live edge by it on purpose. - pub present: Option, - /// Whether the next keyframe resumes after a join or a loss, so a catalog `warmup` - /// applies. Cleared for an encoder's first picture, which has nothing before it. - pub joined: bool, waiting_keyframe: bool, pub ended: bool, } @@ -35,8 +28,6 @@ impl Default for Buffer { oldest: VecDeque::new(), generation: 0, floor: None, - present: None, - joined: true, // A join resumes at its first keyframe, like any other break. waiting_keyframe: true, ended: false, @@ -45,14 +36,6 @@ impl Default for Buffer { } impl Buffer { - /// A buffer for a fresh encoder, whose first keyframe has nothing before it to warm up from. - pub fn fresh() -> Self { - Self { - joined: false, - ..Self::default() - } - } - /// The earliest presentation time still buffered, wherever it sits in /// decode order. pub fn oldest(&self) -> Option { @@ -68,19 +51,13 @@ impl Buffer { Some(frame) } - /// Append `frame`, aging out what trails the newest by more than `max_delay`. After a - /// join or a loss, the pictures before the resuming keyframe plus `warmup` are warming. - pub fn push(&mut self, frame: Frame, max_delay: Duration, warmup: Duration) { + pub fn push(&mut self, frame: Frame, max_delay: Duration) { let edge = *self.edge.get_or_insert(frame.timestamp); // Timestamp has no `Ord`; one that doesn't compare keeps the edge. self.edge = Some(if frame.timestamp > edge { frame.timestamp } else { edge }); if frame.keyframe { if self.waiting_keyframe { self.floor = Some(frame.timestamp); - self.present = self - .floor - .filter(|_| self.joined) - .and_then(|floor| warmed(floor, warmup)); } self.waiting_keyframe = false; } @@ -89,19 +66,10 @@ impl Buffer { } self.index(frame); let edge = self.edge.unwrap(); - let present = self.present; - // An unscaled timestamp has no age, so it never goes stale. A warming one may - // trail by the warmup too. - let stale = |oldest: &moq_net::Timestamp| { - let warming = present.is_some_and(|present| *oldest < present); - let budget = match warming { - true => max_delay.saturating_add(warmup), - false => max_delay, - }; - match (edge.as_micros(), oldest.as_micros()) { - (Ok(edge), Ok(oldest)) => edge.saturating_sub(oldest) > budget.as_micros(), - _ => false, - } + // An unscaled timestamp has no age, so it never goes stale. + let stale = |oldest: &moq_net::Timestamp| match (edge.as_micros(), oldest.as_micros()) { + (Ok(edge), Ok(oldest)) => edge.saturating_sub(oldest) > max_delay.as_micros(), + _ => false, }; if self.oldest.front().is_some_and(stale) { // Dropping a reference picture invalidates the rest of its GOP. Resume @@ -113,9 +81,6 @@ impl Buffer { self.generation += 1; self.waiting_keyframe = self.frames.is_empty(); self.floor = self.frames.front().map(|frame| frame.timestamp); - // Resuming past a loss warms up again, even at a keyframe that has yet to arrive. - self.joined = true; - self.present = self.floor.and_then(|floor| warmed(floor, warmup)); // The retained keyframe's leading pictures referenced what was just dropped. let mut frames = std::mem::take(&mut self.frames); frames.retain(|frame| !self.below_floor(frame)); @@ -142,21 +107,6 @@ impl Buffer { } } -/// Where a `warmup` from `floor` ends, at its scale, or `None` when there is none. Rounds -/// up to the next tick, so a coarse scale never hides less than the whole warmup. -fn warmed(floor: moq_net::Timestamp, warmup: Duration) -> Option { - if warmup.is_zero() { - return None; - } - let scale = floor.scale()?; - let ticks = warmup - .as_nanos() - .checked_mul(u128::from(scale.as_u64()))? - .div_ceil(1_000_000_000); - let warmup = moq_net::Timestamp::new(u64::try_from(ticks).ok()?, scale).ok()?; - floor.checked_add(warmup).ok() -} - #[cfg(test)] mod tests { use super::*; @@ -174,20 +124,20 @@ mod tests { fn eviction_keeps_only_decodable_media_inside_the_age_window() { let mut buffer = Buffer::default(); let age = Duration::from_millis(100); - buffer.push(frame(0, true), age, Duration::ZERO); - buffer.push(frame(50, false), age, Duration::ZERO); - buffer.push(frame(100, true), age, Duration::ZERO); - buffer.push(frame(150, false), age, Duration::ZERO); + buffer.push(frame(0, true), age); + buffer.push(frame(50, false), age); + buffer.push(frame(100, true), age); + buffer.push(frame(150, false), age); assert_eq!(buffer.frames.front().unwrap().timestamp.as_millis().unwrap(), 100); assert_eq!(buffer.bytes, 14); buffer.pop(); assert_eq!(buffer.bytes, 7); - buffer.push(frame(260, false), age, Duration::ZERO); + buffer.push(frame(260, false), age); assert!(buffer.frames.is_empty(), "no retained keyframe"); assert_eq!(buffer.bytes, 0); - buffer.push(frame(280, false), age, Duration::ZERO); + buffer.push(frame(280, false), age); assert!(buffer.frames.is_empty(), "cannot resume on a dependent picture"); - buffer.push(frame(300, true), age, Duration::ZERO); + buffer.push(frame(300, true), age); assert_eq!(buffer.bytes, 7); } @@ -203,15 +153,15 @@ mod tests { (90, false), (110, false), ] { - buffer.push(frame(ms, keyframe), age, Duration::ZERO); + buffer.push(frame(ms, keyframe), age); } - buffer.push(frame(95, false), age, Duration::ZERO); + buffer.push(frame(95, false), age); let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); assert_eq!(kept, [100, 110], "80, 90 and 95 reference the evicted group"); assert_eq!(buffer.bytes, 14); assert_eq!(buffer.oldest().unwrap().as_millis().unwrap(), 100); - buffer.push(frame(200, true), age, Duration::ZERO); - buffer.push(frame(180, false), age, Duration::ZERO); + buffer.push(frame(200, true), age); + buffer.push(frame(180, false), age); assert_eq!(buffer.frames.len(), 4, "a continuous group keeps its leading pictures"); } @@ -219,7 +169,7 @@ mod tests { fn reordered_timestamps_keep_decode_order_and_the_newest_edge() { let mut buffer = Buffer::default(); for (ms, keyframe) in [(0, true), (99, false), (33, false), (66, false)] { - buffer.push(frame(ms, keyframe), Duration::from_millis(100), Duration::ZERO); + buffer.push(frame(ms, keyframe), Duration::from_millis(100)); } assert_eq!( buffer @@ -236,85 +186,16 @@ mod tests { let mut buffer = Buffer::default(); let age = Duration::from_millis(100); for (ms, keyframe) in [(0, true), (99, false), (33, false)] { - buffer.push(frame(ms, keyframe), age, Duration::ZERO); + buffer.push(frame(ms, keyframe), age); } buffer.pop(); - buffer.push(frame(150, false), age, Duration::ZERO); + buffer.push(frame(150, false), age); assert!( buffer.frames.is_empty(), "the 33ms picture expired behind the 99ms reference" ); assert_eq!(buffer.bytes, 0); - buffer.push(frame(160, true), age, Duration::ZERO); + buffer.push(frame(160, true), age); assert_eq!(buffer.frames.len(), 1); } - - /// After a join the warmup's pictures trail the live edge on purpose, so they age out the - /// warmup later than other media, and an eviction past a loss warms up again from the - /// keyframe it resumes at. - #[test] - fn a_join_ages_out_the_warmup_a_warmup_later() { - let mut buffer = Buffer::default(); - let (age, warmup) = (Duration::from_millis(100), Duration::from_millis(100)); - for ms in [0, 50, 100, 150, 200] { - buffer.push(frame(ms, ms % 100 == 0), age, warmup); - } - let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); - assert_eq!( - kept, - [0, 50, 100, 150, 200], - "the warmup trails by the age plus the warmup" - ); - assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 100); - - buffer.push(frame(250, false), age, warmup); - let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); - assert_eq!( - kept, - [200, 250], - "the warmup past its budget was evicted, then the stale group" - ); - assert_eq!( - buffer.present.unwrap().as_millis().unwrap(), - 300, - "resuming past a loss warms up" - ); - } - - /// An eviction that empties a fresh encoder's segment is a loss: the keyframe it resumes at - /// warms up. - #[test] - fn an_emptied_fresh_segment_warms_up_at_its_next_keyframe() { - let mut buffer = Buffer::fresh(); - let warmup = Duration::from_millis(100); - buffer.push(frame(0, true), Duration::ZERO, warmup); - assert!(buffer.present.is_none()); - buffer.push(frame(50, false), Duration::ZERO, warmup); - assert!(buffer.frames.is_empty(), "aged out with no keyframe to resume at"); - buffer.push(frame(300, true), Duration::ZERO, warmup); - assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 400); - } - - /// A coarse scale rounds the warmup up a tick rather than hiding less than all of it. - #[test] - fn a_coarse_scale_rounds_the_warmup_up() { - let floor = moq_net::Timestamp::from_scale(10, 10).unwrap(); - let present = warmed(floor, Duration::from_millis(150)).unwrap(); - assert_eq!(present.value(), 12, "150 ms at 10 Hz ends at the second tick"); - let present = warmed(floor, Duration::from_millis(200)).unwrap(); - assert_eq!(present.value(), 12, "an exact warmup does not round"); - assert!(warmed(moq_net::Timestamp::unscaled(3), Duration::from_millis(150)).is_none()); - } - - /// An encoder's first picture has nothing before it to warm up from. - #[test] - fn a_fresh_encoder_needs_no_warmup() { - let mut buffer = Buffer::fresh(); - buffer.push(frame(0, true), Duration::ZERO, Duration::from_millis(100)); - assert!(buffer.present.is_none()); - // A gap starts a fresh buffer, which resumes after a loss. - let mut buffer = Buffer::default(); - buffer.push(frame(500, true), Duration::ZERO, Duration::from_millis(100)); - assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 600, "a loss warms up"); - } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index c2444d3a923..dbe664d3070 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -297,17 +297,14 @@ impl Media { min: Some(self.args.video_delay()), max: Some(max_delay), }; - // An intra-refresh rendition decodes from a group earlier, presenting from - // its recovery point plus the warmup, as `moq_video::decode::Consumer` does. - let warmup = config.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); - let playout = moq_mux::container::Playout::new(delay, warmup); + let playout = moq_mux::container::Playout::new(delay, Duration::ZERO); let track = rendition.track(hang::catalog::Reference::track(config, name))?; // The whole delay from the first request, so a cold join never states a // zero minimum before the container's update. The live edge, from the // groups each hop holds from its live feed, never a warm copy's backlog, // so it names no cached group for the shared publisher subscription to // rewind to. - let mut subscriber = track + let subscriber = track .subscribe( playout.subscription( moq_net::track::Subscription::default() @@ -316,36 +313,15 @@ impl Media { ), ) .await?; - // A warmup widens the subscription to the group before the live edge, the - // one containing the key just below it, as starts are sparse. The live - // edge is known only once the feed is live: a warm copy still waiting on - // its source reports nothing yet. The wait ends with the track. Every wire - // carries a time start, where a relative one past the edge is refused - // before lite-07; there it reaches the edge alone. - if !warmup.is_zero() - && let Some(edge) = subscriber.largest().await - { - let start = moq_net::track::Anchor::Time(edge.group.saturating_sub(1)); - let subscription = subscriber - .subscription() - .with_ranges([moq_net::track::Range { start, end: None }]); - subscriber.update(subscription)?; - } let format = catalog::hang::Container::try_from(config)?; - // The playout delay is the target latency, and the max how late a group may - // be; the start reaches the warmup further back. + // The playout delay is the target latency, and the max how late a group may be. let track = playout.consumer(subscriber, format); - Ok::<_, anyhow::Error>((track, decoder, warmup)) + Ok::<_, anyhow::Error>((track, decoder)) } .await; match opened { - Ok((track, decoder, warmup)) => { - tracing::info!( - track = name, - decoder = decoder.name(), - ?warmup, - "playing video rendition" - ); + Ok((track, decoder)) => { + tracing::info!(track = name, decoder = decoder.name(), "playing video rendition"); let (delay, held) = tokio::sync::watch::channel(None); let video = Video { presentation: self.presentation.clone(), @@ -355,7 +331,6 @@ impl Media { max_delay, min_delay: self.args.video_delay(), delay: held, - warmup, }; let (stop, stopped) = tokio::sync::oneshot::channel(); let (retire, retired) = tokio::sync::oneshot::channel(); @@ -1608,7 +1583,6 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, - warmup: Duration::ZERO, }; let audio_task = tokio::spawn(async move { play_audio(consumer, audio_play, std::future::pending()).await.unwrap(); @@ -1744,7 +1718,6 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, - warmup: Duration::ZERO, }; let audio_task = tokio::spawn(async move { play_audio(consumer, audio_play, std::future::pending()) @@ -2046,7 +2019,6 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, - warmup: Duration::ZERO, }; let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), @@ -2813,7 +2785,6 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, - warmup: Duration::ZERO, } .run(track, decoder), ); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 43fea509254..c3414ff922e 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -57,9 +57,6 @@ pub(super) struct Video { pub min_delay: Duration, /// The rendition's catalog `delay`, held as this track's playout target. pub delay: tokio::sync::watch::Receiver>, - /// The rendition's catalog `warmup`: after a join or a skip, pictures before the - /// recovery point plus this decode but never show. - pub warmup: Duration, } /// The production sink and the deterministic buffered decoder used by tests. @@ -140,8 +137,7 @@ impl Video { shared.saturating_sub(self.min_delay) } - /// A subscription's budget with `held` on top of its own. The container adds the - /// warmup to the subscription's max itself. + /// A subscription's budget with `held` on top of its own. fn budget(&self, held: Duration) -> moq_net::track::Delay { moq_net::track::Delay { min: Some(self.min_delay + held), @@ -186,8 +182,6 @@ impl Video { // The group of a declared end no later group has followed yet. The media // buffered before the end still plays out. let mut end = None; - // Whether a frame was read yet, so only the first can be the track's start. - let mut started = false; loop { // Stop reading while media queued behind the front segment arrived more @@ -242,33 +236,21 @@ impl Video { buffers.remove(index); } } - // A fresh encoder opens on a picture of its own, with nothing before it - // to warm up from. - buffers.push_back(Segment { - media: Buffer::fresh(), - ..Default::default() - }); + buffers.push_back(Segment::default()); // Another timeline: its arrivals stop holding the clock until this one. role.reset(); } - // The clock anchors on the arrival furthest ahead of its media, so a warmup - // trailing the live edge never holds playback back. if role.received(frame.timestamp, Instant::now()) { self.output.send(Event::Wake); } let back = buffers.back_mut().unwrap(); - // A group with nothing below it is the track's first, decoded from the - // encoder's first picture. - if !std::mem::replace(&mut started, true) && track.is_skipped(0..group) { - back.joined = false; - } - back.push(frame, self.max_delay + held, self.warmup); + back.push(frame, self.max_delay + held); back.arrived.get_or_insert_with(Instant::now); false } - // Lost media discards every picture still buffered, and what follows warms - // up again. An end declared before the loss still holds, as on the audio - // side: a later group comes from the replacement encoder. + // Lost media discards every picture still buffered. An end declared + // before the loss still holds, as on the audio side: a later group comes + // from the replacement encoder. Some(Read::Gap { .. }) => { buffers.clear(); buffers.push_back(Segment::default()); @@ -429,24 +411,19 @@ impl Video { } } // A codec may return pictures it held across the keyframe that resumed a - // skipped timeline, which no longer have a slot, and pictures still warming up - // after it, which would show refresh stripes. - let (floor, present) = { - let buffers = buffers.lock().unwrap(); - let front = buffers.front().unwrap(); - (front.floor, front.present) - }; - // A picture that doesn't compare with a bound isn't known to precede it. - let below = |at: moq_net::Timestamp, bound: Option| { - bound.is_some_and( - |bound| matches!((at.as_micros(), bound.as_micros()), (Ok(at), Ok(bound)) if at < bound), + // skipped timeline, which no longer have a slot. + let floor = buffers.lock().unwrap().front().unwrap().floor; + // A picture that doesn't compare with the floor isn't known to precede it. + let below = |at: moq_net::Timestamp| { + floor.is_some_and( + |floor| matches!((at.as_micros(), floor.as_micros()), (Ok(at), Ok(floor)) if at < floor), ) }; while let Some(frame) = decoder.receive()? { // A codec returns display order, so a picture held before one it // returned was dropped rather than delayed. held.retain(|held| held.0 > frame.timestamp); - if !below(frame.timestamp, floor) && !below(frame.timestamp, present) { + if !below(frame.timestamp) { pending.push_back(frame); } } @@ -556,7 +533,6 @@ mod tests { max_delay, min_delay: max_delay, delay: tokio::sync::watch::channel(None).1, - warmup: Duration::ZERO, } } @@ -1237,72 +1213,6 @@ mod tests { ); } - /// Play a group starting at `first` holding media from 1 s, and the live group from 2 s, - /// through a rendition with a one-second warmup. Returns each picture shown, in - /// milliseconds, and when after the groups were published. - async fn warmup_join(first: u64) -> Vec<(u128, Duration)> { - let delay = Duration::from_secs(2); - let warmup = Duration::from_secs(1); - let media = media(delay); - let broadcast = moq_net::broadcast::Info::new().produce(); - let published = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let playout = moq_mux::container::Playout::new( - moq_net::track::Delay { - min: Some(delay), - max: Some(delay), - }, - warmup, - ); - let subscriber = published - .consume() - .subscribe(playout.subscription(moq_net::track::Subscription::default())) - .await - .unwrap(); - let wire = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - for (start, stamps) in [(first, 1_000..=1_900), (2_000, 2_000..=2_200)] { - let mut group = published.append_group(start).unwrap(); - let frames: Vec<_> = stamps - .step_by(100) - .enumerate() - .map(|(index, ms)| frame(ms, index == 0)) - .collect(); - wire().write(&mut group, &frames).unwrap(); - group.close().unwrap(); - } - let _published = published.finish().unwrap(); - let begin = Instant::now(); - let video = Video { - warmup, - ..playback(&media, delay) - }; - let task = tokio::spawn(video.run(playout.consumer(subscriber, wire()), Buffered::new(0))); - let mut shown = Vec::new(); - for _ in 0..10_000 { - tokio::task::yield_now().await; - if let Some(timestamp) = media.output.present(&media) { - shown.push((timestamp.as_millis().unwrap(), Instant::now() - begin)); - } - tokio::time::advance(Duration::from_millis(1)).await; - } - task.await.unwrap().unwrap(); - shown - } - - /// A join mid-track decodes the group before the live one as its warmup and never shows - /// it, so the live group's pictures play when they would have anyway. The track's first - /// group decodes from the encoder's first picture, so nothing warms up there. - #[tokio::test(start_paused = true)] - async fn a_warmup_join_shows_only_the_recovered_pictures() { - let joined = warmup_join(1_000).await; - let start = warmup_join(0).await; - let millis = |shown: &[(u128, Duration)]| shown.iter().map(|(ms, _)| *ms).collect::>(); - assert_eq!(millis(&joined), [2_000, 2_100, 2_200]); - assert_eq!(millis(&start), (1_000..=2_200).step_by(100).collect::>()); - assert_eq!(joined, start[10..], "the warmup moved the live group's schedule"); - } - /// Play one burst of 33 ms pictures, given by slot in decode order, through a /// codec holding `depth` pictures, and check the window shows every one on /// time. diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 7eaacedcf2c..78b1a990747 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -154,11 +154,6 @@ pub struct MoqVideo { pub framerate: Option, /// The framing used to carry this rendition. pub container: MoqContainer, - /// How much media a decoder needs before its output is clean: after a join, decode from the - /// group start, present at the start plus this, and join that much earlier. Absent means none. - /// [`MoqMediaTrackConsumer::subscribe`] and the decoding consumers start this much earlier. - #[uniffi(default = None)] - pub warmup: Option, } /// An audio rendition and the configuration needed to decode it. @@ -643,7 +638,6 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog std::time::Duration { match self { - Self::Video { config, .. } => config.warmup, + Self::Video { .. } => None, Self::Audio { config, .. } => config.warmup, } .unwrap_or_default() @@ -1568,14 +1563,12 @@ mod read_tests { const WARMUP: Duration = Duration::from_millis(300); const DELAY: Duration = Duration::from_millis(200); - /// Subscribes to a rendition whose catalog warmup is [`WARMUP`], at a [`DELAY`] min and max delay. - async fn subscribe_warmup( - video: bool, - ) -> ( + /// Subscribes to an audio rendition whose catalog warmup is [`WARMUP`], at a [`DELAY`] min and + /// max delay. + async fn subscribe_warmup() -> ( moq_net::broadcast::Producer, moq_net::track::Producer, Arc, - moq_mux::container::Kind, ) { use crate::consumer::{MoqDelay, MoqSubscription, moq_subscription_defaults}; @@ -1585,25 +1578,10 @@ mod read_tests { let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); audio.warmup = Some(WARMUP); catalog.audio.renditions.insert("media".into(), audio); - let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); - config.warmup = Some(WARMUP); - catalog.video.renditions.insert("media".into(), config); let catalog = convert_catalog(&catalog); - let (rendition, kind) = match video { - true => ( - MoqRendition::Video { - id: "media".into(), - config: catalog.video["media"].clone(), - }, - moq_mux::container::Kind::Video, - ), - false => ( - MoqRendition::Audio { - id: "media".into(), - config: catalog.audio["media"].clone(), - }, - moq_mux::container::Kind::Audio, - ), + let rendition = MoqRendition::Audio { + id: "media".into(), + config: catalog.audio["media"].clone(), }; let reader = MoqMediaTrackConsumer::subscribe( &MoqBroadcastConsumer::new(broadcast.consume()), @@ -1620,19 +1598,17 @@ mod read_tests { ) .await .unwrap(); - (broadcast, track, reader, kind) + (broadcast, track, reader) } /// A catalog rendition's warmup survives conversion and widens the raw subscription past the /// caller's max delay, keeping the min delay as asked. #[tokio::test] async fn raw_media_catalog_warmup_widens_the_subscription() { - for video in [false, true] { - let (_broadcast, track, _reader, _) = subscribe_warmup(video).await; - let subscription = track.subscription().unwrap(); - assert_eq!(subscription.delay.min, Some(DELAY), "video={video}"); - assert_eq!(subscription.delay.max, Some(DELAY + WARMUP), "video={video}"); - } + let (_broadcast, track, _reader) = subscribe_warmup().await; + let subscription = track.subscription().unwrap(); + assert_eq!(subscription.delay.min, Some(DELAY)); + assert_eq!(subscription.delay.max, Some(DELAY + WARMUP)); } /// The warmup starts the reader that much further back than the min delay. @@ -1642,38 +1618,36 @@ mod read_tests { /// writes and the read would rightly skip the groups this checks. #[tokio::test(start_paused = true)] async fn raw_media_catalog_warmup_reaches_the_subscription() { - for video in [false, true] { - let (_broadcast, track, reader, kind) = subscribe_warmup(video).await; - let mut media = reader.reads.take(|_| Ok(())).unwrap(); - let write = |sequence: u64, micros| { - let mut group = track.create_group(sequence.into()).unwrap(); - moq_mux::catalog::hang::Container::Legacy(kind) - .write( - &mut group, - &[moq_mux::container::Frame { - timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), - payload: bytes::Bytes::from_static(b"media"), - keyframe: true, - duration: None, - }], - ) - .unwrap(); - group.close().unwrap(); - }; - // Newest first, all before the first read. Group 4 covers the min delay alone, so - // without the warmup the start would skip group 3; only the warmup's reach keeps it. - write(5, 5_000_000); - write(4, 4_700_000); - write(3, 4_400_000); - let _track = track.finish().unwrap(); - let mut micros = Vec::new(); - while let Some(read) = media.read().await.unwrap() { - if let moq_mux::container::Read::Frame { frame, .. } = read { - micros.push(frame.timestamp.as_micros().unwrap()); - } + let (_broadcast, track, reader) = subscribe_warmup().await; + let mut media = reader.reads.take(|_| Ok(())).unwrap(); + let write = |sequence: u64, micros| { + let mut group = track.create_group(sequence.into()).unwrap(); + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio) + .write( + &mut group, + &[moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + payload: bytes::Bytes::from_static(b"media"), + keyframe: true, + duration: None, + }], + ) + .unwrap(); + group.close().unwrap(); + }; + // Newest first, all before the first read. Group 4 covers the min delay alone, so + // without the warmup the start would skip group 3; only the warmup's reach keeps it. + write(5, 5_000_000); + write(4, 4_700_000); + write(3, 4_400_000); + let _track = track.finish().unwrap(); + let mut micros = Vec::new(); + while let Some(read) = media.read().await.unwrap() { + if let moq_mux::container::Read::Frame { frame, .. } = read { + micros.push(frame.timestamp.as_micros().unwrap()); } - assert_eq!(micros, [4_400_000, 4_700_000, 5_000_000], "video={video}"); } + assert_eq!(micros, [4_400_000, 4_700_000, 5_000_000]); } #[tokio::test(start_paused = true)] @@ -1885,7 +1859,7 @@ impl MoqMediaTrackConsumer { /// sibling broadcast or under another track name subscribes to the right track, and fails /// one whose broadcast is proven to serve another stream than its instance. /// - /// A rendition's [`warmup`](MoqVideo::warmup) starts the read that much further back than + /// An audio rendition's [`warmup`](MoqAudio::warmup) starts the read that much further back than /// the subscription's min delay, and widens the max delay the publisher serves by as much, /// while the reader still skips at the max delay alone. #[uniffi::constructor] diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 7b55eaeedf9..9e865a08b66 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -317,7 +317,6 @@ fn rendition(id: &str, container: MoqContainer) -> MoqRendition { enabled: true, framerate: None, container, - warmup: None, }, } } @@ -9746,7 +9745,6 @@ async fn video_subscribe_configuration_boundary() { display_aspect: None, bitrate: None, framerate: None, - warmup: None, }; let result = MoqVideoConsumer::subscribe( &consumer, diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 51026fe02bd..374332f58ff 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -1312,7 +1312,6 @@ fn video_config(catalog_video: crate::media::MoqVideo) -> Result cmaf); the broadcast and track identify the media, apart from its catalog ID. /// -/// The catalog `warmup` rides along without being part of the shape: it matters only to a join, -/// so a changed one reaches the next reader without restarting the current one. +/// An audio rendition's catalog `warmup` rides along without being part of the shape: it matters +/// only to a join, so a changed one reaches the next reader without restarting the current one. #[derive(Clone)] struct Shape { caps: gst::Caps, @@ -1109,7 +1109,7 @@ fn reconcile( TrackKind::Video, video_caps(config), &config.container, - config.warmup, + None, config, ); } @@ -3029,54 +3029,54 @@ mod session_tests { super::RUNTIME.block_on(session).unwrap().unwrap(); } - /// A rendition's catalog warmup widens its subscription past the session max delay, so the - /// publisher keeps the media its decoder needs before the start. + /// An audio rendition's catalog warmup widens its subscription past the session max delay, so + /// the publisher keeps the media its decoder needs before the start. #[test] fn a_rendition_subscribes_with_its_catalog_warmup() { let warmup = Duration::from_millis(300); - for kind in [TrackKind::Video, TrackKind::Audio] { - let element = element(); - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let mut dynamic = broadcast.requests(); - let mut catalog = - moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - { - let mut guard = catalog.modify().unwrap(); - match kind { - TrackKind::Video => { - let mut video = video_rendition(); - video.warmup = Some(warmup); - guard.video.renditions = BTreeMap::from([("media".to_string(), video)]); - } - TrackKind::Audio => { - let mut audio = audio_rendition(); - audio.warmup = Some(warmup); - guard.audio.renditions = BTreeMap::from([("media".to_string(), audio)]); - } - } - } + let element = element(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut dynamic = broadcast.requests(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + { + let mut guard = catalog.modify().unwrap(); + let mut audio = audio_rendition(); + audio.warmup = Some(warmup); + guard.audio.renditions = BTreeMap::from([("media".to_string(), audio)]); + } - let max_delay = Duration::from_secs(5); - let (shutdown, mut cancelled) = watch::channel(false); - let weak = element.downgrade(); - let source = moq_mux::Source::new(origin().consume(), broadcast.consume()); - let session = super::RUNTIME.spawn(async move { - let stopping = cancelled.clone(); - Session::new(weak, stopping, max_delay) - .follow_catalog(source, &mut cancelled) - .await - }); + let max_delay = Duration::from_secs(5); + let (shutdown, mut cancelled) = watch::channel(false); + let weak = element.downgrade(); + let source = moq_mux::Source::new(origin().consume(), broadcast.consume()); + let session = super::RUNTIME.spawn(async move { + let stopping = cancelled.clone(); + Session::new(weak, stopping, max_delay) + .follow_catalog(source, &mut cancelled) + .await + }); - let request = await_request(&mut dynamic); - assert_eq!( - request.subscription().unwrap().delay.max, - Some(max_delay + warmup), - "{kind:?}" - ); + let request = await_request(&mut dynamic); + assert_eq!(request.subscription().unwrap().delay.max, Some(max_delay + warmup)); - let _ = shutdown.send(true); - super::RUNTIME.block_on(session).unwrap().unwrap(); - } + let _ = shutdown.send(true); + super::RUNTIME.block_on(session).unwrap().unwrap(); + } + + /// A changed warmup alone is the same shape, so it never restarts the reader playing the + /// rendition, while the shape a plan carries holds the new value for the next reader. + #[test] + fn a_warmup_is_not_part_of_the_shape() { + gst::init().unwrap(); + let shape = |warmup| super::Shape { + caps: gst::Caps::new_empty_simple("audio/x-opus"), + container: Container::Legacy, + track: "audio".into(), + broadcast: None, + instance: None, + warmup, + }; + assert!(shape(Duration::from_millis(300)) == shape(Duration::from_millis(600))); } /// A rendition delisted while its reader is still subscribing ends that reader, and answering @@ -3876,8 +3876,6 @@ mod session_tests { video: Option>, /// The video track once [`Self::finish_video`] finished it. finished: Option>, - /// The video track, for what its readers subscribe with. - track: moq_net::track::Producer, start: kio::clock::Instant, } @@ -3921,7 +3919,6 @@ mod session_tests { let mut guard = catalog.modify().unwrap(); guard.video.renditions = BTreeMap::from([("video".to_string(), config)]); } - let track = video.clone(); let video = moq_mux::container::Producer::new( video, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), @@ -3932,7 +3929,6 @@ mod session_tests { catalog, video: Some(video), finished: None, - track, start: kio::clock::now(), } } @@ -4541,83 +4537,6 @@ mod session_tests { recorder.assert_one_pad_held(); } - /// A warmup-only catalog update neither restarts the reader playing the rendition nor is lost: - /// the next instance's reader subscribes with the updated warmup. - #[test] - fn a_warmup_update_keeps_the_reader_and_reaches_the_next_one() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let mut config = video_rendition(); - config.broadcast = Some(moq_net::path::Relative::new("./cam").unwrap()); - config.warmup = Some(Duration::from_millis(300)); - let mut room = Publisher::new(&origin, Default::default(), config); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - assert_eq!( - first.track.subscription().unwrap().delay.max, - Some(DEFAULT_MAX_DELAY + Duration::from_millis(300)) - ); - - // Change only the warmup, then list an audio rendition in the same catalog. Its pad shows - // the run took the warmup too, since a catalog snapshot carries both, and the cam follower - // takes a plan before any instance event. - let audio = room - .broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let mut audio = moq_mux::container::Producer::new( - audio, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), - ); - { - let mut guard = room.catalog.modify().unwrap(); - guard.video.renditions.get_mut("video").unwrap().warmup = Some(Duration::from_millis(600)); - guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); - } - audio.write(room.frame(9)).unwrap(); - await_pad(&element, "audio_"); - assert_eq!(recorder.written(&mut first, 2), pad); - - let mut follow = watch(&origin, "cam"); - await_event(&mut follow, |event| { - matches!( - event, - moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::Start, - .. - } - ) - }); - drop(first); - await_event(&mut follow, |event| { - matches!( - event, - moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, - .. - } - ) - }); - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!(recorder.written(&mut second, 3), pad, "the return moved to another pad"); - assert_eq!( - second.track.subscription().unwrap().delay.max, - Some(DEFAULT_MAX_DELAY + Duration::from_millis(600)) - ); - stop(session); - - let seen = recorder.seen(); - assert_eq!( - recorder.count(|seen| matches!(seen, Seen::Start(start) if *start == pad)), - 1, - "only the next instance starts a new stream on the pad: {seen:?}" - ); - } - /// A rendition that joins a sibling instance after another started on it shares that instance's /// timeline, so both map the instance's timestamps to one running time. #[test] diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 805ae842346..42ae763fafc 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -318,9 +318,8 @@ fn gap(edge: std::time::Duration, resume: Timestamp) -> Result, - /// The catalog's `warmup`, capped at [`moq_mux::container::DELAY_MAX`]. - warmup: std::time::Duration, - /// Pictures before this, in nanoseconds, are still warming up after a join or a - /// gap, so they are decoded but never returned. Nanoseconds, because a backend may - /// hand a picture back at another scale than its access unit's. - present: Option, - /// Whether a frame has been read, so a later keyframe is not the track's first. - started: bool, } impl Consumer { @@ -224,8 +206,7 @@ impl Consumer { /// Read the next decoded picture or boundary, or `None` after the track ends and /// the decoder's buffered tail has been drained. /// - /// Join and delivery-gap recovery trim leading pictures whose previous GOP is absent, - /// and, with a catalog `warmup`, the pictures before the recovery point plus the warmup. + /// Join and delivery-gap recovery trim leading pictures whose previous GOP is absent. /// /// Cancel-safe: a decode in flight when the read is dropped is kept and /// finished by the next read. A codec error fails the read. @@ -289,7 +270,7 @@ impl Consumer { impl Reader { /// Subscribe per `options`, starting at the live edge when the caller asked for - /// [`Start::Latest`], or at the group before it with a catalog `warmup`. + /// [`Start::Latest`]. async fn open( broadcast: &moq_net::broadcast::Consumer, catalog: &VideoConfig, @@ -303,9 +284,7 @@ impl Reader { err => err, })?; - let warmup = catalog.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); - // The subscription's max carries the warmup, and the start reaches it further back. - let playout = moq_mux::container::Playout::new(options.delay, warmup); + let playout = moq_mux::container::Playout::new(options.delay, std::time::Duration::ZERO); let track = broadcast.track(hang::catalog::Reference::track(catalog, name))?; // The whole delay from the first request, as the container keeps it, so the publisher // never sees a stated zero min before the container's update. @@ -330,17 +309,9 @@ impl Reader { let mut subscriber = track.subscribe(subscription).await?; // The live edge is known only once the feed is live: a warm copy still waiting on its // source reports nothing yet. Opening waits for it, or the track's end, as `Start::Latest` - // promises. A warmup widens the same subscription to the group before it, before - // anything is read, so the start can reach the warmup back. - if options.start == Start::Latest - && let Some(edge) = subscriber.largest().await - && !warmup.is_zero() - { - let start = moq_net::track::Anchor::Time(edge.group.saturating_sub(1)); - let subscription = subscriber - .subscription() - .with_ranges([moq_net::track::Range { start, end: None }]); - subscriber.update(subscription)?; + // promises. + if options.start == Start::Latest { + subscriber.largest().await; } let track = subscriber; // The catalog says how the track is framed, and it is not always the legacy @@ -348,8 +319,7 @@ impl Reader { // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(catalog)?; // On a cold network join the live edge's stream often lands first. The container starts - // at the group covering the live edge less the min delay and the warmup, waiting for it, - // so the decoder still gets the group its warmup needs. + // at the group covering the live edge less the min delay, waiting for it. let track = playout.consumer(track, container); let reader = Self { @@ -360,9 +330,6 @@ impl Reader { drained: false, recover: true, leading_floor: None, - warmup, - present: None, - started: false, }; Ok(reader) } @@ -379,7 +346,7 @@ impl Reader { } let mux_frame = match self.track.read().await? { - Some(Encoded::Frame { group, frame }) => Some((group, frame)), + Some(Encoded::Frame { frame, .. }) => Some(frame), // Drained so the pictures from before the boundary all come out ahead of it, // rather than whenever the media after it pushes them through. The decoder // stays usable and restarts on the next group's keyframe. @@ -389,19 +356,16 @@ impl Reader { self.recover = true; continue; } - // A fresh encoder has no preceding GOP to reference and nothing to warm up from, - // even after a join or a gap, and its timeline may start over below the warmup's. + // A fresh encoder has no preceding GOP to reference, so no recovery. Some(Encoded::End { end, .. }) => { self.drain().await?; self.pending.push_back((Read::End { end }, None)); - self.recover = false; - self.present = None; continue; } None => None, }; - let Some((group, mux_frame)) = mux_frame else { + let Some(mux_frame) = mux_frame else { // The flag goes up only once the tail is in hand, so a read // dropped before the drain ran retries it rather than reporting // an end the stream has not reached. Flushing twice is safe: the @@ -424,17 +388,7 @@ impl Reader { continue; }; - // A group with nothing below it is the track's first, decoded from the encoder's - // first picture, so it needs no warmup. - let track_start = !std::mem::replace(&mut self.started, true) && self.track.is_skipped(0..group); if mux_frame.keyframe { - if self.recover && !self.warmup.is_zero() { - // A frame with no time has no presentation to hold back. - let start = mux_frame.timestamp.as_nanos().ok(); - self.present = start - .filter(|_| !track_start) - .map(|start| start.saturating_add(self.warmup.as_nanos())); - } self.leading_floor = self.recover.then_some(mux_frame.timestamp); self.recover = false; } @@ -493,11 +447,6 @@ impl Reader { }); Some(arrived) }); - // Still warming up: decoded for its references, never shown. - let warming = self.present.zip(picture.timestamp.as_nanos().ok()); - if warming.is_some_and(|(present, at)| at < present) { - continue; - } self.pending.push_back((Read::Frame(picture), arrival)); } Ok(()) @@ -1024,7 +973,7 @@ mod tests { let (broadcast, track) = live_track(); let mut consumer = Consumer::new( &broadcast.consume(), - &warmup_catalog(None), + &catalog(), "video", Options { decoder: Config { @@ -1077,7 +1026,7 @@ mod tests { let (broadcast, track) = live_track(); let mut consumer = Consumer::new( &broadcast.consume(), - &warmup_catalog(None), + &catalog(), "video", Options { decoder: Config { @@ -1340,18 +1289,13 @@ mod tests { /// One frame per second. const SECOND: u64 = 1_000_000; - /// A sweep of four frames. - const WARMUP: std::time::Duration = std::time::Duration::from_secs(4); - - fn warmup_catalog(warmup: Option) -> VideoConfig { - let mut catalog = VideoConfig::new(hang::catalog::H264 { + fn catalog() -> VideoConfig { + VideoConfig::new(hang::catalog::H264 { inline: true, profile: 0x42, constraints: 0, level: 30, - }); - catalog.warmup = warmup; - catalog + }) } fn live_track() -> (moq_net::broadcast::Producer, moq_net::track::Producer) { @@ -1362,148 +1306,6 @@ mod tests { (broadcast, track) } - /// Append frames at `seconds` to `group`, the first frame of a sweep being its recovery point. - fn write(group: &mut moq_net::group::Producer, seconds: &[u64]) { - let wire = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - let frames: Vec<_> = seconds - .iter() - .map(|second| moq_mux::container::Frame { - timestamp: Timestamp::from_micros(second * SECOND).unwrap(), - duration: None, - payload: Bytes::from_static(b"access unit"), - keyframe: *second % 4 == 0, - }) - .collect(); - wire.write(group, &frames).unwrap(); - } - - /// Append a group at `start` holding frames at `seconds`, skipping the starts since the last. - fn group(track: &mut moq_net::track::Producer, start: u64, seconds: &[u64]) -> moq_net::group::Producer { - let mut group = track.append_group(start).unwrap(); - write(&mut group, seconds); - group - } - - /// A consumer of a rendition with a four-second warmup and a zero max delay. - async fn warm(broadcast: &moq_net::broadcast::Producer, start: Start) -> Consumer { - Consumer::new( - &broadcast.consume(), - &warmup_catalog(Some(WARMUP)), - "video", - Options { - decoder: Config { - kind: Kind::Named(probe::IMMEDIATE_NAME.into()), - ..Config::new() - }, - start, - delay: Delay { - max: Some(std::time::Duration::ZERO), - ..Delay::default() - }, - }, - ) - .await - .unwrap() - } - - /// Read one picture, in seconds. - async fn read(consumer: &mut Consumer) -> u64 { - let frame = kio::clock::timeout(std::time::Duration::from_secs(10), consumer.next_frame()) - .await - .expect("a frame within ten seconds") - .unwrap() - .expect("the track is live"); - let seconds = u64::try_from(frame.timestamp.as_micros().unwrap()).unwrap() / SECOND; - assert!(consumer.arrival().is_some(), "picture {seconds} lost its arrival"); - seconds - } - - /// A join mid-track starts a group before the live edge, on one subscription whose max - /// holds the warmup, and shows nothing until the recovery point plus the warmup. Groups - /// are keyed by their start, so the one before the live edge lies a whole group's span - /// below it: the start counts a group back, not one below the live edge's start. - #[tokio::test] - async fn a_join_starts_a_group_earlier_and_presents_after_the_warmup() { - let (broadcast, mut track) = live_track(); - let published = track.clone(); - group(&mut track, 4 * SECOND, &[4, 5, 6, 7]).close().unwrap(); - let mut live = group(&mut track, 8 * SECOND, &[8, 9]); - - let mut consumer = warm(&broadcast, Start::Latest).await; - let subscription = published.subscription().expect("the consumer subscribed"); - assert_eq!(subscription.delay.max, Some(WARMUP), "the max does not hold the warmup"); - assert_eq!( - subscription.ranges, - vec![moq_net::track::Range { - start: moq_net::track::Anchor::Time(8 * SECOND - 1), - end: None, - }], - "the subscription does not reach the group before the live edge" - ); - - // 4 through 7 decode as the warmup, never shown. - assert_eq!(read(&mut consumer).await, 8); - assert_eq!(read(&mut consumer).await, 9); - write(&mut live, &[10]); - assert_eq!(read(&mut consumer).await, 10); - } - - /// A catalog warmup past the cap widens the subscription by the cap alone. - #[tokio::test] - async fn a_huge_warmup_reaches_back_by_the_cap() { - let (broadcast, mut track) = live_track(); - let published = track.clone(); - group(&mut track, 0, &[0]).close().unwrap(); - let _consumer = Consumer::new( - &broadcast.consume(), - &warmup_catalog(Some(std::time::Duration::from_secs(3600))), - "video", - Options { - decoder: Config { - kind: Kind::Named(probe::IMMEDIATE_NAME.into()), - ..Config::new() - }, - start: Start::Latest, - delay: Delay { - max: Some(std::time::Duration::ZERO), - ..Delay::default() - }, - }, - ) - .await - .unwrap(); - assert_eq!( - published.subscription().and_then(|sub| sub.delay.max), - Some(moq_mux::container::DELAY_MAX) - ); - } - - /// The track's first group decodes from the encoder's first picture, so it shows at - /// once. A skip past a stalled group warms up again: the pictures before its recovery - /// point plus the warmup decode but never show. - #[tokio::test] - async fn a_gap_warms_up_again_but_the_track_start_does_not() { - let (broadcast, mut track) = live_track(); - let mut consumer = warm(&broadcast, Start::Oldest).await; - - // Written as it is read: a zero budget would skip a backlog. - group(&mut track, 0, &[0, 1, 2, 3]).close().unwrap(); - let mut shown = Vec::new(); - for _ in 0..4 { - shown.push(read(&mut consumer).await); - } - let mut stalled = group(&mut track, 1, &[4]); - shown.push(read(&mut consumer).await); - let mut live = group(&mut track, 2, &[8, 9, 10, 11, 12]); - shown.push(read(&mut consumer).await); - - write(&mut stalled, &[5, 6, 7]); - stalled.close().unwrap(); - write(&mut live, &[13]); - shown.push(read(&mut consumer).await); - assert_eq!(shown, [0, 1, 2, 3, 4, 12, 13]); - } - /// The first request already states the min delay: a latest start waits for the live edge /// with its subscription open, before the container could update it. #[tokio::test] @@ -1514,7 +1316,7 @@ mod tests { max: Some(std::time::Duration::from_millis(500)), }; let consume = broadcast.consume(); - let catalog = warmup_catalog(None); + let catalog = catalog(); let mut open = Box::pin(Consumer::new( &consume, &catalog, @@ -1537,40 +1339,6 @@ mod tests { assert_eq!(first.map(|sub| sub.delay), Some(delay)); } - /// A rendition without `warmup` subscribes at the max delay alone and shows every picture - /// after a join. - #[tokio::test] - async fn no_warmup_changes_nothing() { - for warmup in [None, Some(std::time::Duration::ZERO)] { - let (broadcast, mut track) = live_track(); - let published = track.clone(); - group(&mut track, 4 * SECOND, &[4, 5]).close().unwrap(); - let _track = track.finish().unwrap(); - let max_delay = std::time::Duration::from_secs(10); - let mut consumer = Consumer::new( - &broadcast.consume(), - &warmup_catalog(warmup), - "video", - Options { - decoder: Config { - kind: Kind::Named(probe::IMMEDIATE_NAME.into()), - ..Config::new() - }, - delay: Delay { - max: Some(max_delay), - ..Delay::default() - }, - ..Options::new() - }, - ) - .await - .unwrap(); - assert_eq!(published.subscription().and_then(|sub| sub.delay.max), Some(max_delay)); - assert_eq!(read(&mut consumer).await, 4); - assert_eq!(read(&mut consumer).await, 5); - } - } - /// A device that cannot decode the stream refuses the rendition, so a caller /// can pick another one, the same as for a codec this build lacks. #[tokio::test] @@ -1578,7 +1346,7 @@ mod tests { let (broadcast, _track) = live_track(); let result = Consumer::new( &broadcast.consume(), - &warmup_catalog(None), + &catalog(), "video", Options { decoder: Config { @@ -1603,7 +1371,7 @@ mod tests { let (broadcast, track) = live_track(); let mut consumer = Consumer::new( &broadcast.consume(), - &warmup_catalog(None), + &catalog(), "video", Options { decoder: Config { @@ -1665,7 +1433,7 @@ mod tests { let (broadcast, track) = live_track(); let mut consumer = Consumer::new( &broadcast.consume(), - &warmup_catalog(None), + &catalog(), "video", Options { decoder: Config { From 6dccb76f6f728780e12b73deff614ee273eb7cc4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:28:17 -0800 Subject: [PATCH 3/3] fix(drafts): drop the stray >>> left where the video warmup section was cut Co-Authored-By: Claude Sonnet 5.5 --- drafts/draft-lcurley-moq-hang.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 3715b23541c..1de4b452fdd 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -806,8 +806,6 @@ For example: - A B-frame stream contributes its reorder delay, the presentation timestamp minus the decode timestamp. - A fragment or packet batch contributes the media span between its earliest and latest timestamps. ->>> - ### delay {#field-delay} The maximum amount, in milliseconds, by which a frame's presentation timestamp has trailed one that another rendition of the broadcast had already written. If absent, a consumer SHOULD assume the rendition does not trail the others.