From 5c0065ebeaba83d7a01dd3f8dc0f0f34161673ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 21:39:13 -0700 Subject: [PATCH 1/5] quest(m1): claim refresh/video Co-Authored-By: Claude Opus 5.5 From 216532a6bc6bcf8fe341e3c644cb066235805b65 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 22:32:27 -0700 Subject: [PATCH 2/5] feat(video): force a keyframe when a joiner wants a fresh group moq-video capture and the js/publish WebCodecs encoder advertise track::Info::refresh (ratio 2) and cut a keyframe on the next frame whenever Demand::wanted passes the largest group. The configured GOP stays the upper bound. Capture advertises only when every backend the encoder may open honors a cut, so V4L2 never advertises. The 500ms floor between forced keyframes is gone from Cuts and the WebCodecs encoder: the ratio already rate-limits joins by bytes. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/publish.md | 4 +- doc/lib/rs/moq-video.md | 5 +- js/publish/src/broadcast.ts | 14 +- js/publish/src/video/encoder.test.ts | 113 +++++++++----- js/publish/src/video/encoder.ts | 36 +++-- quest/m1/refresh/README.md | 1 - quest/m1/refresh/video.md | 44 ------ rs/moq-video/src/encode/backend/mod.rs | 51 +++++- rs/moq-video/src/encode/cuts.rs | 173 +++++++------------- rs/moq-video/src/encode/encoder.rs | 4 +- rs/moq-video/src/encode/producer.rs | 208 ++++++++++++++++++++----- 11 files changed, 395 insertions(+), 258 deletions(-) delete mode 100644 quest/m1/refresh/video.md diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 548844290e3..2c9c2c1b988 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -49,7 +49,9 @@ for a permission, device, or settings change instead of prompting again. The video encoder follows its share of the connection's send-rate estimate, so several publishers on one connection do not each target the whole uplink. Codec, resolution, framerate, and bitrate are tunable through -`el.video.config`, and `el.video.cut()` asks for an extra keyframe. Size, +`el.video.config`, and `el.video.cut()` asks for an extra keyframe. A viewer +who joins once a group's deltas outweigh twice its keyframe gets a fresh one, +so `keyframeInterval` is only the longest a group runs. Size, bitrate, and framerate change in place, while a codec or audio format change keeps the rendition's catalog ID under a fresh `track` name that viewers follow through the catalog. For diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 9dd00515ea9..027d2f1e5d4 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -27,7 +27,10 @@ hardware-only, AV1 decodes through NVDEC and MediaCodec, and VP8 and VP9 decode in software through the opt-in `vpx` feature. Capture is pull-driven: `encode::publish_capture` advertises the track up -front and opens the camera only while someone subscribes. Backends that can +front and opens the camera only while someone subscribes. A viewer who joins +once the current group's deltas outweigh twice its keyframe gets a fresh +keyframe, so the GOP is only the longest a group runs; V4L2 encoders keep to +their GOP, since a driver may refuse the request. Backends that can change bitrate live follow the connection's send estimate. Where the platform allows it, frames stay on the GPU between capture, decode, encode, and render. diff --git a/js/publish/src/broadcast.ts b/js/publish/src/broadcast.ts index 40c60d031f6..80a7d2c6368 100644 --- a/js/publish/src/broadcast.ts +++ b/js/publish/src/broadcast.ts @@ -10,6 +10,10 @@ import { errors } from "./internal"; import { Baseline } from "./jitter"; import { type Kind, Rendition } from "./rendition"; +// The refresh ratio video tracks advertise: a joiner forces a keyframe once a group's deltas +// outweigh twice its keyframe. Matches moq-video's capture `REFRESH`; see it for the measurements. +const VIDEO_REFRESH = 2; + // Signals the broadcast reads. Whoever owns the backing Signal (the element, or another component // whose output is wired in, e.g. a Video.Capture's `display`) does the writing. export type BroadcastInput = { @@ -133,7 +137,12 @@ export class Broadcast { this.#signals.run(this.#run.bind(this)); } - /** Register a video rendition under a full track name (e.g. `"video/hd"`). Throws if the name is taken. */ + /** + * Register a video rendition under a full track name (e.g. `"video/hd"`). Throws if the name is taken. + * + * Its track advertises a refresh ratio, so its producer opens a group with a keyframe whenever + * `track.demand().wanted` passes the largest group, as the video `Encoder` does. + */ video(name: string): Rendition { return this.#register(name, "video"); } @@ -413,9 +422,10 @@ export class Broadcast { continue; } if (!entry) { + const info = Container.trackInfo({ maxAge, priority: Catalog.PRIORITY[rendition.kind] }); const track = broadcast.createTrack( name, - Container.trackInfo({ maxAge, priority: Catalog.PRIORITY[rendition.kind] }), + rendition.kind === "video" ? { ...info, refresh: VIDEO_REFRESH } : info, ); entry = { track, owner: rendition }; tracks.set(rendition.name, entry); diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 4f9378dc929..b8ca9649e98 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -849,8 +849,30 @@ test("a re-probe swaps the catalog entry in one update", async () => { } }); -test("cut forces a keyframe, coalescing requests and spacing them at least 500ms apart", async () => { +// Drive a live encoder over a recording `VideoEncoder` that outputs every frame it is handed, so the +// track holds real groups: a keyframe of 100 bytes, each delta 60. `run` sends one frame per call +// at `millis` of media time and reads back which frames were forced to be keyframes. +async function recordKeyframes( + info: Partial, + run: (ctx: { + encoder: Encoder; + track: Moq.Track.Producer; + send: (millis: number) => Promise; + }) => Promise, +): Promise { const keys: number[] = []; + + class Frame { + readonly timestamp: number; + constructor(timestamp: number) { + this.timestamp = timestamp; + } + clone(): Frame { + return new Frame(this.timestamp); + } + close(): void {} + } + class RecordingVideoEncoder { state: CodecState = "unconfigured"; #output: VideoEncoderInit["output"]; @@ -869,9 +891,13 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms this.#codec = config.codec; } - // Records the test's frames only, not the probe's. + // Records the test's frames only, not the probe's, and outputs each a task later as WebCodecs does. encode(frame: VideoFrame, options?: VideoEncoderEncodeOptions): void { - if (options?.keyFrame && frame instanceof Frame) keys.push(frame.timestamp / 1000); + if (!(frame instanceof Frame)) return; + const key = !!options?.keyFrame; + if (key) keys.push(frame.timestamp / 1000); + const chunk = { type: key ? "key" : "delta", timestamp: frame.timestamp, byteLength: key ? 100 : 60 }; + queueMicrotask(() => this.#output({ ...chunk, copyTo: () => {} } as never, {} as never)); } // Only the probe flushes; report the configured codec, as Chrome does. @@ -894,17 +920,6 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms const videoFrame = Object.getOwnPropertyDescriptor(globalThis, "VideoFrame"); Object.defineProperty(globalThis, "VideoFrame", { configurable: true, value: FakeVideoFrame, writable: true }); - class Frame { - readonly timestamp: number; - constructor(timestamp: number) { - this.timestamp = timestamp; - } - clone(): Frame { - return new Frame(this.timestamp); - } - close(): void {} - } - const { Fanout } = await import("../fanout"); let controller!: ReadableStreamDefaultController; const fanout = new Fanout( @@ -916,7 +931,7 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms { clone: (frame) => frame.clone(), release: (frame) => frame.close() }, ); - const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MILLI }); + const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MICRO, ...info }); const rendition = { config: new Signal(undefined), track: new Signal(track), @@ -928,35 +943,19 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition } as never, + broadcast: { video: () => rendition, baseline: new Baseline() } as never, capture: capture as never, }); - // One frame per 100ms of media time, after calling cut() when `cut` says so. - const send = async (millis: number, cut = false) => { - if (cut) encoder.cut(); + const send = async (millis: number) => { controller.enqueue(new Frame(millis * 1000) as unknown as VideoFrame); await new Promise((resolve) => setTimeout(resolve, 5)); }; try { await settle(); - - // The opening keyframe serves a request made before it. - await send(0, true); - await send(100); - // Several requests before one frame produce one keyframe. - encoder.cut(); - encoder.cut(); - await send(600, true); - await send(700); - // Too soon after the last: deferred until 500ms have passed, not dropped. - await send(800, true); - await send(1000); - await send(1100); - await send(1200); - - expect(keys).toEqual([0, 600, 1100]); + await run({ encoder, track, send }); + return keys; } finally { encoder.close(); fanout.close(); @@ -966,6 +965,50 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms if (videoFrame) Object.defineProperty(globalThis, "VideoFrame", videoFrame); else Reflect.deleteProperty(globalThis, "VideoFrame"); } +} + +test("cut forces a keyframe on the next frame, coalescing requests without spacing them", async () => { + const keys = await recordKeyframes({}, async ({ encoder, send }) => { + // The opening keyframe serves a request made before it. + encoder.cut(); + await send(0); + await send(100); + // Several requests before one frame produce one keyframe. + encoder.cut(); + encoder.cut(); + await send(200); + // The very next frame can be cut again. + encoder.cut(); + await send(300); + await send(400); + }); + expect(keys).toEqual([0, 200, 300]); +}); + +test("a join over the refresh ratio forces a keyframe on the next frame", async () => { + const keys = await recordKeyframes({ refresh: 2 }, async ({ track, send }) => { + const watching = track.subscribe(); + const joined: Moq.Track.Subscriber[] = []; + try { + await send(0); + await send(50); + // One 60 byte delta is under twice the 100 byte keyframe. + joined.push(track.subscribe()); + await send(100); + await send(150); + await send(200); + // Four deltas are over it, 200ms after the keyframe. + joined.push(track.subscribe()); + await send(250); + // A join after the fresh keyframe reads its group. + joined.push(track.subscribe()); + await send(300); + } finally { + watching.close(); + for (const subscriber of joined) subscriber.close(); + } + }); + expect(keys).toEqual([0, 250]); }); test("a failed video encoder disables only its rendition and recovers on a fresh track", async () => { diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index fbb092e4733..64a3d26893b 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -48,7 +48,8 @@ export interface Config { // When combined with maxPixels, the smaller resulting cap wins. maxScale?: number; - // The interval at which to insert keyframes. (default: 2000 milliseconds) + // The longest interval between keyframes. A joining subscriber forces one sooner once the + // group's deltas outweigh the track's refresh ratio. (default: 2000 milliseconds) keyframeInterval?: Milli; // If not provided, the encoder will use the best bitrate for the given width, height, and framerate. @@ -368,9 +369,17 @@ export class Encoder { if (!capture) return; const producer = new Container.Legacy.Producer(track, new Container.Legacy.Format("video")); + const demand = track.demand(); let lastKeyframe: Micro | undefined; let lastEncoded: Micro | undefined; + // The largest group demand asked for that a keyframe already answers. A join over the refresh + // ratio asks for one past the largest, which the next frame opens. + let answered: number | undefined; + const answer = () => { + const wanted = demand.wanted.peek(); + if (wanted !== undefined) answered = Math.max(answered ?? 0, wanted); + }; let tearingDown = false; effect.spawn(async () => { @@ -388,6 +397,8 @@ export class Encoder { })); producer.encode(frame, frame.timestamp as Micro, key); + // Including a keyframe the encoder placed on its own, which opened a group too. + if (key) answer(); if (this.#estimator.flush(frame.timestamp, baseline)) { this.#estimate.set({ ...this.#estimator.estimate }); } @@ -458,19 +469,23 @@ export class Encoder { const interval = config?.keyframeInterval ?? Milli.fromSecond(2 as Second); - // Force a keyframe if this is the first frame (no group yet), the GOP elapsed, or - // the caller asked for one and the last is old enough. + // Force a keyframe if this is the first frame (no group yet), the GOP elapsed, the + // caller asked for one, or a join wants a group past the largest. No spacing: the + // refresh ratio already rate-limits joins by the bytes a group holds. const since = lastKeyframe === undefined ? undefined : frame.timestamp - lastKeyframe; + const wanted = demand.wanted.peek(); const keyFrame = resized || since === undefined || since >= Micro.fromMilli(interval) || - (this.#cut && since >= Micro.fromMilli(MIN_CUT_INTERVAL)); + this.#cut || + (wanted !== undefined && (answered === undefined || wanted > answered)); if (keyFrame) { lastKeyframe = frame.timestamp as Micro; resized = false; - // Any keyframe serves an outstanding request. + // Any keyframe serves an outstanding request and every join before it. this.#cut = false; + answer(); } encoder.encode(frame, { keyFrame }); @@ -804,9 +819,10 @@ export class Encoder { * Request a keyframe, opening a new group at a frame no earlier than this call. * * For a resume, a recording cut, or a known tune-in moment; {@link Config.keyframeInterval} is the - * cadence. Requests coalesce into the next keyframe, and forced keyframes land at least 500ms - * apart so a caller in a loop cannot pin the encoder at all-keyframe. A request while not encoding - * is served by the keyframe every encode starts with. + * longest a group runs, and a joining subscriber already opens one through the track's refresh + * ratio. Requests between two frames coalesce into one keyframe on the next, with no spacing: a + * caller in a loop pins the encoder at all-keyframe. A request while not encoding is served by the + * keyframe every encode starts with. */ cut(): void { this.#cut = true; @@ -817,10 +833,6 @@ export class Encoder { } } -// The closest two requested keyframes may land. A keyframe costs several times a predicted frame, -// and this stays well under the default two-second GOP so a request still beats the cadence. -const MIN_CUT_INTERVAL = 500 as Milli; - // The source's nominal frame rate: what the capture device settled on, or what a frame stream // declared. Undefined when nothing reports one. function sourceFrameRate(source: Source): number | undefined { diff --git a/quest/m1/refresh/README.md b/quest/m1/refresh/README.md index 708e994f19d..5d96bf43ca9 100644 --- a/quest/m1/refresh/README.md +++ b/quest/m1/refresh/README.md @@ -64,7 +64,6 @@ Work owned by this README, after the children land: - [Lite wire](/quest/m1/refresh/lite.md) - lite-07 TRACK_INFO advertises the ratio and SUBSCRIBE_UPDATE carries the wanted group, so a join at any relay reaches the origin - [moq-json](/quest/m1/refresh/json.md) - snapshot and window groups roll on a qualifying join, reading the ratio from the track; `delta_ratio` and `op_ratio` become `deltas` and `ops` switches in every binding -- [Video](/quest/m1/refresh/video.md) - video publishers advertise refresh and force a keyframe on demand, keeping the encoder's max GOP ## Related diff --git a/quest/m1/refresh/video.md b/quest/m1/refresh/video.md deleted file mode 100644 index 1ad728911c4..00000000000 --- a/quest/m1/refresh/video.md +++ /dev/null @@ -1,44 +0,0 @@ -# [M] Video keyframes on demand - -## Goal - -Rust and browser video publishers advertise `track::Info::refresh` and -force a keyframe in a new group when `Demand::wanted` passes the largest -group. The encoder's configured max GOP stays as the upper bound. With no -joiners, GOPs run to the max; a joiner gets a fresh keyframe once the GOP's -bytes after its keyframe exceed the ratio. - -## Plan - -Decided 2026-10-16 (see the [epic](/quest/m1/refresh/README.md)). - -- Open-ended GOPs: a keyframe comes on a refresh or at the max GOP. The cap - stays as the bound behind hops that cannot forward requests, and against - decoder drift. -- Scope: advertise refresh only on backends whose keyframe forcing is known - before the track is created, because `track::Info` is immutable. That - covers `js/publish` through WebCodecs (`keyFrame: true`) and the - moq-video encoders that support it. V4L2 does not advertise, because some - drivers reveal that they refuse keyframe requests only after - `Control::new` has created the track. Passthrough importers (CMAF, TS, - RTMP, SRT) cannot force one either. Decided in the #5734 review: never - advertise a request the backend might refuse. -- Remove the 500 ms floor between keyframes from moq-video's `Cuts` and the - WebCodecs publisher, and let refresh-driven keyframes take over. The byte - ratio already bounds the rate, and the floor would let a high-bitrate - joiner download well past the ratio (maintainer, #5734 review). -- Open, settled with the maintainer before any code: where the ratio is - configured (an encoder option, a rendition option, or a fixed default - with no public field), its default, and whether it varies per rendition. - The publishers create their tracks internally, so this decides how - `track::Info::refresh` gets populated. Measure the bytes a joiner - downloads against the extra keyframes sent on a representative stream, - then decide. -- Docs: the encoder options in `doc/` that describe GOP length. - -Tests: a forced-keyframe test per backend that supports it, using simulated -time where the encoder allows, and a browser test for `js/publish`. Each -covers a ratio crossing within 500 ms of the previous keyframe. - -Public API: set by the open decision above, either an encoder or rendition -config field for the ratio, or none for a fixed default. Wire: none. diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index ca0df34862b..2753d4d5a4e 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -167,11 +167,14 @@ pub const NAMES: &[&str] = &[ "openh264", ]; -/// A backend constructor: name, the codecs it can emit, and an opener. +/// A backend constructor: name, the codecs it can emit, an opener, and whether it can always cut. struct Candidate { name: &'static str, codecs: &'static [Codec], open: fn(&Config) -> Result, Error>, + /// Whether every encoder this opens honors a cut ([`Backend::can_cut`]), known before + /// opening one. V4L2 learns it from the driver at open, so it cannot promise. + cuts: bool, #[cfg(target_os = "linux")] vulkan: bool, } @@ -184,6 +187,7 @@ const HARDWARE: &[Candidate] = &[ name: videotoolbox::NAME, codecs: &[Codec::H264, Codec::H265], open: videotoolbox::VideoToolbox::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -192,6 +196,7 @@ const HARDWARE: &[Candidate] = &[ name: mediafoundation::NAME, codecs: &[Codec::H264, Codec::H265], open: mediafoundation::MediaFoundation::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -200,6 +205,7 @@ const HARDWARE: &[Candidate] = &[ name: mediacodec::NAME, codecs: &[Codec::H264, Codec::H265], open: mediacodec::MediaCodec::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -208,6 +214,7 @@ const HARDWARE: &[Candidate] = &[ name: nvenc::NAME, codecs: &[Codec::H264, Codec::H265], open: nvenc::Nvenc::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: true, }, @@ -216,6 +223,7 @@ const HARDWARE: &[Candidate] = &[ name: vaapi::NAME, codecs: &[Codec::H264], open: vaapi::Vaapi::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -227,6 +235,7 @@ const HARDWARE: &[Candidate] = &[ name: v4l2::NAME, codecs: &[Codec::H264], open: v4l2::V4l2::open, + cuts: false, #[cfg(target_os = "linux")] vulkan: false, }, @@ -241,6 +250,7 @@ const SOFTWARE: &[Candidate] = &[ name: openh264::NAME, codecs: &[Codec::H264], open: openh264::Openh264::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -255,6 +265,7 @@ const NAMED_ONLY: &[Candidate] = &[ name: probe::NAME, codecs: &[Codec::H264], open: probe::Probe::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -262,6 +273,7 @@ const NAMED_ONLY: &[Candidate] = &[ name: probe::NO_CUT, codecs: &[Codec::H264], open: probe::Probe::open_no_cut, + cuts: false, #[cfg(target_os = "linux")] vulkan: false, }, @@ -297,12 +309,23 @@ impl<'a> Attempt<'a> { /// Open the best encoder for `config.codec` + `config.kind`, trying candidates /// in priority order and falling back until one succeeds. pub(crate) fn open(config: &Config) -> Result, Error> { - let codec = config.codec; + select(attempts(config.codec, &config.kind), config) +} + +/// Whether whichever backend [`open`] picks for `codec` + `kind` honors a cut, answered +/// before opening one: every candidate it may fall to must promise it. +pub(crate) fn always_cuts(codec: Codec, kind: &Kind) -> bool { + let attempts = attempts(codec, kind); + !attempts.is_empty() && attempts.iter().all(|attempt| attempt.candidate.cuts) +} + +/// The candidates [`open`] tries for `codec` + `kind`, in order. +fn attempts(codec: Codec, kind: &Kind) -> Vec> { let supports = move |c: &Candidate| c.codecs.contains(&codec); let hardware = HARDWARE.iter().filter(|c| supports(c)).map(Attempt::hardware); let software = SOFTWARE.iter().filter(|c| supports(c)).map(Attempt::software); - let attempts: Vec = match &config.kind { + match kind { Kind::Auto => hardware.chain(software).collect(), Kind::Hardware => hardware.collect(), Kind::Software => software.collect(), @@ -312,9 +335,7 @@ pub(crate) fn open(config: &Config) -> Result, Error> { .chain(SOFTWARE.iter().chain(NAMED_ONLY.iter()).map(Attempt::software)) .filter(|a| supports(a.candidate) && a.candidate.name == name) .collect(), - }; - - select(attempts, config) + } } /// Try `attempts` in order and return the first encoder that opens, warning when @@ -540,6 +561,7 @@ mod tests { name: "stub", codecs: &[Codec::H264], open: Stub::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }; @@ -550,6 +572,7 @@ mod tests { name: "driverless", codecs: &[Codec::H264], open: |_| Err(Error::Codec(anyhow::anyhow!("driver libraries not found"))), + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }; @@ -558,6 +581,22 @@ mod tests { Config::new(320, 240, crate::Rate::new(30, 1).unwrap()) } + /// Only a selection whose every candidate promises cuts may advertise them, and a + /// candidate's promise holds for the encoder it opens. + #[test] + fn always_cuts_needs_every_candidate_to_promise() { + let named = |name: &str| Kind::Named(name.to_owned()); + assert!(always_cuts(Codec::H264, &named(probe::NAME))); + assert!(!always_cuts(Codec::H264, &named(probe::NO_CUT))); + assert!(!always_cuts(Codec::H264, &named("v4l2")), "V4L2 learns it at open"); + assert!(!always_cuts(Codec::H264, &named("nonexistent")), "nothing to open"); + + for candidate in NAMED_ONLY { + let backend = (candidate.open)(&config()).unwrap(); + assert_eq!(backend.can_cut(), candidate.cuts, "{}", candidate.name); + } + } + #[tracing_test::traced_test] #[test] fn falling_past_hardware_warns() { diff --git a/rs/moq-video/src/encode/cuts.rs b/rs/moq-video/src/encode/cuts.rs index 49f823c7f6c..65748f2f7e3 100644 --- a/rs/moq-video/src/encode/cuts.rs +++ b/rs/moq-video/src/encode/cuts.rs @@ -1,76 +1,50 @@ //! [`Cuts`]: which captured frames to force a keyframe on. -use std::time::Duration; - -use moq_net::Timestamp; - -/// The closest a forced keyframe may land after any other keyframe, in media time. +/// One encoder's view of the keyframe asks: decides which frames to cut, coalescing. +/// Built fresh for every encoder the capture opens. /// -/// A keyframe costs several times a predicted frame, so a caller asking in a loop -/// would otherwise pin the encoder at all-IDR and starve the rest of the uplink. -/// Well under the default two-second GOP, so a request still beats the cadence. -const MIN_INTERVAL: Duration = Duration::from_millis(500); - -/// One encoder's view of the keyframe requests: decides which frames to cut, -/// coalescing and rate limiting. Built fresh for every encoder the capture opens. +/// Two asks open a group. `Control::cut` requests arrive as a running +/// count, so any number between two frames read as one. A subscriber joining over the +/// track's refresh ratio arrives as the group +/// [`Demand::wanted`](moq_net::track::Demand::wanted) asks for once it passes the largest +/// written, which only grows, so joins while the keyframe is still in the encoder ask for the +/// same group and coalesce into it. /// -/// Requests arrive as a running count, so any number between two frames read as one. -/// Every keyframe the encoder produces counts, including its own GOP cadence: it -/// serves the requests made before its frame and restarts the spacing window. +/// There is no spacing between cuts: the refresh ratio already rate-limits joins by the bytes +/// a group holds, and a floor would let a joiner download well past that ratio. pub(super) struct Cuts { /// The request count already accounted for. - seen: u64, - /// The frame the latest unserved request was noticed at, or `None` when none waits. - /// A keyframe at or after it serves every request so far. - pending: Option, - /// The latest keyframe, forced or not, or `None` before this encoder's first frame. - last: Option, + requests: u64, + /// The largest wanted group already cut for, or `None` before any. + wanted: Option, + /// Whether this encoder produced its first frame. + opened: bool, } impl Cuts { /// Start from `requests`, the count already served by earlier encoders. pub fn new(requests: u64) -> Self { Self { - seen: requests, - pending: None, - last: None, + requests, + wanted: None, + opened: false, } } - /// Whether the frame at `timestamp` should be cut, given the running request - /// count, recording it if so. - pub fn due(&mut self, requests: u64, timestamp: Timestamp) -> bool { - if requests != self.seen { - self.seen = requests; - self.pending = Some(timestamp); - } + /// Whether the next frame should be cut, given the running request count and `wanted`, + /// the group demand asks for when it is past the largest written. + pub fn due(&mut self, requests: u64, wanted: Option) -> bool { + let asked = requests != self.requests || wanted > self.wanted; + self.requests = requests; + self.wanted = self.wanted.max(wanted); // A fresh encoder opens with a keyframe on every backend, which serves anything - // requested before it. - let Some(last) = self.last else { - self.last = Some(timestamp); - self.pending = None; - return false; - }; - - if self.pending.is_none() || Duration::from(timestamp).saturating_sub(Duration::from(last)) < MIN_INTERVAL { + // asked for before it. + if !self.opened { + self.opened = true; return false; } - - self.pending = None; - self.last = Some(timestamp); - true - } - - /// Record a keyframe the encoder produced for the frame at `timestamp`, whether - /// forced or on its own cadence. - pub fn keyframe(&mut self, timestamp: Timestamp) { - // A buffering encoder can hand back a keyframe older than the latest request, - // which is not a frame that request asked for. - if self.pending.is_some_and(|pending| timestamp >= pending) { - self.pending = None; - } - self.last = Some(self.last.map_or(timestamp, |last| last.max(timestamp))); + asked } } @@ -78,94 +52,55 @@ impl Cuts { mod tests { use super::*; - fn at(millis: u64) -> Timestamp { - Timestamp::from_millis(millis).unwrap() - } - #[test] - fn the_opening_keyframe_serves_earlier_requests() { + fn the_opening_keyframe_serves_earlier_asks() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(1, at(0))); - assert!(!cuts.due(1, at(1_000)), "the request was already served"); + assert!(!cuts.due(1, Some(4))); + assert!(!cuts.due(1, Some(4)), "the asks were already served"); } #[test] fn requests_before_a_frame_coalesce() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(cuts.due(10, at(1_000))); - assert!(!cuts.due(10, at(2_000)), "ten requests before one frame cut once"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(10, None)); + assert!(!cuts.due(10, None), "ten requests before one frame cut once"); } + /// Back-to-back requests each cut: the caller decides how often to ask. #[test] - fn a_request_too_soon_is_deferred_not_dropped() { + fn requests_are_not_spaced() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(!cuts.due(1, at(100))); - assert!(!cuts.due(1, at(499))); - assert!(cuts.due(1, at(500)), "held until the interval elapsed"); - } - - #[test] - fn a_caller_in_a_loop_cannot_force_all_idr() { - let mut cuts = Cuts::new(0); - - // Ten seconds at 30 fps with a request before every frame. - let cut = (0..300u64) - .filter(|frame| cuts.due(frame + 1, at(frame * 1000 / 30))) - .count(); - assert_eq!(cut, 19, "one forced keyframe per half second after the opening one"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(1, None)); + assert!(cuts.due(2, None), "a request one frame after a cut still cuts"); } #[test] fn a_reopened_encoder_ignores_requests_already_counted() { let mut cuts = Cuts::new(5); - assert!(!cuts.due(5, at(0))); - assert!(!cuts.due(5, at(1_000))); - } - - /// The encoder's own GOP keyframe lands while a request waits out the spacing - /// window, so the request is served and no extra keyframe is forced. - #[test] - fn a_cadence_keyframe_serves_a_waiting_request() { - let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(cuts.due(1, at(1_000)), "forced"); - cuts.keyframe(at(1_000)); - - assert!(!cuts.due(2, at(1_100)), "too soon after the forced keyframe"); - assert!(!cuts.due(2, at(1_200))); - cuts.keyframe(at(1_200)); - - // Well past the window, and nothing left to force. - assert!(!cuts.due(2, at(1_600))); - assert!(!cuts.due(2, at(3_000))); + assert!(!cuts.due(5, None)); + assert!(!cuts.due(5, None)); } - /// A cadence keyframe restarts the spacing window, so a request right after it - /// cannot put two keyframes a frame apart. + /// A join cuts once per wanted group: the frames still in the encoder ask for the same + /// group, and a group that lands drops the ask until another join raises it. #[test] - fn a_request_right_after_a_cadence_keyframe_waits_the_interval() { + fn a_wanted_group_cuts_once() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(!cuts.due(0, at(2_000))); - cuts.keyframe(at(2_000)); - - assert!(!cuts.due(1, at(2_033))); - assert!(!cuts.due(1, at(2_499))); - assert!(cuts.due(1, at(2_500)), "held until the interval elapsed"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(0, Some(1)), "a join past the largest group"); + assert!(!cuts.due(0, Some(1)), "the keyframe is still in the encoder"); + assert!(!cuts.due(0, None), "group 1 landed"); + assert!(cuts.due(0, Some(2)), "the next join"); } - /// A buffering encoder can return a keyframe for a frame read before the - /// request, which does not serve it. + /// A request and a join before the same frame cut it once. #[test] - fn a_keyframe_older_than_the_request_does_not_serve_it() { + fn a_request_and_a_join_coalesce() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(!cuts.due(1, at(100)), "too soon"); - cuts.keyframe(at(50)); - - assert!(!cuts.due(1, at(549)), "the window restarts at the latest keyframe"); - assert!(cuts.due(1, at(550)), "the request is still owed a keyframe"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(1, Some(1))); + assert!(!cuts.due(1, Some(1))); } } diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index c9bd11e4d03..3ea92a8eee9 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -437,7 +437,9 @@ impl Encoder { /// Rarely needed: the encoder opens groups automatically, so reach for this /// only when something outside the encoder needs a boundary at a specific /// frame. A source group boundary (transcode mirrors the source's groups) is - /// the usual reason; a scene change or a source switch is another. + /// the usual reason; a scene change, a source switch, or a subscriber joining + /// over [`track::Info::refresh`](moq_net::track::Info::refresh) are others. + /// Capture cuts for joiners itself. /// /// The request waits for the next [`encode`](Self::encode) rather than applying /// at once, so it is safe to call before the frame exists. Calling it repeatedly diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 274c1ed42f2..2d8ea2eb45f 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -5,8 +5,12 @@ //! nothing has been encoded yet), and the encoder itself only runs while a //! subscriber is watching. Capture opens its camera once at startup to learn //! the mode it negotiates, then keeps it closed between viewers. This mirrors -//! `moq-boy`, which pauses its emulator on `track::Producer::used()` / -//! `unused()`. +//! `moq-boy`, which pauses its emulator while +//! [`Demand::wanted`](moq_net::track::Demand::wanted) is `None`. +//! +//! Capture advertises [`track::Info::refresh`](moq_net::track::Info::refresh) when its encoder +//! is sure to honor a cut, and forces a keyframe whenever a joining subscriber asks for a group +//! past the largest. The encoder's GOP stays the upper bound between keyframes. use moq_mux::catalog::hang::CatalogExt; #[cfg(feature = "capture")] @@ -34,6 +38,17 @@ use super::encoder::Codec; #[cfg(feature = "capture")] const DEFAULT_FRAMERATE: Rate = Rate::integer(30); +/// The [`track::Info::refresh`](moq_net::track::Info::refresh) ratio a capture advertises: a +/// joiner forces a keyframe once a group's deltas outweigh twice its keyframe, so it downloads +/// at most about three keyframes' worth before it can decode. +/// +/// Measured on realtime CBR encodes (two-second GOPs at 2 to 4 Mbps), where a GOP's deltas run +/// three to five keyframes: one join a second halves the mean joiner download for 10 to 13% +/// more bitrate, and one every ten seconds costs about 1%. A ratio scales with the keyframe, +/// so it holds for every rendition. +#[cfg(feature = "capture")] +const REFRESH: u32 = 2; + /// Convert the probed rendition into the importer hint published before the first frame. fn rendition_hint(rendition: hang::catalog::VideoConfig) -> moq_mux::catalog::VideoHint { let mut hint = moq_mux::catalog::VideoHint::default(); @@ -78,6 +93,12 @@ enum Codecs { /// publishes into a catalog of the same shape. pub struct Producer { codecs: Codecs, + /// The published track, read for its largest group. + #[cfg_attr(not(feature = "capture"), allow(dead_code))] + track: moq_net::track::Producer, + /// The largest group published, as last read from `track`. + #[cfg_attr(not(feature = "capture"), allow(dead_code))] + largest: Option, _ext: std::marker::PhantomData E>, } @@ -123,6 +144,7 @@ impl Producer { reserved: moq_mux::catalog::Reserved, rendition: hang::catalog::VideoConfig, ) -> Result { + let published = track.clone(); let codecs = match &rendition.codec { hang::catalog::VideoCodec::H264(_) => Codecs::H264 { split: moq_mux::codec::h264::Split::new(), @@ -141,6 +163,8 @@ impl Producer { }; Ok(Self { codecs, + track: published, + largest: None, _ext: std::marker::PhantomData, }) } @@ -155,6 +179,18 @@ impl Producer { } } + /// The group a subscriber wants past the largest published, which the next frame should + /// open with a keyframe, or `None` while the largest serves. + #[cfg(feature = "capture")] + fn wanted(&mut self) -> Option { + let wanted = self.track.demand().wanted()?; + // Ready only when the largest moved since the last read. + if let std::task::Poll::Ready(Some(largest)) = self.track.poll_largest(&kio::Waiter::noop()) { + self.largest = Some(largest.group); + } + self.largest.is_none_or(|largest| wanted > largest).then_some(wanted) + } + /// Publish already-encoded frames, each at its own timestamp. Each frame is one /// whole access unit in the producer's codec framing. pub fn publish(&mut self, encoded: &[Encoded]) -> Result<(), Error> { @@ -316,7 +352,11 @@ impl Control { Codec::H264 => ".avc3", Codec::H265 => ".hev1", }; - let track = broadcast.unique_track(suffix, catalog.track_info(hang::catalog::PRIORITY.video))?; + // Advertised only when every backend the encoder may open honors a cut, since the info + // is fixed before the driver learns which one it got. + let refresh = super::backend::always_cuts(options.encode.codec, &options.encode.kind).then_some(REFRESH); + let info = catalog.track_info(hang::catalog::PRIORITY.video).with_refresh(refresh); + let track = broadcast.unique_track(suffix, info)?; let requests = kio::Producer::new(Requests::default()); let driver = Driver { track, @@ -330,10 +370,11 @@ impl Control { /// Request a keyframe, opening a new group at a frame no earlier than this call. /// /// For a resume, a recording cut, or a known tune-in moment; [`Config::gop`](super::Config::gop) - /// is the cadence. Requests coalesce into the next keyframe, whether forced or on that cadence, - /// and a forced one lands at least 500ms after any other, so a caller in a loop cannot pin the - /// encoder at all-IDR; a request that arrives too soon waits rather than being dropped. A - /// request while nothing is watching is served by the keyframe every fresh encoder opens with. + /// is the longest a group runs, and a joining subscriber already opens one through + /// [`track::Info::refresh`](moq_net::track::Info::refresh). Requests between two frames + /// coalesce into one keyframe on the next, with no spacing: a caller in a loop pins the + /// encoder at all-IDR. A request while nothing is watching is served by the keyframe every + /// fresh encoder opens with. /// /// # Errors /// @@ -714,7 +755,8 @@ async fn capture_loop( }; // A reopen can land on another backend than the probe did, so the controls // learn this one's answer before any request is counted against it. - let served = learn(requests, refusal(&mut encoder).await?); + let refused = refusal(&mut encoder).await?; + let served = learn(requests, refused); tracing::info!(encoder = encoder.name(), device = camera.label(), "capturing"); // A reopen can negotiate a different mode (a display resized while nothing was @@ -730,7 +772,7 @@ async fn capture_loop( // reopened camera starts optimistic again rather than inheriting the // backed-off rate from whatever the link was doing last time. let mut rate = Some((reservation.consumer(), rate::Control::new(rate::Policy::new(ceiling)))); - // Per encoder, so a reopen forgets the old encoder's last keyframe along with it. + // Per encoder, since a fresh one opens with a keyframe that serves every earlier ask. let mut forced = Cuts::new(served); loop { @@ -760,16 +802,15 @@ async fn capture_loop( let Some(mut frame) = frame else { break }; frame.timestamp = map_capture_timestamp(capture_epoch, frame.timestamp)?; let count = counted.read().count; - if forced.due(count, frame.timestamp) { + // A backend that cannot cut leaves every group to its cadence. + let wanted = refused.is_none().then(|| producer.wanted()).flatten(); + if forced.due(count, wanted) { // Never refused: `Control::cut` stops counting once this encoder is known to. encoder.cut().await?; } let Some(encoded) = wait_capture(demand, encoder.encode(frame)).await? else { break; }; - for unit in encoded.iter().filter(|unit| unit.keyframe) { - forced.keyframe(unit.timestamp); - } producer.publish(&encoded)?; } @@ -1523,8 +1564,38 @@ mod tests { } } - /// Drive cuts through the public control and read their group boundaries from the track. - async fn capture_cuts(cut_after_cadence: bool) -> Vec { + /// What the driver is fed next, one at a time. + enum Step { + /// A frame 10ms after the previous one, waited on until the first subscriber has it. + Frame(crate::frame::Surface), + /// A keyframe request through the public control. + Cut, + /// Another subscriber joining at the live edge, held until the end. + Join, + } + + /// A frame of seeded noise, which no encoder can predict from the one before it. + fn noise(seed: u32) -> crate::frame::Surface { + let mut state = seed.wrapping_mul(2_654_435_761) | 1; + let data = (0..320 * 240 * 3 / 2) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + 120 + (state as u8 >> 4) + }) + .collect(); + crate::frame::Surface::I420(crate::frame::I420 { + width: 320, + height: 240, + data, + color: None, + }) + } + + /// Drive the real driver through `steps`, returning the info the track advertised and the + /// payload size of every frame the first subscriber read, grouped as it read them. + async fn capture_groups(steps: Vec) -> (moq_net::track::Info, Vec>) { use moq_mux::container::Container as _; // Off Apple, encoding waits on an OS thread. Keep Tokio from auto-advancing @@ -1535,7 +1606,7 @@ mod tests { }); let started = tokio::time::Instant::now(); // Capture streams are !Send on macOS. - let groups = tokio::task::LocalSet::new() + let result = tokio::task::LocalSet::new() .run_until(async { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); @@ -1543,10 +1614,12 @@ mod tests { moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); let mut options = Capture::default(); options.encode.kind = encoder::Kind::Software; + // Room for noise, which rate control would otherwise skip frames of. + options.encode.bitrate = Some(moq_net::bandwidth::Rate::from_mbps(4)); let (control, driver) = Control::new(broadcast.clone(), catalog, options).unwrap(); - // Ten nominal fps makes the default two-second GOP twenty frames. Feed them - // 10ms apart in media time so cadence falls inside the 500ms cut spacing window. + // Ten nominal fps makes the default two-second GOP twenty frames, which every + // script below stays inside. let rate = crate::Rate::new(10, 1).unwrap(); let (_probe, probe) = Synthetic::open(crate::Size::new(320, 240), rate); let (camera, stream) = Synthetic::open(crate::Size::new(320, 240), rate); @@ -1555,28 +1628,30 @@ mod tests { moq_mux::catalog::Consumer::<()>::new(&consumer, moq_mux::catalog::CatalogFormat::Hang) .await .unwrap(); - let (mut subscriber, format) = loop { + let (name, mut subscriber, format) = loop { let snapshot = catalog.next().await.unwrap().expect("a catalog"); if let Some((name, rendition)) = snapshot.video.renditions.iter().next() { break ( + name.clone(), consumer.track(name).unwrap().subscribe(None).await.unwrap(), moq_mux::catalog::hang::Container::try_from(rendition).unwrap(), ); } }; + let info = subscriber.info().clone(); let (published, mut received) = tokio::sync::mpsc::unbounded_channel(); let collect = tokio::task::spawn_local(async move { let mut groups = Vec::new(); while let Some(mut group) = subscriber.recv_group().await.unwrap() { - let mut frames = 0; + let mut frames = Vec::new(); while let Some(batch) = kio::wait(|waiter| format.poll_read(&mut group, waiter)).await.unwrap() { for frame in batch { // A duration marker closes the group but is not an encoded picture. if format.end(&frame).is_none() { - frames += 1; + frames.push(frame.payload.len()); published.send(()).unwrap(); } } @@ -1586,36 +1661,97 @@ mod tests { groups }); - let timestamps = (0..=20).map(|frame| frame * 10).chain([201, 699, 700, 1200]); - for (frame, millis) in timestamps.enumerate() { - camera.push_native(surface(), us(millis * 1000)); - // Wait for delivery before requesting a cut or replacing the capture's latest slot. - received.recv().await.expect("the frame reached the subscriber"); - if frame == 0 || (frame == 20 && cut_after_cadence) { - control.cut().unwrap(); + let mut joined = Vec::new(); + let mut millis = 0; + for step in steps { + match step { + Step::Frame(surface) => { + camera.push_native(surface, us(millis * 1000)); + millis += 10; + // Wait for delivery before the next step, which also keeps the + // capture's latest slot from being replaced. + received.recv().await.expect("the frame reached the subscriber"); + } + Step::Cut => control.cut().unwrap(), + Step::Join => joined.push(consumer.track(&name).unwrap().subscribe(None).await.unwrap()), } } drop(control); run.await.unwrap().unwrap(); - collect.await.unwrap() + (info, collect.await.unwrap()) }) .await; assert_eq!(started.elapsed(), Duration::ZERO); drop(hold); clock.await.unwrap(); - groups + result } + /// Each request opens a group at the next frame, with no spacing between them. #[tokio::test(start_paused = true)] - async fn cadence_keyframe_serves_a_pending_capture_cut() { - // The keyframe at 200ms serves the pending cut. Passing 700ms must not force another. - assert_eq!(capture_cuts(false).await, [20, 5]); + async fn capture_cuts_land_on_the_next_frame() { + let steps = vec![ + Step::Frame(surface()), + Step::Cut, + Step::Frame(surface()), + Step::Frame(surface()), + Step::Cut, + Step::Frame(surface()), + Step::Frame(surface()), + ]; + let (_, groups) = capture_groups(steps).await; + let lengths: Vec<_> = groups.iter().map(Vec::len).collect(); + assert_eq!(lengths, [1, 2, 2]); } + /// A capture whose encoder always cuts advertises the refresh ratio, and a join opens a + /// group at the next frame only once the deltas outweigh it, well within 500ms of the + /// keyframe before. #[tokio::test(start_paused = true)] - async fn capture_cut_after_cadence_waits_for_spacing() { - // A new cut after 200ms waits through 699ms, opens at 700ms, and fires only once. - assert_eq!(capture_cuts(true).await, [20, 3, 2]); + async fn a_join_over_the_refresh_ratio_forces_a_keyframe() { + let mut steps = vec![ + Step::Frame(noise(0)), + Step::Frame(surface()), + Step::Frame(surface()), + // Two flat deltas cost next to nothing against a noise keyframe. + Step::Join, + Step::Frame(surface()), + ]; + steps.extend((1..=10).map(|seed| Step::Frame(noise(seed)))); + steps.extend([Step::Join, Step::Frame(surface()), Step::Frame(surface())]); + let (info, groups) = capture_groups(steps).await; + assert_eq!(info.refresh, Some(REFRESH)); + + let lengths: Vec<_> = groups.iter().map(Vec::len).collect(); + assert_eq!(lengths, [14, 2], "only the second join cut, at the frame after it"); + let ratio = usize::try_from(REFRESH).unwrap(); + let first = &groups[0]; + assert!( + first[1..3].iter().sum::() <= ratio * first[0], + "the first join was under the ratio: {first:?}" + ); + assert!( + first[1..].iter().sum::() > ratio * first[0], + "the second join was over the ratio: {first:?}" + ); + } + + /// Refresh is advertised only when every backend the encoder may open honors a cut, since + /// the track info is fixed before the driver learns which one it got. + #[test] + fn a_backend_that_may_refuse_cuts_advertises_no_refresh() { + for (kind, refresh) in [ + (encoder::Kind::Software, Some(REFRESH)), + (encoder::Kind::Named(probe::NO_CUT.into()), None), + ] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut options = Capture::default(); + options.encode.kind = kind; + let (_control, driver) = Control::new(broadcast.clone(), catalog, options).unwrap(); + assert_eq!(driver.track.info().refresh, refresh); + } } /// Drop the controls while the driver sits in its `opens`th open, which never finishes. From c3fb25c266eb98fff3a4f6e771bed16a0954a995 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 22:55:26 -0700 Subject: [PATCH 3/5] fix(video): allow the cut promise to sit unread without capture Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/encode/backend/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index 2753d4d5a4e..937eaf3e416 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -174,6 +174,7 @@ struct Candidate { open: fn(&Config) -> Result, Error>, /// Whether every encoder this opens honors a cut ([`Backend::can_cut`]), known before /// opening one. V4L2 learns it from the driver at open, so it cannot promise. + #[cfg_attr(not(feature = "capture"), allow(dead_code))] cuts: bool, #[cfg(target_os = "linux")] vulkan: bool, @@ -314,6 +315,7 @@ pub(crate) fn open(config: &Config) -> Result, Error> { /// Whether whichever backend [`open`] picks for `codec` + `kind` honors a cut, answered /// before opening one: every candidate it may fall to must promise it. +#[cfg_attr(not(feature = "capture"), allow(dead_code))] pub(crate) fn always_cuts(codec: Codec, kind: &Kind) -> bool { let attempts = attempts(codec, kind); !attempts.is_empty() && attempts.iter().all(|attempt| attempt.candidate.cuts) From 630ea7ef95414b327b2b47110b3f66b8d0c94c4a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 23:04:12 -0700 Subject: [PATCH 4/5] fix(video): qualify the refresh claim and lock the probe in its test The docs now state that through a moq-lite relay only the join that opens the upstream subscription counts, and that a v4l2 build keeps every Auto or Hardware selection to its GOP. The candidate-promise test holds the probe lock while it opens probe backends, so it cannot pollute another test's log under cargo test. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/publish.md | 3 ++- doc/lib/rs/moq-video.md | 7 +++++-- rs/moq-video/src/encode/backend/mod.rs | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 2c9c2c1b988..a3519673654 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -51,7 +51,8 @@ several publishers on one connection do not each target the whole uplink. Codec, resolution, framerate, and bitrate are tunable through `el.video.config`, and `el.video.cut()` asks for an extra keyframe. A viewer who joins once a group's deltas outweigh twice its keyframe gets a fresh one, -so `keyframeInterval` is only the longest a group runs. Size, +so `keyframeInterval` is only the longest a group runs. Through a moq-lite +relay, only the join that opens the relay's upstream subscription counts. Size, bitrate, and framerate change in place, while a codec or audio format change keeps the rendition's catalog ID under a fresh `track` name that viewers follow through the catalog. For diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 027d2f1e5d4..3259aa4080b 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -29,8 +29,11 @@ decode in software through the opt-in `vpx` feature. Capture is pull-driven: `encode::publish_capture` advertises the track up front and opens the camera only while someone subscribes. A viewer who joins once the current group's deltas outweigh twice its keyframe gets a fresh -keyframe, so the GOP is only the longest a group runs; V4L2 encoders keep to -their GOP, since a driver may refuse the request. Backends that can +keyframe, so the GOP is only the longest a group runs. Through a moq-lite +relay, only the join that opens the relay's upstream subscription counts. A +build with the `v4l2` feature keeps every `Auto` or `Hardware` selection to its +GOP, whichever encoder it lands on, since a V4L2 driver may refuse the request. +Backends that can change bitrate live follow the connection's send estimate. Where the platform allows it, frames stay on the GPU between capture, decode, encode, and render. diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index 937eaf3e416..ea9b000437e 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -593,6 +593,7 @@ mod tests { assert!(!always_cuts(Codec::H264, &named("v4l2")), "V4L2 learns it at open"); assert!(!always_cuts(Codec::H264, &named("nonexistent")), "nothing to open"); + let _probe = probe::exclusive(); for candidate in NAMED_ONLY { let backend = (candidate.open)(&config()).unwrap(); assert_eq!(backend.can_cut(), candidate.cuts, "{}", candidate.name); From f92a7c7859efe3edb28a69af3451227da14d5b3a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 23:17:11 -0700 Subject: [PATCH 5/5] docs(video): migrate guides for the unlimited cut and refresh contract Co-Authored-By: Claude Opus 5.5 --- js/publish/MIGRATE.md | 7 +++++++ rs/moq-video/MIGRATE.md | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/js/publish/MIGRATE.md b/js/publish/MIGRATE.md index 0e66059ea83..f32a6a2a47f 100644 --- a/js/publish/MIGRATE.md +++ b/js/publish/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Encoder.cut()` is not rate limited, and video tracks advertise refresh (#6048) + +`Encoder.cut()` no longer spaces keyframes 500 ms apart: every call forces one on the next frame, so frequent calls produce all-keyframe video. +Debounce your own calls if you relied on the spacing. +Every `Broadcast.video()` track now advertises a refresh ratio of 2, so a producer registered there must open a group with a keyframe whenever `track.demand().wanted` passes the largest group. +The bundled `Encoder` already does; a custom producer has to as well. + ### A disabled rendition stays in the catalog (#4915) `enabled: false` keeps the rendition in the catalog with `enabled: false` instead of removing it, and viewers skip it. diff --git a/rs/moq-video/MIGRATE.md b/rs/moq-video/MIGRATE.md index f4e25fefa9e..e8292cbf31f 100644 --- a/rs/moq-video/MIGRATE.md +++ b/rs/moq-video/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Control::cut` is not rate limited, and capture advertises refresh (#6048) + +`encode::Control::cut` no longer spaces keyframes 500 ms apart: every request forces one on the next frame, so frequent calls produce all-keyframe video. +Debounce your own calls if you relied on the spacing. +Capture tracks now advertise `refresh: Some(2)` when every backend the selection may open honors a cut, and the driver forces a keyframe when `Demand::wanted` passes the largest group. + ### OpenH264 is opt-in (#5771) The default features drop `openh264`, so a default build has no software H.264 encoder or decoder and a GPU-less host fails to open one.