From fd11688ebd30658da9f03b17d8cea4672ba8e3e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 19:49:32 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m1/precut-fixes/video-decode-lifecycle.md Quest-Claim: 4005c6eda7b94f8d From 300893b426082b7e1fcb53e5a44e6950be2a6047 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 20:16:54 -0800 Subject: [PATCH 2/4] fix(moq-video): a failed drain keeps its boundary, and colour switches stay bounded Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/decode/backend/mod.rs | 8 +- rs/moq-video/src/decode/backend/probe.rs | 19 ++- rs/moq-video/src/decode/consumer.rs | 168 ++++++++++++++++++++--- rs/moq-video/src/decode/decoder.rs | 76 +++++++++- rs/moq-video/src/decode/sink.rs | 7 +- rs/moq-video/src/decode/vui.rs | 30 ++-- 6 files changed, 272 insertions(+), 36 deletions(-) diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index 850aa2472c6..ee2d9b18ad5 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -297,6 +297,12 @@ const NAMED_ONLY: &[Candidate] = &[ open: probe::Buffered::open, drives: Drives::Own, }, + Candidate { + name: probe::FAILING_FLUSH_NAME, + supports: |c| matches!(c, Codec::H264), + open: probe::Buffered::open_failing_flush, + drives: Drives::Own, + }, Candidate { name: probe::REORDER_NAME, supports: |c| matches!(c, Codec::H264), @@ -305,7 +311,7 @@ const NAMED_ONLY: &[Candidate] = &[ }, Candidate { name: probe::HOLDING_NAME, - supports: |c| matches!(c, Codec::H264), + supports: |c| matches!(c, Codec::H264 | Codec::H265), open: probe::Reorder::open_holding, drives: Drives::Own, }, diff --git a/rs/moq-video/src/decode/backend/probe.rs b/rs/moq-video/src/decode/backend/probe.rs index 3b6aa933ae4..ebfde73fdf1 100644 --- a/rs/moq-video/src/decode/backend/probe.rs +++ b/rs/moq-video/src/decode/backend/probe.rs @@ -25,6 +25,10 @@ pub(crate) const NAME: &str = "probe"; /// A test decoder that holds one picture until the next call or a flush. pub(crate) const BUFFERED_NAME: &str = "probe-buffered"; +/// A test decoder that holds one picture like [`BUFFERED_NAME`], whose flush +/// readies it and then fails, as a codec that errors partway through a drain. +pub(crate) const FAILING_FLUSH_NAME: &str = "probe-failing-flush"; + /// A test decoder that reorders like B-frames do: it holds two pictures and /// returns the earliest once it has a third. It hands pictures back at /// nanosecond scale, as MediaCodec does, whatever scale the access unit came in. @@ -148,6 +152,7 @@ pub(crate) const SIZE: Size = Size { pub(crate) struct Probe(VecDeque); pub(crate) struct Buffered { + name: &'static str, held: Option, ready: VecDeque, } @@ -180,6 +185,15 @@ impl Probe { impl Buffered { pub(crate) fn open(_stream: &Stream, _config: &Config) -> Result, Error> { Ok(Box::new(Self { + name: BUFFERED_NAME, + held: None, + ready: VecDeque::new(), + })) + } + + pub(crate) fn open_failing_flush(_stream: &Stream, _config: &Config) -> Result, Error> { + Ok(Box::new(Self { + name: FAILING_FLUSH_NAME, held: None, ready: VecDeque::new(), })) @@ -381,11 +395,14 @@ impl Backend for Buffered { fn flush(&mut self) -> Result<(), Error> { self.ready.extend(self.held.take()); + if self.name == FAILING_FLUSH_NAME { + return Err(anyhow::anyhow!("probe: flush failed").into()); + } Ok(()) } fn name(&self) -> &str { - BUFFERED_NAME + self.name } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 810b1610d9c..c2d8a121810 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -350,16 +350,20 @@ impl Reader { // Drained so the pictures from before the boundary all come out ahead of it, // rather than whenever the media after it pushes them through. The decoder // stays usable and restarts on the next group's keyframe. + // The marker is consumed, so a failed drain still queues it and arms recovery + // before reporting, or the next read would never see it. Some(Encoded::Gap { timestamp, duration }) => { - self.drain().await?; + let drained = self.drain().await; self.pending.push_back((Read::Gap { timestamp, duration }, None)); self.recover = true; + drained?; continue; } // A fresh encoder has no preceding GOP to reference, so no recovery. Some(Encoded::End { end, .. }) => { - self.drain().await?; + let drained = self.drain().await; self.pending.push_back((Read::End { end }, None)); + drained?; continue; } None => None, @@ -370,7 +374,7 @@ impl Reader { // dropped before the drain ran retries it rather than reporting // an end the stream has not reached. Flushing twice is safe: the // second hands back nothing. - let tail = self.decoder.flush().await; + let drained = self.drain().await; // Set before the error is returned, not after. A flush that // failed once fails the same way every time, and the track has // ended either way, so leaving the flag down turns one bad @@ -378,13 +382,7 @@ impl Reader { // nothing in between to wait on. A caller that treats a codec // error as one lost picture and carries on then spins. self.drained = true; - // Queue what the flush readied before reporting its error, so a - // later read still hands those pictures over. - let received = self.receive(); - // Whatever the flush did not return never comes back. - self.arrivals.clear(); - tail?; - received?; + drained?; continue; }; @@ -452,15 +450,14 @@ impl Reader { Ok(()) } - /// Queue every picture the decoder still holds. + /// Queue every picture the decoder still holds. A flush that fails still + /// queues what it readied, so a later read hands those pictures over. async fn drain(&mut self) -> Result<(), Error> { - let drained = match self.decoder.flush().await { - Ok(()) => self.receive(), - Err(err) => Err(err), - }; + let flushed = self.decoder.flush().await; + let received = self.receive(); // Whatever the flush did not return never comes back. self.arrivals.clear(); - drained + flushed.and(received) } } @@ -1144,6 +1141,145 @@ mod tests { assert_eq!(reads, ["frame 100000", "end 133332", "frame 200000", "none"]); } + /// Every read until the track is done, as text: a codec error is one read too. + async fn read_all(consumer: &mut Consumer) -> Vec { + let mut reads = Vec::new(); + loop { + reads.push(match consumer.read().await { + Ok(Some(Read::Frame(frame))) => format!("frame {}", frame.timestamp.as_micros().unwrap()), + Ok(Some(Read::Gap { timestamp, .. })) => format!("gap {}", timestamp.as_micros().unwrap()), + Ok(Some(Read::End { end })) => format!("end {}", end.as_micros().unwrap()), + Ok(None) => return reads, + Err(_) => "error".to_string(), + }); + } + } + + /// A drain that fails at an end still delivers the pictures it readied ahead + /// of the end, and the end itself, after reporting the error. + #[tokio::test] + async fn a_failed_drain_keeps_its_end() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + let picture = |micros: u64| moq_mux::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), + duration: Some(Timestamp::from_micros(33_333).unwrap()), + payload: Bytes::from_static(b"access unit"), + keyframe: true, + }; + producer.write(picture(100_000)).unwrap(); + producer.cut(None).unwrap(); + producer.discontinuity().unwrap(); + producer.write(picture(200_000)).unwrap(); + let _finished = producer.finish().unwrap(); + + let catalog = VideoConfig::new(hang::catalog::H264 { + inline: true, + profile: 0x42, + constraints: 0, + level: 30, + }); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "video", + Options { + decoder: Config { + kind: Kind::Named(probe::FAILING_FLUSH_NAME.into()), + ..Config::new() + }, + delay: Delay { + max: Some(std::time::Duration::from_secs(10)), + ..Delay::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + + assert_eq!( + read_all(&mut consumer).await, + ["error", "frame 100000", "end 133332", "error", "frame 200000"] + ); + } + + /// A drain that fails at a gap still delivers the gap after the pictures it + /// readied, and still trims the leading pictures of the recovery group. + #[tokio::test(start_paused = true)] + async fn a_failed_drain_keeps_its_gap_and_recovery() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let catalog = VideoConfig::new(hang::catalog::H264 { + inline: true, + profile: 0x42, + constraints: 0, + level: 30, + }); + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "video", + Options { + decoder: Config { + kind: Kind::Named(probe::FAILING_FLUSH_NAME.into()), + ..Config::new() + }, + delay: Delay { + max: Some(std::time::Duration::from_secs(10)), + ..Delay::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + let wire = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let write = |sequence, stamps: &[u64]| { + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); + let frames: Vec<_> = stamps + .iter() + .enumerate() + .map(|(index, stamp)| moq_mux::container::Frame { + timestamp: Timestamp::from_micros(*stamp).unwrap(), + duration: None, + payload: Bytes::from_static(b"access unit"), + keyframe: index == 0, + }) + .collect(); + wire.write(&mut group, &frames).unwrap(); + group.finish().unwrap(); + }; + write(10, &[1000, 1100]); + let first = consumer.next_frame().await.unwrap().unwrap(); + assert_eq!(first.timestamp.as_micros().unwrap(), 1000); + + consumer.reader().track.set_delay(Delay::default()); + write(11, &[2000]); + write(12, &[3000, 2800, 2900, 3100]); + let _track = track.finish().unwrap(); + let reads = read_all(&mut consumer).await; + assert_eq!( + reads[..3], + ["error", "frame 1100", "gap 1100"], + "the drained picture, then its gap: {reads:?}" + ); + assert_eq!( + reads[3..], + ["frame 3000", "error", "frame 3100"], + "recovery trims the leading pictures: {reads:?}" + ); + } + /// A read dropped while a threaded flush is in flight loses nothing: the /// next read finishes that same flush rather than issuing another into a /// poisoned sink, and only then reports the end. diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 5516d65d3ae..539773d972a 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -139,6 +139,12 @@ impl Conversion { } } +/// The most colour switches a decoder waits on. Each one is a keyframe whose +/// pictures the backend still holds behind earlier ones, so a decoder never needs +/// more than it holds pictures: at most 16 for H.264 and H.265, and this leaves +/// room for a hardware pipeline beyond that, as the consumer's arrivals do. +const SWITCHES: usize = 32; + /// Decodes container payloads (the codec bitstream) into raw [`Frame`]s. /// /// The bring-your-own-payload layer under [`Consumer`](super::Consumer): use it @@ -294,13 +300,34 @@ impl Decoder { /// Convert one access unit and hand it to the backend. fn submit(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { let access_unit = self.conversion.access_unit(payload, keyframe)?; - if keyframe - && let Some(color) = super::vui::signalled(self.codec, &access_unit) - && self.switches.back().map_or(self.color, |(_, color)| *color) != color + let signalled = keyframe.then(|| super::vui::signalled(self.codec, &access_unit)).flatten(); + // A refused keyframe yields no picture, so its switch would wait for one forever. + self.backend.decode(access_unit, timestamp, keyframe)?; + if keyframe { + self.switch(timestamp, signalled); + } + Ok(()) + } + + /// Record the space a keyframe at `timestamp` switched to, if it signalled one. + fn switch(&mut self, timestamp: Timestamp, signalled: Option>) { + // Nothing decoded before a keyframe presents at or after it, so a switch there was + // never going to be reached: only timestamps that run backwards leave one. + while self.switches.back().is_some_and(|(at, _)| *at >= timestamp) { + self.switches.pop_back(); + } + let Some(color) = signalled else { return }; + if self.switches.back().map_or(self.color, |(_, color)| *color) == color { + return; + } + // A backend that drops pictures leaves switches no output reaches. The oldest is + // applied early rather than kept, so the queue stays bounded. + if self.switches.len() == SWITCHES + && let Some((_, color)) = self.switches.pop_front() { - self.switches.push_back((timestamp, color)); + self.color = color; } - self.backend.decode(access_unit, timestamp, keyframe) + self.switches.push_back((timestamp, color)); } /// Pop the oldest ready raw frame, or `None` while the decoder holds the rest. @@ -875,6 +902,45 @@ mod tests { assert_eq!(tail[0].surface.color(), Some(Color::BT601)); } + /// The colour switches a decoder waits on stay bounded whatever a publisher's + /// keyframes signal: spaces that alternate while no picture comes back, and + /// keyframe timestamps that run backwards. + #[test] + fn color_switches_stay_bounded() { + let mut catalog = hang::catalog::VideoConfig::new(hang::catalog::H265 { + in_band: true, + profile_space: 0, + profile_idc: 2, + profile_compatibility_flags: [0x20, 0, 0, 0], + tier_flag: false, + level_idc: 0x1e, + constraint_flags: [0x90, 0, 0, 0, 0, 0], + }); + catalog.coded_width = Some(64); + catalog.coded_height = Some(64); + let kind = super::Kind::Named(probe::HOLDING_NAME.into()); + let mut decoder = super::Decoder::new(&catalog, &decode_config(kind)).unwrap(); + let keyframe = |index: u64| { + let sps = match index % 2 { + 0 => super::super::vui::SPS_HDR10, + _ => super::super::vui::SPS_NVENC, + }; + bytes::Bytes::from([&[0, 0, 0, 1], sps].concat()) + }; + + for index in 0..100u64 { + let at = Timestamp::from_micros(index * 1_000).unwrap(); + decoder.decode(&keyframe(index), at, true).unwrap(); + assert!(decoder.switches.len() <= super::SWITCHES); + } + assert_eq!(decoder.switches.len(), super::SWITCHES); + for index in (0..100u64).rev() { + let at = Timestamp::from_micros(index).unwrap(); + decoder.decode(&keyframe(index), at, true).unwrap(); + assert!(decoder.switches.len() <= 1, "a switch after a later keyframe stays queued"); + } + } + /// A VP9 catalog that declares BT.2020 PQ labels the pictures a backend /// hands back unlabelled with it, rather than the coded size's guess; one /// with the codec string's defaults keeps the guess. diff --git a/rs/moq-video/src/decode/sink.rs b/rs/moq-video/src/decode/sink.rs index 714af4cbd68..7ff8fc7f4e7 100644 --- a/rs/moq-video/src/decode/sink.rs +++ b/rs/moq-video/src/decode/sink.rs @@ -163,8 +163,11 @@ mod threaded { let _ = resp.send(decoded); } Request::Flush { resp } => { - let flushed = decoder.flush().and_then(|()| collect(&mut decoder, &queue)); - let _ = resp.send(flushed); + // Collected even when the flush failed, so the pictures it readied come out + // ahead of whatever the caller queues behind the drain. + let flushed = decoder.flush(); + let collected = collect(&mut decoder, &queue); + let _ = resp.send(flushed.and(collected)); } } } diff --git a/rs/moq-video/src/decode/vui.rs b/rs/moq-video/src/decode/vui.rs index fb26cdeb53c..a97769e51bd 100644 --- a/rs/moq-video/src/decode/vui.rs +++ b/rs/moq-video/src/decode/vui.rs @@ -122,6 +122,23 @@ fn range(full: bool) -> Range { } } +/// An H.265 SPS signalling BT.2020 and PQ (moq-mux's `SPS_HDR10`, a 64x64 x265 encode). +#[cfg(test)] +pub(super) const SPS_HDR10: &[u8] = &[ + 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, 0xa0, + 0x20, 0x81, 0x04, 0xd9, 0x65, 0x65, 0x4a, 0x4c, 0x2f, 0x01, 0x6a, 0x12, 0x20, 0x12, 0x08, 0x00, 0x00, 0x03, 0x00, + 0x08, 0x00, 0x00, 0x03, 0x00, 0xc8, 0x40, +]; + +/// An H.265 SPS with no colour description, at 1280x720 (moq-mux's `SPS_NVENC`). +#[cfg(test)] +pub(super) const SPS_NVENC: &[u8] = &[ + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, + 0x02, 0x80, 0x80, 0x2e, 0x1f, 0x13, 0x96, 0x5d, 0x29, 0x08, 0x46, 0x45, 0xd5, 0x0c, 0x04, 0x04, 0x00, 0x00, 0x03, + 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf1, 0x80, 0x17, 0x79, 0x44, 0x00, 0x02, 0xdc, 0x6c, 0x00, 0x00, 0x2d, 0xc6, + 0xc0, 0x40, +]; + #[cfg(test)] mod tests { use super::*; @@ -206,11 +223,7 @@ mod tests { use crate::color::{Matrix, Primaries, Transfer}; let annexb = |nal: &[u8]| Bytes::from([&[0, 0, 0, 1], nal].concat()); - let hdr10 = annexb(&[ - 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, - 0xa0, 0x20, 0x81, 0x04, 0xd9, 0x65, 0x65, 0x4a, 0x4c, 0x2f, 0x01, 0x6a, 0x12, 0x20, 0x12, 0x08, 0x00, 0x00, - 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, 0xc8, 0x40, - ]); + let hdr10 = annexb(SPS_HDR10); let pq = Color { matrix: Matrix::Bt2020, range: Range::Limited, @@ -219,12 +232,7 @@ mod tests { }; assert_eq!(signalled(Codec::H265, &hdr10), Some(Some(pq))); - let nvenc = annexb(&[ - 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, - 0xa0, 0x02, 0x80, 0x80, 0x2e, 0x1f, 0x13, 0x96, 0x5d, 0x29, 0x08, 0x46, 0x45, 0xd5, 0x0c, 0x04, 0x04, 0x00, - 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf1, 0x80, 0x17, 0x79, 0x44, 0x00, 0x02, 0xdc, 0x6c, 0x00, - 0x00, 0x2d, 0xc6, 0xc0, 0x40, - ]); + let nvenc = annexb(SPS_NVENC); assert_eq!(signalled(Codec::H265, &nvenc), Some(Some(Color::BT709))); } } From 263d764ee3ddc2c2f49fabfaaf72cd2dae12dc68 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 21:14:52 -0800 Subject: [PATCH 3/4] fix(moq-video): capture drains its encoder before dropping it Co-Authored-By: Claude Opus 5.5 --- .../m1/precut-fixes/video-decode-lifecycle.md | 34 -------- rs/moq-video/src/decode/decoder.rs | 9 ++- rs/moq-video/src/encode/backend/mod.rs | 10 +++ rs/moq-video/src/encode/backend/probe.rs | 67 +++++++++++++++ rs/moq-video/src/encode/producer.rs | 81 ++++++++++++++++++- 5 files changed, 164 insertions(+), 37 deletions(-) delete mode 100644 quest/m1/precut-fixes/video-decode-lifecycle.md diff --git a/quest/m1/precut-fixes/video-decode-lifecycle.md b/quest/m1/precut-fixes/video-decode-lifecycle.md deleted file mode 100644 index 50d133ccee9..00000000000 --- a/quest/m1/precut-fixes/video-decode-lifecycle.md +++ /dev/null @@ -1,34 +0,0 @@ -# [S] moq-video keeps a boundary through a failed drain and bounds its colour queue - -## Goal - -The video decoder and capture keep their lifecycles whole: a Gap or End is -delivered and leading-picture recovery is armed even when the drain before -it fails; the colour-switch queue stays bounded by the decoder's reorder -depth whatever a publisher's keyframes signal; and capture flushes its -encoder before dropping it, so the frames it holds are published. - -## Plan - -Found by the 2026-11-17 pre-cut correctness audit, from source; reproduce -first (inject a flush error through the `probe` backend). - -- `rs/moq-video/src/decode/consumer.rs`: at a Gap or End, `self.drain().await?` - returns before queueing `Read::Gap`/`Read::End` and setting `recover`, and - the marker is already consumed. Keep the drained pictures ahead of their - boundary: take the drain's result without `?`, queue whatever it drained, - then the boundary, arm recovery, and only then return the drain's error. - Read the end-of-track branch, which already survives a failed drain. From - [moq-dev/moq#6475](https://github.com/moq-dev/moq/pull/6475). -- `rs/moq-video/src/decode/decoder.rs`: an entry pushes per keyframe whose SPS - changes colour space and pops only when a delivered picture reaches it; a - publisher alternating colour spaces with a dropped first picture grows it - until the next Gap. Drop entries at or after a new keyframe's timestamp and - cap at the reorder depth, as `arrivals` is. From - [moq-dev/moq#5808](https://github.com/moq-dev/moq/pull/5808). -- `rs/moq-video/src/encode/producer.rs` `capture_loop`: `drop(encoder)` with - no `flush()`, against the crate's own rule from #8372; Media Foundation and - MediaCodec lose their last frames on a source end or resize. On `release` - too. - -Public API: none. Wire: none. diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 539773d972a..fa502df749b 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -300,7 +300,9 @@ impl Decoder { /// Convert one access unit and hand it to the backend. fn submit(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result<(), Error> { let access_unit = self.conversion.access_unit(payload, keyframe)?; - let signalled = keyframe.then(|| super::vui::signalled(self.codec, &access_unit)).flatten(); + let signalled = keyframe + .then(|| super::vui::signalled(self.codec, &access_unit)) + .flatten(); // A refused keyframe yields no picture, so its switch would wait for one forever. self.backend.decode(access_unit, timestamp, keyframe)?; if keyframe { @@ -937,7 +939,10 @@ mod tests { for index in (0..100u64).rev() { let at = Timestamp::from_micros(index).unwrap(); decoder.decode(&keyframe(index), at, true).unwrap(); - assert!(decoder.switches.len() <= 1, "a switch after a later keyframe stays queued"); + assert!( + decoder.switches.len() <= 1, + "a switch after a later keyframe stays queued" + ); } } diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index 7dd95e9ccf3..1001b1c973a 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -354,6 +354,16 @@ const NAMED_ONLY: &[Candidate] = &[ #[cfg(target_os = "linux")] vulkan: false, }, + #[cfg(feature = "openh264")] + Candidate { + name: probe::PIPELINED, + codecs: &[Codec::H264], + open: probe::Pipelined::open, + cuts: true, + drives: Drives::Own, + #[cfg(target_os = "linux")] + vulkan: false, + }, ]; #[cfg(not(test))] diff --git a/rs/moq-video/src/encode/backend/probe.rs b/rs/moq-video/src/encode/backend/probe.rs index fd214fb1f81..70e32595066 100644 --- a/rs/moq-video/src/encode/backend/probe.rs +++ b/rs/moq-video/src/encode/backend/probe.rs @@ -22,6 +22,11 @@ pub(crate) const NAME: &str = "probe"; /// V4L2 driver that actually behaves that way. pub(crate) const NO_CUT: &str = "probe-nocut"; +/// OpenH264 held back by one access unit, like Media Foundation and MediaCodec: +/// a real bitstream a track can publish, whose last frame only a flush returns. +#[cfg(feature = "openh264")] +pub(crate) const PIPELINED: &str = "probe-pipelined"; + /// What happened to the codec, and where. `open` and `drop` are the pair that /// matters: the Windows backend opens a COM apartment in one and closes it in /// the other, so they have to land on the same thread. @@ -160,6 +165,68 @@ impl Backend for Probe { } } +#[cfg(feature = "openh264")] +pub(crate) struct Pipelined { + inner: Box, + /// The access unit held back by one. + held: Option, + ready: VecDeque, +} + +#[cfg(feature = "openh264")] +impl Pipelined { + pub(crate) fn open(config: &Config) -> Result, Error> { + Ok(Box::new(Self { + inner: super::openh264::Openh264::open(config)?, + held: None, + ready: VecDeque::new(), + })) + } + + /// Hold back the newest access unit the codec readied, releasing the one before it. + fn hold(&mut self) -> Result<(), Error> { + while let Some(encoded) = self.inner.receive()? { + self.ready.extend(self.held.replace(encoded)); + } + Ok(()) + } +} + +#[cfg(feature = "openh264")] +impl Backend for Pipelined { + fn encode(&mut self, frame: &Frame, cut: bool) -> Result<(), Error> { + self.inner.encode(frame, cut)?; + self.hold() + } + + fn receive(&mut self) -> Result, Error> { + Ok(self.ready.pop_front()) + } + + fn flush(&mut self) -> Result<(), Error> { + self.inner.flush()?; + self.hold()?; + self.ready.extend(self.held.take()); + Ok(()) + } + + fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { + self.inner.set_bitrate(bitrate) + } + + fn can_cut(&self) -> bool { + self.inner.can_cut() + } + + fn name(&self) -> &'static str { + PIPELINED + } + + fn applied(&self) -> super::super::Applied { + self.inner.applied() + } +} + impl Drop for Probe { fn drop(&mut self) { record("drop"); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 6af1d98e392..ce7c7d2f101 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -981,6 +981,8 @@ async fn capture_loop( // The camera's skips already accounted for, from its first read on: frames it replaced // while this encoder was still opening are startup, not frames the encoder fell behind on. let mut reported = None; + // An encode released mid-call leaves the sink refusing every later call, so it can't drain. + let mut cancelled = false; loop { // Race the next frame against demand, so an on-demand capture releases the @@ -1048,6 +1050,7 @@ async fn capture_loop( encoder.cut().await?; } if wait_capture(&demand, on_demand, encoder.encode(frame)).await?.is_none() { + cancelled = true; break; } while let Some(encoded) = encoder.receive()? { @@ -1079,14 +1082,31 @@ async fn capture_loop( if let Some(reported) = reported { producer.report_skipped(camera.skipped() - reported); } - // Drop the camera (LED off) and encoder before opening another. + // Drop the camera (LED off) and encoder before opening another. A backend that pipelines + // (Media Foundation, MediaCodec) still holds the last frames, so drain them into the track + // first. The break is marked even when the drain fails: only those frames are lost. drop(camera); + let drained = match cancelled { + true => Ok(()), + false => drain(&mut encoder, producer).await, + }; drop(encoder); capture_stopped(producer)?; + drained?; tracing::info!("capture stopped; released source"); } } +/// Publish every frame the encoder still holds. +#[cfg(feature = "capture")] +async fn drain(encoder: &mut Sink, producer: &mut Producer) -> Result<(), Error> { + encoder.flush().await?; + while let Some(encoded) = encoder.receive()? { + producer.publish(&encoded)?; + } + Ok(()) +} + #[cfg(feature = "capture")] fn map_capture_timestamp(epoch_micros: u64, timestamp: Timestamp) -> Result { let capture_micros = u64::try_from(timestamp.as_micros()?).unwrap_or(u64::MAX); @@ -2361,6 +2381,65 @@ mod tests { () = until(|| probe::take().iter().any(|(event, _)| *event == "encode")) => {} } } + /// A camera that ends drains the encoder into the track before dropping it, so the + /// frame a pipelining backend still holds is published rather than lost. + #[tokio::test] + async fn a_source_end_publishes_the_frames_the_encoder_holds() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); + rendition.kind = encoder::Kind::Software; + let probed = rendition.probe().await.unwrap(); + let container = moq_mux::catalog::hang::Container::try_from(&probed).unwrap(); + let mut producer = Producer::new( + track, + catalog.reserve().with_interval(rendition.effective_join()), + probed, + ) + .unwrap(); + let subscriber = broadcast + .consume() + .track("video") + .unwrap() + .subscribe( + moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(600))), + ) + .await + .unwrap(); + let mut published = moq_mux::container::Consumer::new(subscriber, container); + + let (camera, stream) = camera(); + let mut source = Streams::new([stream]); + let options = Capture { + kind: encoder::Kind::Named(probe::PIPELINED.into()), + ..Capture::default() + }; + let opens = source.opens.clone(); + let clock = catalog.clock(); + let state = starting(); + let run = capture_loop(&mut producer, &mut source, &options, &clock, &state); + + // Held back by one, the only frame reaches the track through the drain alone. + camera.push_native(surface(), us(0)); + camera.close(); + // The loop reopening the camera has handled the end. + tokio::select! { + res = run => panic!("the capture ended: {res:?}"), + () = until(|| opens.load(Ordering::SeqCst) == 2) => {} + } + let _finished = producer.finish().unwrap(); + loop { + match published.read().await.unwrap() { + Some(moq_mux::container::Read::Frame { .. }) => break, + Some(_) => {} + None => panic!("the encoder's last frame was never published"), + } + } + } } /// Rate control against the probe backend, which logs every bitrate it accepts. From 25e71cff61175d51f9cf4d66b9415df64e838eb6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 21:18:11 -0800 Subject: [PATCH 4/4] fix(moq-video): capture follows a track move its drain caused Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/encode/producer.rs | 48 ++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index ce7c7d2f101..b3080fd4ae2 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1091,6 +1091,12 @@ async fn capture_loop( false => drain(&mut encoder, producer).await, }; drop(encoder); + // The drained frames can move the rendition to a new track, as any published frame can. + if producer.track().name() != demand.name() { + demand = producer.demand(); + tracing::info!(track = %demand.name(), "capture moved to a new track"); + *claim = options.bandwidth.reserve(&demand, ceiling); + } capture_stopped(producer)?; drained?; tracing::info!("capture stopped; released source"); @@ -1481,6 +1487,11 @@ mod tests { impl Fixture { /// Start the capture loop on a broadcast whose clock began `behind` ago, at `wall`. async fn start(behind: Duration, wall: Wall) -> Self { + Self::start_on(behind, wall, encoder::Kind::Software).await + } + + /// [`start`](Self::start), capturing through the encoder `kind`. + async fn start_on(behind: Duration, wall: Wall, kind: encoder::Kind) -> Self { let time = kio::clock::Manual::new().install(&tokio::runtime::Handle::current()); time.advance(behind).unwrap(); let epoch = kio::clock::now() @@ -1517,7 +1528,7 @@ mod tests { let mut source = Opens(rx); // Each subscription opens the camera, so a fixture can idle and restart it. let options = Capture { - kind: encoder::Kind::Software, + kind, on_demand: true, ..Capture::default() }; @@ -1796,6 +1807,41 @@ mod tests { .await } + /// A frame that only the drain at a source end returns can still move the rendition to a + /// new track, and capture follows it there rather than watching the ended one. + #[tokio::test] + async fn a_drained_frame_that_moves_the_track_is_followed() { + tokio::task::LocalSet::new() + .run_until(async { + let kind = encoder::Kind::Named(crate::encode::backend::probe::PIPELINED.into()); + let fixture = Fixture::start_on(Duration::from_secs(1), kio::clock::wall(), kind).await; + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + camera.push_native(surface(), us(0)); + camera.close(); + read(&mut track).await; + + // Reopened four times the size, its one frame comes out of the drain alone. + fixture.time.advance(Duration::from_millis(40)).unwrap(); + let camera = fixture.camera_sized(640, 480); + camera.push_native(surface_sized(640, 480), us(40_000)); + camera.close(); + while track.read().await.unwrap().is_some() {} + + let mut moved = fixture.subscribe().await; + let first = read(&mut moved).await; + // A viewer of the new track opens the camera again. + fixture.time.advance(Duration::from_millis(40)).unwrap(); + let camera = fixture.camera_sized(640, 480); + camera.push_native(surface_sized(640, 480), us(80_000)); + camera.close(); + let second = read_new(&mut moved, &[first]).await; + assert!(second > first, "the new track carries the reopened camera"); + fixture.finish().await; + }) + .await + } + /// A reopen onto a mode the probed level cannot carry is a new codec string, so it moves /// the rendition to a new track: the old one ends, and a viewer who follows the catalog /// there reopens the camera on it.