From d53ae60e9b28813fb996d78302cce6dca8c1ed5c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 14:38:33 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m1/explicit-state/video-drain.md Quest-Claim: 3dc9d0dcd1f9efc6 From 70e014a479627c08fd209e96a213ca509215e07d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 15:01:57 -0800 Subject: [PATCH 2/6] fix(video): codec drains resume or fail for good, and the keyframe gate opens on a decoded keyframe Co-Authored-By: Claude Opus 5.5 --- rs/moq-v4l/src/io/mmap/stream.rs | 55 +++--- rs/moq-v4l/src/io/mod.rs | 11 ++ rs/moq-v4l/src/io/userptr/stream.rs | 51 +++--- rs/moq-video/src/decode/backend/mediacodec.rs | 65 +++++-- rs/moq-video/src/decode/backend/nvdec.rs | 5 +- rs/moq-video/src/decode/backend/openh264.rs | 77 ++++---- rs/moq-video/src/decode/backend/v4l2.rs | 165 +++++++++++++----- rs/moq-video/src/decode/backend/vaapi.rs | 48 +++-- rs/moq-video/src/decode/backend/vpx.rs | 47 +++-- rs/moq-video/src/decode/decoder.rs | 43 ++++- rs/moq-video/src/encode/backend/mediacodec.rs | 78 ++++++--- rs/moq-video/src/encode/backend/v4l2.rs | 88 +++++++++- rs/moq-video/src/v4l2.rs | 13 +- 13 files changed, 518 insertions(+), 228 deletions(-) diff --git a/rs/moq-v4l/src/io/mmap/stream.rs b/rs/moq-v4l/src/io/mmap/stream.rs index 93dfe2a7564..45b2a8b4c2e 100644 --- a/rs/moq-v4l/src/io/mmap/stream.rs +++ b/rs/moq-v4l/src/io/mmap/stream.rs @@ -5,6 +5,7 @@ use std::{io, mem, sync::Arc}; use crate::buffer::{Metadata, Type}; use crate::device::{Device, Handle}; use crate::io::mmap::arena::Arena; +use crate::io::State; use crate::io::traits::{CaptureStream, OutputStream, Stream as StreamTrait}; use crate::memory::Memory; use crate::sys::*; @@ -20,11 +21,7 @@ pub struct Stream<'a> { buf_type: Type, buf_meta: Vec, timeout: Option, - - active: bool, - /// Whether userspace holds the buffer at `arena_index`, so the next frame - /// queues it back first. A dequeue that timed out holds none. - held: bool, + state: State, } impl<'a> Stream<'a> { @@ -63,9 +60,8 @@ impl<'a> Stream<'a> { arena_index: 0, buf_type, buf_meta, - active: false, timeout: None, - held: false, + state: State::Stopped, }) } @@ -124,7 +120,7 @@ impl<'a> StreamTrait for Stream<'a> { )?; } - self.active = true; + self.state = State::Streaming { held: false }; Ok(()) } @@ -138,7 +134,7 @@ impl<'a> StreamTrait for Stream<'a> { )?; } - self.active = false; + self.state = State::Stopped; Ok(()) } } @@ -179,7 +175,9 @@ impl<'a, 'b> CaptureStream<'b> for Stream<'a> { )?; } self.arena_index = v4l2_buf.index as usize; - self.held = true; + if let State::Streaming { held } = &mut self.state { + *held = true; + } self.buf_meta[self.arena_index] = Metadata { bytesused: v4l2_buf.bytesused, @@ -193,20 +191,27 @@ impl<'a, 'b> CaptureStream<'b> for Stream<'a> { } fn next(&'b mut self) -> io::Result<(&'b Self::Item, &'b Metadata)> { - if !self.active { - // STREAMOFF handed every buffer back, so none is held over. - self.held = false; - // Enqueue all buffers once on stream start - for index in 0..self.arena.bufs.len() { - CaptureStream::queue(self, index)?; + match self.state { + State::Stopped => { + // Enqueue all buffers once on stream start. A start that fails + // partway hands back what it queued, which STREAMOFF does even on a + // queue that never streamed, so the next call starts from the same + // place rather than queueing a buffer twice. + let started = (0..self.arena.bufs.len()) + .try_for_each(|index| CaptureStream::queue(self, index)) + .and_then(|()| self.start()); + if let Err(err) = started { + self.stop()?; + return Err(err); + } } - - self.start()?; - } else if self.held { - // After a timed-out dequeue the driver already holds every buffer, and - // queueing one twice fails with EINVAL. - CaptureStream::queue(self, self.arena_index)?; - self.held = false; + State::Streaming { held: true } => { + // After a timed-out dequeue the driver already holds every buffer, and + // queueing one twice fails with EINVAL. + CaptureStream::queue(self, self.arena_index)?; + self.state = State::Streaming { held: false }; + } + State::Streaming { held: false } => {} } self.arena_index = CaptureStream::dequeue(self)?; @@ -273,8 +278,8 @@ impl<'a, 'b> OutputStream<'b> for Stream<'a> { } fn next(&'b mut self) -> io::Result<(&'b mut Self::Item, &'b mut Metadata)> { - let init = !self.active; - if !self.active { + let init = self.state == State::Stopped; + if init { self.start()?; } diff --git a/rs/moq-v4l/src/io/mod.rs b/rs/moq-v4l/src/io/mod.rs index 983175c20b1..6ef013c5922 100644 --- a/rs/moq-v4l/src/io/mod.rs +++ b/rs/moq-v4l/src/io/mod.rs @@ -2,3 +2,14 @@ pub mod traits; pub mod mmap; pub mod userptr; + +/// Whether a stream is running, and what userspace holds while it is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum State { + /// Not streaming: every buffer is userspace's, none of them queued. + Stopped, + /// Streaming. `held` is whether userspace holds the buffer at + /// `arena_index`, so the next frame queues it back first. A dequeue that + /// timed out holds none. + Streaming { held: bool }, +} diff --git a/rs/moq-v4l/src/io/userptr/stream.rs b/rs/moq-v4l/src/io/userptr/stream.rs index 816e3833d1a..2a761ca79eb 100644 --- a/rs/moq-v4l/src/io/userptr/stream.rs +++ b/rs/moq-v4l/src/io/userptr/stream.rs @@ -4,6 +4,7 @@ use std::{io, mem, sync::Arc}; use crate::buffer::{Metadata, Type}; use crate::device::{Device, Handle}; +use crate::io::State; use crate::io::traits::{CaptureStream, Stream as StreamTrait}; use crate::io::userptr::arena::Arena; use crate::memory::Memory; @@ -20,11 +21,7 @@ pub struct Stream { buf_type: Type, buf_meta: Vec, timeout: Option, - - active: bool, - /// Whether userspace holds the buffer at `arena_index`, so the next frame - /// queues it back first. A dequeue that timed out holds none. - held: bool, + state: State, } impl Stream { @@ -63,9 +60,8 @@ impl Stream { arena_index: 0, buf_type, buf_meta, - active: false, timeout: None, - held: false, + state: State::Stopped, }) } @@ -124,7 +120,7 @@ impl StreamTrait for Stream { )?; } - self.active = true; + self.state = State::Streaming { held: false }; Ok(()) } @@ -138,7 +134,7 @@ impl StreamTrait for Stream { )?; } - self.active = false; + self.state = State::Stopped; Ok(()) } } @@ -183,7 +179,9 @@ impl<'a> CaptureStream<'a> for Stream { )?; } self.arena_index = v4l2_buf.index as usize; - self.held = true; + if let State::Streaming { held } = &mut self.state { + *held = true; + } self.buf_meta[self.arena_index] = Metadata { bytesused: v4l2_buf.bytesused, @@ -197,20 +195,27 @@ impl<'a> CaptureStream<'a> for Stream { } fn next(&'a mut self) -> io::Result<(&'a Self::Item, &'a Metadata)> { - if !self.active { - // STREAMOFF handed every buffer back, so none is held over. - self.held = false; - // Enqueue all buffers once on stream start - for index in 0..self.arena.bufs.len() { - self.queue(index)?; + match self.state { + State::Stopped => { + // Enqueue all buffers once on stream start. A start that fails + // partway hands back what it queued, which STREAMOFF does even on a + // queue that never streamed, so the next call starts from the same + // place rather than queueing a buffer twice. + let started = (0..self.arena.bufs.len()) + .try_for_each(|index| self.queue(index)) + .and_then(|()| self.start()); + if let Err(err) = started { + self.stop()?; + return Err(err); + } } - - self.start()?; - } else if self.held { - // After a timed-out dequeue the driver already holds every buffer, and - // queueing one twice fails with EINVAL. - self.queue(self.arena_index)?; - self.held = false; + State::Streaming { held: true } => { + // After a timed-out dequeue the driver already holds every buffer, and + // queueing one twice fails with EINVAL. + self.queue(self.arena_index)?; + self.state = State::Streaming { held: false }; + } + State::Streaming { held: false } => {} } self.arena_index = self.dequeue()?; diff --git a/rs/moq-video/src/decode/backend/mediacodec.rs b/rs/moq-video/src/decode/backend/mediacodec.rs index 5ffe0a115cb..703598341c2 100644 --- a/rs/moq-video/src/decode/backend/mediacodec.rs +++ b/rs/moq-video/src/decode/backend/mediacodec.rs @@ -115,13 +115,38 @@ pub(crate) struct MediaCodec { image_generation: u64, /// Codec output buffers released for rendering but not yet acquired as images. pending_images: usize, - /// Whether this stream has accepted any input and therefore needs EOS. - fed: bool, + /// Whether the codec holds a stream to drain, or failed for good. + input: Input, /// Whether the last acquire found every slot held, so the stall is reported /// once on the way in rather than on every poll. stalled: bool, } +/// Where the codec's input stands between flushes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Input { + /// Nothing has gone in since the codec started or was last flushed, so + /// there is no tail to drain. + Idle, + /// Access units have gone in, so a flush signals end of stream and drains. + Fed, + /// A flush could not take the codec back out of end of stream, so it + /// refuses input for good. + Failed, +} + +impl Input { + /// Fails once the codec can take no more input. + fn check(self) -> Result<(), Error> { + match self { + Input::Failed => Err(Error::Codec(anyhow::anyhow!( + "the MediaCodec decoder failed to flush after a drain and takes no more input" + ))), + Input::Idle | Input::Fed => Ok(()), + } + } +} + #[derive(Default)] struct ImageSignal { generation: Mutex, @@ -211,7 +236,7 @@ impl MediaCodec { images, image_generation: 0, pending_images: 0, - fed: false, + input: Input::Idle, stalled: false, })) } @@ -254,7 +279,7 @@ impl MediaCodec { self.codec .queue_input_buffer(buffer, 0, access_unit.len(), time, 0) .map_err(|e| codec_err("queue an input buffer", e))?; - self.fed = true; + self.input = Input::Fed; true } DequeuedInputBufferResult::TryAgainLater => false, @@ -371,10 +396,6 @@ impl MediaCodec { /// Signal end of input and wait until every delayed output buffer is released. fn drain_tail(&mut self) -> Result, Error> { let mut out = Vec::new(); - if !self.fed { - return Ok(out); - } - self.signal_end_of_input(&mut out)?; let mut ended = false; for _ in 0..DRAIN_ROUNDS { @@ -442,6 +463,7 @@ impl MediaCodec { impl Backend for MediaCodec { fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + self.input.check()?; let mut out = Vec::new(); // Collect what the codec finished while the caller was elsewhere, which is // also what frees the input buffer the submission below needs. @@ -452,15 +474,26 @@ impl Backend for MediaCodec { } fn flush(&mut self) -> Result, Error> { - let out = self.drain_tail()?; - if self.fed { - // EOS leaves the codec refusing input. Synchronous MediaCodec resumes after - // flush without another start call, and the front end requires the next - // access unit to be a keyframe after this method returns. - self.codec.flush().map_err(|e| codec_err("flush", e))?; - self.fed = false; + self.input.check()?; + if self.input == Input::Idle { + return Ok(Vec::new()); } - Ok(out) + + // Flushed whatever the drain did: one that never saw end of stream still + // left EOS signalled, or the codec somewhere short of it, and only a flush + // takes either back. Synchronous MediaCodec resumes after it without + // another start call, and the front end requires the next access unit to + // be a keyframe after this method returns. + let tail = self.drain_tail(); + // Whatever the codec still held is dropped by the flush, so no image is + // owed any more. + self.pending_images = 0; + if let Err(err) = self.codec.flush() { + self.input = Input::Failed; + return Err(codec_err("flush", err)); + } + self.input = Input::Idle; + tail } fn name(&self) -> &str { diff --git a/rs/moq-video/src/decode/backend/nvdec.rs b/rs/moq-video/src/decode/backend/nvdec.rs index f8b6529aca0..71a4ad6727e 100644 --- a/rs/moq-video/src/decode/backend/nvdec.rs +++ b/rs/moq-video/src/decode/backend/nvdec.rs @@ -232,9 +232,10 @@ impl Backend for Nvdec { payload: ptr::null(), timestamp: 0, }; - let frames = self.parse(&mut packet)?; + // Set ahead of the parse: whatever it did, the next access unit opens a + // new stream. self.discontinuity = true; - Ok(frames) + self.parse(&mut packet) } fn name(&self) -> &str { diff --git a/rs/moq-video/src/decode/backend/openh264.rs b/rs/moq-video/src/decode/backend/openh264.rs index 2b68dd0bee8..801ec550339 100644 --- a/rs/moq-video/src/decode/backend/openh264.rs +++ b/rs/moq-video/src/decode/backend/openh264.rs @@ -212,6 +212,43 @@ impl Openh264 { self.pending = kept.into_iter().collect(); } } + + /// The pictures the decoder still holds, each stamped with its access unit's + /// timestamp. + fn drain_tail(&mut self) -> Result, Error> { + let tail = self + .decoder + .flush_remaining() + .map_err(|e| Error::Codec(anyhow::anyhow!("openh264 flush: {e}")))?; + // `tail` borrows the decoder, so copy the pictures before taking their + // timestamps. The placeholder is replaced before any frame reaches a + // caller; an unstamped picture is dropped below. + let decoded: Vec = tail + .iter() + .map(|yuv| picture(yuv, Timestamp::ZERO)) + .collect::>()?; + drop(tail); + + let mut frames = Vec::with_capacity(decoded.len()); + let mut unstamped = 0usize; + for mut frame in decoded { + match self.picture_out() { + Some(timestamp) => { + frame.timestamp = timestamp; + frames.push(frame); + } + None => unstamped += 1, + } + } + if unstamped > 0 { + tracing::warn!( + decoder = NAME, + dropped = unstamped, + "flushed pictures had no timestamp waiting for them" + ); + } + Ok(frames) + } } /// Copies one decoded picture out of openh264's own buffers. @@ -261,43 +298,14 @@ impl Backend for Openh264 { /// Return the reordered pictures still buffered at the end of the stream. fn flush(&mut self) -> Result, Error> { - let tail = self - .decoder - .flush_remaining() - .map_err(|e| Error::Codec(anyhow::anyhow!("openh264 flush: {e}")))?; - // `tail` borrows the decoder, so copy the pictures before taking their - // timestamps. The placeholder is replaced before any frame reaches a - // caller; an unstamped picture is dropped below. - let decoded: Vec = tail - .iter() - .map(|yuv| picture(yuv, Timestamp::ZERO)) - .collect::>()?; - drop(tail); - - let mut frames = Vec::with_capacity(decoded.len()); - let mut unstamped = 0usize; - for mut frame in decoded { - match self.picture_out() { - Some(timestamp) => { - frame.timestamp = timestamp; - frames.push(frame); - } - None => unstamped += 1, - } - } - if unstamped > 0 { - tracing::warn!( - decoder = NAME, - dropped = unstamped, - "flushed pictures had no timestamp waiting for them" - ); - } + let tail = self.drain_tail(); self.pending.clear(); self.lost = 0; - // `FlushFrame` leaks a single-threaded decoder's picture-pool slots, so - // replace the drained decoder before it is reused for another stream. + // Replaced whatever the drain did, so a failed one does not carry its + // state into the next stream. `FlushFrame` also leaks a single-threaded + // decoder's picture-pool slots, so a drained decoder is never reused. self.decoder = new_decoder()?; - Ok(frames) + tail } fn name(&self) -> &str { @@ -305,6 +313,7 @@ impl Backend for Openh264 { } } + #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/rs/moq-video/src/decode/backend/v4l2.rs b/rs/moq-video/src/decode/backend/v4l2.rs index 6c58653ef42..b80b01dcdf4 100644 --- a/rs/moq-video/src/decode/backend/v4l2.rs +++ b/rs/moq-video/src/decode/backend/v4l2.rs @@ -106,12 +106,7 @@ pub(crate) struct V4l2 { device: Device, /// The OUTPUT queue: access units going in. coded: Queue, - /// The CAPTURE queue, which exists only once the driver has reported the - /// stream's size. - pictures: Option, - /// When the first access unit went in, which is what - /// [`SOURCE_CHANGE_LIMIT`] is measured from. - since: Option, + phase: Phase, /// Access units the driver has taken, by the key their picture will carry, /// with the timestamp to give that picture. The buffer carries whole /// microseconds, and a [`Timestamp`] is an instant at its own scale, which a @@ -119,6 +114,26 @@ pub(crate) struct V4l2 { submitted: VecDeque<(Duration, Timestamp)>, } +/// Where the decoder stands, with what each stage needs. +enum Phase { + /// The driver has not reported the stream's size, so there is no CAPTURE + /// queue yet. `since` is when the first access unit went in, which is what + /// [`SOURCE_CHANGE_LIMIT`] is measured from. + Sizing { since: Option }, + /// Pictures come back on the negotiated CAPTURE queue. + Decoding(Pictures), + /// A drain left the decoder in a state it could not be brought back from, + /// so it takes no more input. + Failed, +} + +/// The error every call reports once the decoder has failed. +fn failed() -> Error { + Error::Codec(anyhow::anyhow!( + "the V4L2 decoder could not be restarted after a drain and takes no more input" + )) +} + /// The CAPTURE queue plus where a picture sits in one of its buffers. struct Pictures { queue: Queue, @@ -170,8 +185,7 @@ impl V4l2 { Ok(Box::new(Self { device, coded, - pictures: None, - since: None, + phase: Phase::Sizing { since: None }, submitted: VecDeque::new(), })) } @@ -228,9 +242,14 @@ impl V4l2 { Ok(reclaimed) } - /// Negotiate the CAPTURE queue against the size the driver has just reported. + /// Negotiate the CAPTURE queue against the size the driver has just reported, + /// releasing the old one first. + /// + /// The decoder is [`Phase::Failed`] unless this succeeds: the driver has + /// already consumed the source change, so nothing would announce the size + /// again. fn negotiate(&mut self) -> Result<(), Error> { - if let Some(pictures) = self.pictures.take() { + if let Phase::Decoding(pictures) = std::mem::replace(&mut self.phase, Phase::Failed) { pictures.queue.release(&self.device)?; } @@ -271,7 +290,7 @@ impl V4l2 { } queue.stream_on(&self.device)?; - self.pictures = Some(Pictures { queue, planes }); + self.phase = Phase::Decoding(Pictures { queue, planes }); Ok(()) } @@ -325,7 +344,7 @@ impl V4l2 { /// `V4L2_BUF_FLAG_LAST` on its last picture and then with `EPIPE` on any /// dequeue past it. fn drain(&mut self, frames: &mut Vec) -> Result { - let Some(pictures) = &self.pictures else { + let Phase::Decoding(pictures) = &self.phase else { return Ok(false); }; @@ -435,6 +454,73 @@ impl V4l2 { self.device.wait(POLL_INTERVAL); } } + + /// Restart both queues, leaving the decoder waiting for the next stream's + /// keyframe or [`Phase::Failed`]. + /// + /// Restarting OUTPUT is the seek sequence, which returns every access unit + /// and resumes at the next keyframe. It is also, with restarting CAPTURE, + /// the only way out of a drain that did not finish: `V4L2_DEC_CMD_START` + /// fails with `EBUSY` while one is in progress. Restarting CAPTURE also + /// clears an end of sequence the drain may already have dequeued. See + /// `Documentation/userspace-api/media/v4l/dev-decoder.rst`, "Drain" and + /// "Seek". + fn reset(&mut self) -> Result<(), Error> { + let restarted = self.restart(); + if restarted.is_err() { + self.phase = Phase::Failed; + } + restarted + } + + /// Restart both queues, handing every CAPTURE buffer back to the driver. + fn restart(&mut self) -> Result<(), Error> { + self.submitted.clear(); + self.coded.restart(&self.device)?; + let Phase::Decoding(pictures) = &mut self.phase else { + return Ok(()); + }; + pictures.queue.restart(&self.device)?; + while let Some(index) = pictures.queue.take_free() { + pictures.queue.queue(&self.device, index, &[], Duration::ZERO)?; + } + Ok(()) + } + + /// End the stream and drain it, returning the decoder ready for the next + /// stream's keyframe or [`Phase::Failed`]. + fn drain_stream(&mut self) -> Result, Error> { + let mut frames = Vec::new(); + loop { + // Refused, no drain began, so the decoder carries on as it was. + self.device.decoder_cmd(V4L2_DEC_CMD_STOP)?; + let changed = match self.drain_sequence() { + Ok((tail, changed)) => { + frames.extend(tail); + changed + } + Err(err) => { + self.reset()?; + return Err(err); + } + }; + + if changed { + self.negotiate()?; + } + if let Err(err) = self.device.decoder_cmd(V4L2_DEC_CMD_START) { + self.reset()?; + return Err(err); + } + + // A source change can leave work for the new format inside the decoder + // even after every OUTPUT buffer was returned. Resume it, then always run + // a fresh drain so the flush still covers everything submitted before it. + if !changed { + return Ok(frames); + } + } + } } /// The `struct timeval` an access unit rides the driver under, which the @@ -468,6 +554,9 @@ fn restore(submitted: &mut VecDeque<(Duration, Timestamp)>, key: Duration) -> Re impl Backend for V4l2 { fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + if let Phase::Failed = self.phase { + return Err(failed()); + } self.submit(&access_unit, timestamp)?; // Before the first negotiation there is nowhere for a picture to go, so the @@ -475,8 +564,8 @@ impl Backend for V4l2 { // missed resolution change would decode the rest of the stream at the old // geometry. let mut frames = Vec::new(); - if self.pictures.is_none() { - let since = *self.since.get_or_insert_with(Instant::now); + if let Phase::Sizing { since } = &mut self.phase { + let since = *since.get_or_insert_with(Instant::now); let deadline = Instant::now() + SOURCE_CHANGE_TIMEOUT; while !self.device.take_source_change() { if Instant::now() >= deadline { @@ -508,42 +597,24 @@ impl Backend for V4l2 { } fn flush(&mut self) -> Result, Error> { - // A drain only begins with both queues streaming. Before the driver has - // reported a format there is no CAPTURE queue and therefore no picture it - // can hand back. - if self.pictures.is_none() { - // OUTPUT may still hold undecodable access units. Restarting it triggers - // the stateful decoder's seek/reset sequence and returns those buffers, - // so none can surface after the next stream's keyframe. - self.coded.restart(&self.device)?; - self.submitted.clear(); - self.since = None; - return Ok(Vec::new()); - } - - let mut frames = Vec::new(); - loop { - self.device.decoder_cmd(V4L2_DEC_CMD_STOP)?; - let (tail, changed) = self.drain_sequence()?; - frames.extend(tail); - - if changed { - self.negotiate()?; + match self.phase { + Phase::Failed => Err(failed()), + // A drain only begins with both queues streaming. Before the driver has + // reported a format there is no CAPTURE queue and therefore no picture + // it can hand back. + Phase::Sizing { .. } => { + // OUTPUT may still hold undecodable access units. The seek returns + // them, so none can surface after the next stream's keyframe. + self.reset()?; + self.phase = Phase::Sizing { since: None }; + Ok(Vec::new()) } - self.device.decoder_cmd(V4L2_DEC_CMD_START)?; - - // A source change can leave work for the new format inside the decoder - // even after every OUTPUT buffer was returned. Resume it, then always run - // a fresh drain so the flush still covers everything submitted before it. - if changed { - continue; + Phase::Decoding(_) => { + let frames = self.drain_stream()?; + self.submitted.clear(); + Ok(frames) } - break; } - - self.submitted.clear(); - self.since = None; - Ok(frames) } fn name(&self) -> &str { diff --git a/rs/moq-video/src/decode/backend/vaapi.rs b/rs/moq-video/src/decode/backend/vaapi.rs index 3fe12428bc6..5aabfa78541 100644 --- a/rs/moq-video/src/decode/backend/vaapi.rs +++ b/rs/moq-video/src/decode/backend/vaapi.rs @@ -71,14 +71,22 @@ pub(crate) const NAME: &str = "vaapi"; pub(crate) struct Vaapi { decoder: Decoder, - /// Whether pictures are handed out as DMA-BUFs rather than downloaded, from - /// [`Config::output`]. Cleared if the driver turns out not to export, so a - /// host that decodes but cannot share what it decoded loses the fast path - /// rather than the stream. - exporting: bool, - /// Whether a picture has ever come back as a descriptor, which is what - /// settles the question above. See [`Vaapi::exported`]. - has_exported: bool, + export: Export, +} + +/// Whether pictures are handed out as DMA-BUFs rather than downloaded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Export { + /// [`Config::output`] asked for shared pictures, and none has come back yet, + /// so a failure is read as the driver being unable to export. See + /// [`Vaapi::exported`]. + Trying, + /// A picture has come back as a descriptor, which settles it. + Exporting, + /// Downloaded: asked for, or the driver turned out not to export, so a host + /// that decodes but cannot share what it decoded loses the fast path rather + /// than the stream. + Downloading, } impl Vaapi { @@ -130,23 +138,23 @@ impl Vaapi { Output::Cpu => false, Output::Render => cfg!(feature = "render"), }; + let export = match exporting { + true => Export::Trying, + false => Export::Downloading, + }; tracing::info!( decoder = NAME, codec = stream.codec.label(), exporting, "opened decoder" ); - Ok(Box::new(Self { - decoder, - exporting, - has_exported: false, - })) + Ok(Box::new(Self { decoder, export })) } /// Decodes one access unit into GPU-resident frames, or `None` once the /// driver has shown it will not export. fn decode_shared(&mut self, access_unit: &Bytes, timestamp: u64) -> Option, Error>> { - if !self.exporting { + if self.export == Export::Downloading { return None; } @@ -157,7 +165,7 @@ impl Vaapi { /// The same for the stream's tail, so a track that ends does not switch to /// downloading for its last few pictures. fn flush_shared(&mut self) -> Option, Error>> { - if !self.exporting { + if self.export == Export::Downloading { return None; } @@ -184,13 +192,17 @@ impl Vaapi { Ok(frames) => { // An access unit whose pictures are all still in the DPB proves // nothing about exporting, so only a non-empty answer counts. - self.has_exported |= !frames.is_empty(); + if !frames.is_empty() { + self.export = Export::Exporting; + } Ok(frames) } - Err(err) if self.has_exported => Err(Error::Codec(err.context("VAAPI decode to a shared surface"))), + Err(err) if self.export == Export::Exporting => { + Err(Error::Codec(err.context("VAAPI decode to a shared surface"))) + } Err(err) => { tracing::warn!(%err, "VAAPI cannot hand out decoded surfaces; downloading them instead"); - self.exporting = false; + self.export = Export::Downloading; Ok(Vec::new()) } } diff --git a/rs/moq-video/src/decode/backend/vpx.rs b/rs/moq-video/src/decode/backend/vpx.rs index 028baf44217..7438af3c2c0 100644 --- a/rs/moq-video/src/decode/backend/vpx.rs +++ b/rs/moq-video/src/decode/backend/vpx.rs @@ -31,12 +31,18 @@ pub(crate) const NAME: &str = "vpx"; pub(crate) struct Vpx { ctx: vpx_codec_ctx_t, codec: Codec, - /// A frame was refused, so the reference chain is broken until the next - /// keyframe. Delta frames in between are dropped rather than decoded against - /// references that are missing or wrong. - broken: bool, - /// Frames dropped since the chain broke, so a run of them logs once. - lost: u64, + chain: Chain, +} + +/// Whether the decoder holds the references the next delta frame needs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Chain { + Intact, + /// A frame was refused, so the chain is broken until a keyframe decodes. + /// Delta frames in between are dropped rather than decoded against + /// references that are missing or wrong. `lost` counts the frames dropped + /// since it broke, so a run of them logs once. + Broken { lost: u64 }, } impl Vpx { @@ -81,8 +87,7 @@ impl Vpx { Ok(Self { ctx, codec, - broken: false, - lost: 0, + chain: Chain::Intact, }) } @@ -123,9 +128,12 @@ impl Vpx { /// Records a frame the decoder refused, or one dropped while waiting for a /// keyframe, and logs the first of a run. fn picture_lost(&mut self, reason: &str) { - self.broken = true; - self.lost += 1; - if self.lost == 1 { + let lost = match self.chain { + Chain::Intact => 1, + Chain::Broken { lost } => lost + 1, + }; + self.chain = Chain::Broken { lost }; + if lost == 1 { tracing::warn!( decoder = NAME, codec = self.codec.label(), @@ -219,14 +227,9 @@ fn picture(codec: Codec, img: &vpx_image_t, timestamp: Timestamp) -> Result Result, Error> { - if self.broken { - if !keyframe { - self.picture_lost("reference chain broken"); - return Ok(Vec::new()); - } - tracing::info!(decoder = NAME, lost = self.lost, "picture recovered"); - self.broken = false; - self.lost = 0; + if matches!(self.chain, Chain::Broken { .. }) && !keyframe { + self.picture_lost("reference chain broken"); + return Ok(Vec::new()); } let len = u32::try_from(frame.len()).map_err(|_| { @@ -250,6 +253,12 @@ impl Backend for Vpx { self.picture_lost("corrupt frame"); return Ok(Vec::new()); } + // Only now the keyframe has decoded whole: one refused on the way back + // keeps counting into the same run. + if let Chain::Broken { lost } = self.chain { + tracing::info!(decoder = NAME, lost, "picture recovered"); + self.chain = Chain::Intact; + } // Without frame threading a call hands back at most the one picture it // shows, so the input timestamp is that picture's. A frame that is not diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index c7bfe4bc073..fb3cbb08941 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -265,13 +265,21 @@ impl Decoder { pub fn decode(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { // Wait for the first keyframe: a decoder started mid-GOP can't decode // delta frames, and the parameter sets ride along with the keyframe. - if !self.got_keyframe { - if !keyframe { - return Ok(Vec::new()); - } - self.got_keyframe = true; + if !self.got_keyframe && !keyframe { + return Ok(Vec::new()); + } + + let frames = self.submit(payload, timestamp, keyframe); + // Set by how the keyframe went rather than by its arrival: one that failed + // leaves nothing for the deltas behind it to predict from. + if keyframe { + self.got_keyframe = frames.is_ok(); } + self.deliver(frames?) + } + /// Convert one access unit and hand it to the backend. + fn submit(&mut self, payload: &Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { let access_unit = self.conversion.access_unit(payload, keyframe)?; if keyframe && let Some(color) = super::vui::signalled(self.codec, &access_unit) @@ -279,8 +287,7 @@ impl Decoder { { self.switches.push_back((timestamp, color)); } - let frames = self.backend.decode(access_unit, timestamp, keyframe)?; - self.deliver(frames) + self.backend.decode(access_unit, timestamp, keyframe) } /// Return the frames the backend still holds once the stream has ended. @@ -844,6 +851,28 @@ mod tests { assert_eq!(color(Default::default()), Some(Color::BT601)); } + /// A keyframe the backend fails to decode leaves the gate closed, so the + /// deltas behind it never reach a codec holding no reference for them. + #[test] + fn a_keyframe_that_fails_to_decode_keeps_the_gate_closed() { + let kind = super::Kind::Named(probe::IMMEDIATE_NAME.into()); + let mut decoder = super::Decoder::new(&probe_catalog(), &decode_config(kind)).unwrap(); + let at = |micros| Timestamp::from_micros(micros).unwrap(); + + let corrupt = bytes::Bytes::from_static(probe::CORRUPT); + assert!(decoder.decode(&corrupt, at(0), true).is_err()); + let delta = bytes::Bytes::from_static(&[0]); + assert!(decoder.decode(&delta, at(33_333), false).unwrap().is_empty()); + + // The next keyframe that decodes opens it. + assert_eq!(decoder.decode(&delta, at(66_666), true).unwrap().len(), 1); + assert_eq!(decoder.decode(&delta, at(99_999), false).unwrap().len(), 1); + + // And one that fails mid-stream closes it again. + assert!(decoder.decode(&corrupt, at(133_332), true).is_err()); + assert!(decoder.decode(&delta, at(166_665), false).unwrap().is_empty()); + } + /// An inline-H.264 catalog the test probes accept. fn probe_catalog() -> hang::catalog::VideoConfig { hang::catalog::VideoConfig::new(hang::catalog::H264 { diff --git a/rs/moq-video/src/encode/backend/mediacodec.rs b/rs/moq-video/src/encode/backend/mediacodec.rs index 7674a4841e1..19c360f079e 100644 --- a/rs/moq-video/src/encode/backend/mediacodec.rs +++ b/rs/moq-video/src/encode/backend/mediacodec.rs @@ -149,9 +149,30 @@ pub(crate) struct MediaCodec { /// Set while an IDR is owed: asked for by the caller, or by a flush that /// dropped the reference frames the next picture would predict from. keyframe_pending: bool, - /// True once end of input has been signalled and before the flush that takes - /// it back, since a codec in that state refuses input. - ended: bool, + /// Whether the codec still takes input. What it owes a flush is `pending`. + input: Input, +} + +/// Whether the codec takes input. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Input { + /// Taking frames, and back to it after every flush. + Open, + /// End of input was signalled for the last time ([`Backend::finish`]), or a + /// flush could not take the codec back out of it, so it refuses input for good. + Ended, +} + +impl Input { + /// Fails once the codec can take no more input. + fn check(self) -> Result<(), Error> { + match self { + Input::Ended => Err(Error::Codec(anyhow::anyhow!( + "the MediaCodec encoder has ended its stream and takes no more input" + ))), + Input::Open => Ok(()), + } + } } impl MediaCodec { @@ -189,7 +210,7 @@ impl MediaCodec { last_timestamp: None, frame_index: 0, keyframe_pending: false, - ended: false, + input: Input::Open, })) } @@ -316,15 +337,7 @@ impl MediaCodec { /// reappears ahead of the next group's keyframe. fn drain_tail(&mut self) -> Result, Error> { let mut out = Vec::new(); - // Every frame fed has already come back out, so there is no tail to wait - // for and no reason to cycle the codec through end of stream. This is the - // common case: the format asks for a one-frame pipeline. - if self.ended || self.pending.is_empty() { - return Ok(out); - } - self.signal_end_of_input(&mut out)?; - self.ended = true; for _ in 0..DRAIN_ROUNDS { if self.drain(DRAIN_TIMEOUT, &mut out)? { @@ -377,6 +390,7 @@ impl MediaCodec { impl Backend for MediaCodec { fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + self.input.check()?; let mut out = Vec::new(); // Collect whatever the codec finished while the caller was elsewhere, so // the input buffer asked for below isn't stuck behind an output nobody @@ -388,22 +402,38 @@ impl Backend for MediaCodec { } fn flush(&mut self) -> Result, Error> { - let out = self.drain_tail()?; - if self.ended { - // End of input leaves the codec refusing input, and only a flush takes - // that back. Nothing more is needed in synchronous mode: the NDK - // documents the `AMediaCodec_start` after a flush as an asynchronous-mode - // requirement. - self.codec.flush().map_err(|e| codec_err("flush", e))?; - self.ended = false; - // A flush drops the reference frames, so the next picture has to be an - // IDR. Asked for rather than assumed: not every encoder inserts one. - self.keyframe_pending = true; + self.input.check()?; + // Every frame fed has already come back out, so there is no tail to wait + // for and no reason to cycle the codec through end of stream. This is the + // common case: the format asks for a one-frame pipeline. + if self.pending.is_empty() { + return Ok(Vec::new()); } - Ok(out) + + // Flushed whatever the drain did: end of input leaves the codec refusing + // input, a drain cut short included, and only a flush takes that back. + // Nothing more is needed in synchronous mode: the NDK documents the + // `AMediaCodec_start` after a flush as an asynchronous-mode requirement. + let tail = self.drain_tail(); + // The flush discards whatever the codec still held, so none of it will be + // paired with an output. + self.pending.clear(); + if let Err(err) = self.codec.flush() { + self.input = Input::Ended; + return Err(codec_err("flush", err)); + } + // A flush drops the reference frames, so the next picture has to be an + // IDR. Asked for rather than assumed: not every encoder inserts one. + self.keyframe_pending = true; + tail } fn finish(&mut self) -> Result, Error> { + self.input.check()?; + self.input = Input::Ended; + if self.pending.is_empty() { + return Ok(Vec::new()); + } self.drain_tail() } diff --git a/rs/moq-video/src/encode/backend/v4l2.rs b/rs/moq-video/src/encode/backend/v4l2.rs index 42e11ffe1a9..1d66650530f 100644 --- a/rs/moq-video/src/encode/backend/v4l2.rs +++ b/rs/moq-video/src/encode/backend/v4l2.rs @@ -120,6 +120,7 @@ pub(crate) struct V4l2 { size: Size, /// Frames the driver has taken and not yet answered with an access unit. pending: Pending, + phase: Phase, /// Whether the driver takes `VIDIOC_ENCODER_CMD`. Cleared the first time it /// refuses one, after which a flush can only take what the codec has already /// finished. @@ -130,6 +131,26 @@ pub(crate) struct V4l2 { cuts: bool, } +/// Whether the queues stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Phase { + /// Neither queue streams: no frame has gone in since open or since a drain + /// that failed stopped them. The next frame starts both. + Stopped, + /// Both queues stream. + Streaming, + /// The queues could not be started or stopped, so the encoder takes no + /// more frames. + Failed, +} + +/// The error every call reports once the encoder has failed. +fn failed() -> Error { + Error::Codec(anyhow::anyhow!( + "the V4L2 encoder could not start or stop its queues and takes no more frames" + )) +} + impl V4l2 { pub(crate) fn open(config: &Config) -> Result, Error> { let size = config.size(); @@ -255,6 +276,7 @@ impl V4l2 { planes, size, pending: Pending::default(), + phase: Phase::Stopped, drainable: true, cuts, })) @@ -265,7 +287,24 @@ impl V4l2 { /// bcm2835-codec wants the first OUTPUT buffer queued before `STREAMON` and /// the CAPTURE queue started last. The ordering is legal on every driver, so /// it is unconditional rather than a quirk to detect. + /// + /// A start that fails partway is stopped again, so the next frame starts + /// from the same place rather than finding one queue streaming without the + /// other. fn start(&mut self) -> Result<(), Error> { + match self.start_queues() { + Ok(()) => { + self.phase = Phase::Streaming; + Ok(()) + } + Err(err) => { + self.stop()?; + Err(err) + } + } + } + + fn start_queues(&mut self) -> Result<(), Error> { self.raw.stream_on(&self.device)?; while let Some(index) = self.coded.take_free() { self.coded.queue(&self.device, index, &[], Duration::ZERO)?; @@ -273,6 +312,27 @@ impl V4l2 { self.coded.stream_on(&self.device) } + /// Stop both queues, which takes every buffer back from the driver and ends + /// a drain in progress, or leave the encoder [`Phase::Failed`]. + /// + /// Stopping is the only way out of a drain that did not finish: + /// `V4L2_ENC_CMD_START` fails with `EBUSY` while one is in progress. Stopping + /// CAPTURE also resets the encoder, so the next frame begins a stream that + /// needs nothing from before. See + /// `Documentation/userspace-api/media/v4l/dev-encoder.rst`, "Drain" and + /// "Reset". + fn stop(&mut self) -> Result<(), Error> { + self.phase = Phase::Failed; + self.raw.stop(&self.device)?; + self.coded.stop(&self.device)?; + // Frames the driver handed back unencoded, which no access unit answers, + // and a header carried for a stream the reset encoder starts over. + self.pending.forget(); + self.pending.header = None; + self.phase = Phase::Stopped; + Ok(()) + } + /// Take a raw buffer, waiting for the driver to release one if it holds them /// all. fn free_buffer(&mut self) -> Result { @@ -370,11 +430,13 @@ impl V4l2 { /// boundary would time out and take the broadcast with it. bcm2835-codec /// happens to be one-in-one-out, so a Pi would not have shown this. fn drain_tail(&mut self) -> Result, Error> { - // The sequence needs both queues streaming. `VIDIOC_ENCODER_CMD` succeeds - // without starting one otherwise, and there is nothing to drain before the - // first frame anyway. - if !self.raw.streaming() || !self.coded.streaming() { - return Ok(Vec::new()); + match self.phase { + Phase::Failed => return Err(failed()), + // The sequence needs both queues streaming. `VIDIOC_ENCODER_CMD` + // succeeds without starting one otherwise, and nothing is held + // before a frame goes in anyway. + Phase::Stopped => return Ok(Vec::new()), + Phase::Streaming => {} } if self.drainable @@ -397,8 +459,17 @@ impl V4l2 { } } - /// Run the rest of the drain sequence, up to and including the restart. + /// Run the rest of the drain sequence, up to and including the restart, or + /// stop both queues when it fails. fn drain_to_last(&mut self) -> Result, Error> { + let drained = self.resume_after_last(); + if drained.is_err() { + self.stop()?; + } + drained + } + + fn resume_after_last(&mut self) -> Result, Error> { let mut out = Vec::new(); let mut deadline = Instant::now() + FLUSH_TIMEOUT; loop { @@ -466,6 +537,9 @@ impl V4l2 { impl Backend for V4l2 { fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { + if self.phase == Phase::Failed { + return Err(failed()); + } if frame.size() != self.size { return Err(Error::Codec(anyhow::anyhow!( "V4L2 encoder opened for {} was given a {} frame", @@ -501,7 +575,7 @@ impl Backend for V4l2 { self.raw.queue(&self.device, index, &bytesused, key)?; self.pending.queued(key, frame.timestamp); - if !self.raw.streaming() { + if self.phase == Phase::Stopped { self.start()?; } diff --git a/rs/moq-video/src/v4l2.rs b/rs/moq-video/src/v4l2.rs index b99c4b466bc..999cea0b80c 100644 --- a/rs/moq-video/src/v4l2.rs +++ b/rs/moq-video/src/v4l2.rs @@ -875,19 +875,20 @@ impl Queue { Ok(()) } - /// Stop and restart a queue, taking every buffer back from the driver. - pub(crate) fn restart(&mut self, device: &Device) -> Result<(), Error> { + /// Stop the queue, taking every buffer back from the driver. + pub(crate) fn stop(&mut self, device: &Device) -> Result<(), Error> { if self.streaming { device.stream(self.dir, false)?; self.streaming = false; } self.free = (0..self.buffers.len() as u32).collect(); - self.stream_on(device) + Ok(()) } - /// Whether the queue has been started. - pub(crate) fn streaming(&self) -> bool { - self.streaming + /// Stop and restart a queue, taking every buffer back from the driver. + pub(crate) fn restart(&mut self, device: &Device) -> Result<(), Error> { + self.stop(device)?; + self.stream_on(device) } /// Buffers currently owned by the driver. From e0b03d26152be4b0455b20af4066bbb6888fbdbd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 15:27:16 -0800 Subject: [PATCH 3/6] fix(video): Media Foundation and VideoToolbox drains, and the vpx/VAAPI/V4L2-stream flag pairs, are explicit state Co-Authored-By: Claude Opus 5.5 --- rs/moq-v4l/src/io/mmap/stream.rs | 2 +- rs/moq-v4l/src/io/userptr/stream.rs | 2 +- .../src/decode/backend/mediafoundation.rs | 46 ++-- rs/moq-video/src/decode/backend/openh264.rs | 1 - .../src/decode/backend/videotoolbox.rs | 136 +++++++----- rs/moq-video/src/decode/backend/vpx.rs | 4 +- .../src/encode/backend/mediafoundation.rs | 209 ++++++++++++------ 7 files changed, 251 insertions(+), 149 deletions(-) diff --git a/rs/moq-v4l/src/io/mmap/stream.rs b/rs/moq-v4l/src/io/mmap/stream.rs index 45b2a8b4c2e..68c10047aa5 100644 --- a/rs/moq-v4l/src/io/mmap/stream.rs +++ b/rs/moq-v4l/src/io/mmap/stream.rs @@ -5,8 +5,8 @@ use std::{io, mem, sync::Arc}; use crate::buffer::{Metadata, Type}; use crate::device::{Device, Handle}; use crate::io::mmap::arena::Arena; -use crate::io::State; use crate::io::traits::{CaptureStream, OutputStream, Stream as StreamTrait}; +use crate::io::State; use crate::memory::Memory; use crate::sys::*; use crate::v4l2; diff --git a/rs/moq-v4l/src/io/userptr/stream.rs b/rs/moq-v4l/src/io/userptr/stream.rs index 2a761ca79eb..4a6bcbe76d9 100644 --- a/rs/moq-v4l/src/io/userptr/stream.rs +++ b/rs/moq-v4l/src/io/userptr/stream.rs @@ -4,9 +4,9 @@ use std::{io, mem, sync::Arc}; use crate::buffer::{Metadata, Type}; use crate::device::{Device, Handle}; -use crate::io::State; use crate::io::traits::{CaptureStream, Stream as StreamTrait}; use crate::io::userptr::arena::Arena; +use crate::io::State; use crate::memory::Memory; use crate::sys::*; use crate::v4l2; diff --git a/rs/moq-video/src/decode/backend/mediafoundation.rs b/rs/moq-video/src/decode/backend/mediafoundation.rs index 497952e75be..e4420daca06 100644 --- a/rs/moq-video/src/decode/backend/mediafoundation.rs +++ b/rs/moq-video/src/decode/backend/mediafoundation.rs @@ -41,10 +41,10 @@ use windows::Win32::Media::MediaFoundation::{ MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_STREAM_CHANGE, MF_LOW_LATENCY, MF_MT_FRAME_SIZE, MF_MT_MAJOR_TYPE, MF_MT_MINIMUM_DISPLAY_APERTURE, MF_MT_SUBTYPE, MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample, MFMediaType_Video, MFOffset, MFSampleExtension_Discontinuity, MFT_CATEGORY_VIDEO_DECODER, - MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_SYNCMFT, MFT_MESSAGE_COMMAND_DRAIN, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, - MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, - MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoArea, MFVideoFormat_H264, - MFVideoFormat_HEVC, MFVideoFormat_NV12, + MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_SYNCMFT, MFT_MESSAGE_COMMAND_DRAIN, MFT_MESSAGE_COMMAND_FLUSH, + MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, + MFT_OUTPUT_DATA_BUFFER, MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoArea, + MFVideoFormat_H264, MFVideoFormat_HEVC, MFVideoFormat_NV12, }; use windows::core::{GUID, Interface}; @@ -476,23 +476,31 @@ impl Backend for MediaFoundation { } fn flush(&mut self) -> Result, Error> { - let mut out = Vec::new(); - if self.output_configured { - unsafe { - self.transform - .ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) - .map_err(|e| mf_err("drain", e))?; - } - self.drain_output(&mut out)?; - // A synchronous MFT may accept input immediately after NEED_MORE_INPUT. - // START_OF_STREAM is still useful to implementations that track the - // notification, but it is optional for this MFT category. - unsafe { - let _ = self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); - } - } + // Whatever the drain does, the next access unit opens a new stream. self.pending.clear(); self.discontinuity = true; + if !self.output_configured { + return Ok(Vec::new()); + } + + let mut out = Vec::new(); + let drained = unsafe { self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) } + .map_err(|e| mf_err("drain", e)) + .and_then(|()| self.drain_output(&mut out)); + if let Err(err) = drained { + // A drain cut short leaves the MFT refusing input until it has handed + // back everything it holds. Flushing discards that instead, and is what + // a failure here leaves the decoder ready for the next keyframe with. + unsafe { self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) } + .map_err(|e| mf_err("flush after a failed drain", e))?; + return Err(err); + } + // A synchronous MFT may accept input immediately after NEED_MORE_INPUT. + // START_OF_STREAM is still useful to implementations that track the + // notification, but it is optional for this MFT category. + unsafe { + let _ = self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + } Ok(out) } diff --git a/rs/moq-video/src/decode/backend/openh264.rs b/rs/moq-video/src/decode/backend/openh264.rs index 801ec550339..004b133f2dd 100644 --- a/rs/moq-video/src/decode/backend/openh264.rs +++ b/rs/moq-video/src/decode/backend/openh264.rs @@ -313,7 +313,6 @@ impl Backend for Openh264 { } } - #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/rs/moq-video/src/decode/backend/videotoolbox.rs b/rs/moq-video/src/decode/backend/videotoolbox.rs index 0f2d84e28d9..e57518627f5 100644 --- a/rs/moq-video/src/decode/backend/videotoolbox.rs +++ b/rs/moq-video/src/decode/backend/videotoolbox.rs @@ -112,22 +112,39 @@ struct Sink { error: Option, } +/// A live decompression session and what it was built from. +struct Session { + session: CFRetained, + /// The format description the session and its samples use. + format: CFRetained, + /// The exact ordered parameter sets it was built from, so a mid-stream + /// change triggers a rebuild. + built_from: Vec, +} + +impl Session { + /// Finish the pictures the session delayed into the sink, then invalidate + /// it, returning the status of the wait. + /// + /// The wait is for every callback, so none can outlive the sink. + fn finish(self) -> i32 { + let status = unsafe { self.session.wait_for_asynchronous_frames() }; + unsafe { self.session.invalidate() }; + status + } +} + pub(crate) struct VideoToolbox { /// Which codec's parameter sets and format description to build (H.264 needs /// SPS+PPS; H.265 also needs VPS). codec: Codec, /// Built lazily once the parameter sets first arrive, rebuilt if they change. - session: Option>, - /// Format description the current session + samples use (kept in lockstep - /// with `session`). - format: Option>, + session: Option, /// Latest parameter sets seen, persisted across access units (a delta frame - /// carries none). `built_from` records the exact ordered set the live session - /// was built from, so a mid-stream parameter-set change triggers a rebuild. + /// carries none). vps: Option, sps: Option, pps: Option, - built_from: Option>, /// Synthetic presentation time assigned to the next input sample. sample_index: i64, /// Original container timestamps keyed by the synthetic presentation times @@ -157,11 +174,9 @@ impl VideoToolbox { let mut backend = Self { codec, session: None, - format: None, vps: None, sps: None, pps: None, - built_from: None, sample_index: 0, pending: VecDeque::new(), sink: Box::new(Sink::default()), @@ -188,8 +203,11 @@ impl VideoToolbox { } /// (Re)build the decompression session when the parameter sets first appear - /// or change. Returns `false` if we still don't have a complete set. - fn ensure_session(&mut self, vps: Option, sps: Option, pps: Option) -> Result { + /// or change, leaving none while the set is still incomplete. + /// + /// A session being replaced is finished first, so the pictures it delayed + /// land in the sink ahead of the new session's rather than being lost with it. + fn ensure_session(&mut self, vps: Option, sps: Option, pps: Option) -> Result<(), Error> { if let Some(vps) = vps { self.vps = Some(vps); } @@ -200,12 +218,24 @@ impl VideoToolbox { self.pps = Some(pps); } let Some(params) = self.param_sets() else { - return Ok(false); + return Ok(()); }; // Reuse the existing session if it was built from these exact sets. - if self.session.is_some() && self.built_from.as_ref() == Some(¶ms) { - return Ok(true); + if self + .session + .as_ref() + .is_some_and(|session| session.built_from == params) + { + return Ok(()); + } + if let Some(old) = self.session.take() { + let status = old.finish(); + if status != 0 { + return Err(Error::Codec(anyhow::anyhow!( + "VTDecompressionSessionWaitForAsynchronousFrames failed: {status}" + ))); + } } let format = create_format_description(self.codec, ¶ms)?; @@ -243,10 +273,12 @@ impl VideoToolbox { } })?; - self.session = Some(session); - self.format = Some(format); - self.built_from = Some(params); - Ok(true) + self.session = Some(Session { + session, + format, + built_from: params, + }); + Ok(()) } /// Pair one callback time back to the timestamp of its submitted access unit. @@ -278,37 +310,43 @@ impl Backend for VideoToolbox { // the slices into length-prefixed (4-byte) form. let Split { vps, sps, pps, avcc } = Split::new(self.codec, access_unit)?; - if !self.ensure_session(vps, sps, pps)? { - // No parameter sets yet (e.g. a delta frame before the first keyframe). - return Ok(Vec::new()); - } - if avcc.is_empty() { - // Parameter-set-only access unit: nothing to decode. - return Ok(Vec::new()); - } - - let sample_time = self.sample_index; - let format = self.format.as_ref().expect("format ensured above"); - let sample = make_sample_buffer(&avcc, format, sample_time)?; - let session = self.session.as_ref().expect("session ensured above"); - + // Cleared ahead of the rebuild, which finishes the old session's pictures + // into the sink. self.sink.frames.clear(); self.sink.error = None; + self.ensure_session(vps, sps, pps)?; + // No parameter sets yet (e.g. a delta frame before the first keyframe). + let Some(live) = &self.session else { + return Ok(Vec::new()); + }; - let status = unsafe { session.decode_frame(&sample, VTDecodeFrameFlags(0), ptr::null_mut(), ptr::null_mut()) }; - if status != 0 { - return Err(Error::Codec(anyhow::anyhow!( - "VTDecompressionSessionDecodeFrame failed: {status}" - ))); - } + // A parameter-set-only access unit has nothing to decode. + let decoded = if avcc.is_empty() { + None + } else { + let sample_time = self.sample_index; + let sample = make_sample_buffer(&avcc, &live.format, sample_time)?; + let status = unsafe { + live.session + .decode_frame(&sample, VTDecodeFrameFlags(0), ptr::null_mut(), ptr::null_mut()) + }; + if status != 0 { + return Err(Error::Codec(anyhow::anyhow!( + "VTDecompressionSessionDecodeFrame failed: {status}" + ))); + } + Some(sample_time) + }; if let Some(error) = self.sink.error.take() { return Err(Error::Codec(anyhow::anyhow!( "VideoToolbox decode callback failed: {error}" ))); } - remember_timestamp(&mut self.pending, sample_time, timestamp); - self.sample_index += 1; + if let Some(sample_time) = decoded { + remember_timestamp(&mut self.pending, sample_time, timestamp); + self.sample_index += 1; + } self.take_frames() } @@ -316,17 +354,7 @@ impl Backend for VideoToolbox { self.sink.frames.clear(); self.sink.error = None; - let status = if let Some(session) = self.session.take() { - // This finishes delayed pictures and waits for every callback before the - // session is invalidated, so no callback can outlive `sink`. - let status = unsafe { session.wait_for_asynchronous_frames() }; - unsafe { session.invalidate() }; - status - } else { - 0 - }; - self.format = None; - self.built_from = None; + let status = self.session.take().map_or(0, Session::finish); let result = if status != 0 { Err(Error::Codec(anyhow::anyhow!( @@ -591,11 +619,11 @@ mod tests { }; let mut backend = VideoToolbox::new(&stream).unwrap(); - let opened: *const VTDecompressionSession = &**backend.session.as_ref().expect("no session at open"); + let opened: *const VTDecompressionSession = &*backend.session.as_ref().expect("no session at open").session; let split = Split::new(Codec::H264, parameter_sets).unwrap(); - assert!(backend.ensure_session(split.vps, split.sps, split.pps).unwrap()); - let session: *const VTDecompressionSession = &**backend.session.as_ref().unwrap(); + backend.ensure_session(split.vps, split.sps, split.pps).unwrap(); + let session: *const VTDecompressionSession = &*backend.session.as_ref().expect("no session").session; assert!( ptr::eq(opened, session), "the keyframe's parameter sets rebuilt the session" diff --git a/rs/moq-video/src/decode/backend/vpx.rs b/rs/moq-video/src/decode/backend/vpx.rs index 7438af3c2c0..4375c1dcacf 100644 --- a/rs/moq-video/src/decode/backend/vpx.rs +++ b/rs/moq-video/src/decode/backend/vpx.rs @@ -42,7 +42,9 @@ enum Chain { /// Delta frames in between are dropped rather than decoded against /// references that are missing or wrong. `lost` counts the frames dropped /// since it broke, so a run of them logs once. - Broken { lost: u64 }, + Broken { + lost: u64, + }, } impl Vpx { diff --git a/rs/moq-video/src/encode/backend/mediafoundation.rs b/rs/moq-video/src/encode/backend/mediafoundation.rs index fa584b0bdd1..cf66267fbea 100644 --- a/rs/moq-video/src/encode/backend/mediafoundation.rs +++ b/rs/moq-video/src/encode/backend/mediafoundation.rs @@ -72,22 +72,7 @@ pub(crate) struct MediaFoundation { /// The color space of the input frames, stamped onto both media types so the /// encoder writes it into the bitstream's VUI. color: Color, - /// Lazily configured on the first frame, since the Direct3D11 device to bind - /// (for zero-copy texture input) comes from the frame itself. - started: bool, - /// The MFT allocates its own output samples (true for hardware encoders). - provides_samples: bool, - /// Output buffer size we must allocate when `provides_samples` is false. - /// Cached from `GetOutputStreamInfo` so the output hot path doesn't re-query. - output_size: u32, - /// True once the MFT has asked for input and we haven't fed it since. - needs_input: bool, - /// True once the MFT has reported its drain complete, so a drain knows the - /// tail is out rather than still coming. - drained: bool, - /// Set while the next sample is the first of a restarted stream, so it can be - /// marked as such. - discontinuity: bool, + phase: Phase, sample_index: i64, /// Frames handed to the MFT that haven't come back out, oldest first: the /// sample time we gave the codec paired with the frame's real timestamp. This @@ -104,6 +89,38 @@ pub(crate) struct MediaFoundation { _com: ComGuard, } +/// Where the MFT stands. +enum Phase { + /// Not configured yet: that happens on the first frame, since the + /// Direct3D11 device to bind (for zero-copy texture input) comes from the + /// frame itself. + Unstarted, + Streaming(Stream), + /// The stream ended for the last time ([`Backend::finish`]), or the MFT + /// failed to start, drain, or restart, so it takes no more frames. + Ended, +} + +/// What a streaming MFT needs between calls. +struct Stream { + output: Output, + /// True once the MFT has asked for input and we haven't fed it since. + needs_input: bool, + /// Set while the next sample is the first of a restarted stream, so it can be + /// marked as such. + discontinuity: bool, +} + +/// Who allocates the output samples, cached from `GetOutputStreamInfo` so the +/// output hot path doesn't re-query. +#[derive(Clone, Copy, Debug)] +enum Output { + /// The MFT allocates its own (true for hardware encoders). + Provided, + /// We allocate them, this many bytes each. + Allocated(u32), +} + impl MediaFoundation { pub(crate) fn open(config: &Config) -> Result, Error> { let format = OutputFormat::for_codec(config.codec); @@ -143,12 +160,7 @@ impl MediaFoundation { bitrate: clamp_u32(config.resolved_bitrate().as_bps()), gop: config.keyframe_interval(), color: config.resolved_color(), - started: false, - provides_samples: false, - output_size: 0, - needs_input: false, - drained: false, - discontinuity: false, + phase: Phase::Unstarted, sample_index: 0, pending: VecDeque::new(), last_timestamp: None, @@ -159,7 +171,11 @@ impl MediaFoundation { /// One-time configuration, deferred to the first frame so a texture frame can /// bind its own Direct3D11 device for zero-copy input. + /// + /// A start that fails leaves the encoder [`Phase::Ended`]: the MFT is + /// configured partway, which nothing here can take back. fn start(&mut self, frame: &Surface) -> Result<(), Error> { + self.phase = Phase::Ended; // Bind the frame's D3D11 device when it's a texture, so the MFT reads the // captured surface directly. A CPU frame runs the MFT in system memory. if let Surface::Texture(texture) = frame { @@ -176,7 +192,7 @@ impl MediaFoundation { self.configure_codec_api()?; self.set_output_type()?; self.set_input_type()?; - self.read_output_info()?; + let output = self.read_output_info()?; unsafe { self.transform @@ -186,21 +202,39 @@ impl MediaFoundation { .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) .map_err(|e| mf_err("start of stream", e))?; } - self.started = true; + self.phase = Phase::Streaming(Stream { + output, + needs_input: false, + discontinuity: false, + }); Ok(()) } - /// Cache whether the MFT provides its own output samples and, if not, how - /// big a buffer we must allocate. Re-read after a format renegotiation. - fn read_output_info(&mut self) -> Result<(), Error> { + /// Whether the MFT provides its own output samples and, if not, how big a + /// buffer we must allocate. Re-read after a format renegotiation. + fn read_output_info(&self) -> Result { let info = unsafe { self.transform .GetOutputStreamInfo(0) .map_err(|e| mf_err("GetOutputStreamInfo", e))? }; - self.provides_samples = info.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES.0 as u32 != 0; - self.output_size = info.cbSize; - Ok(()) + Ok(match info.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES.0 as u32 != 0 { + true => Output::Provided, + false => Output::Allocated(info.cbSize), + }) + } + + /// The stream's state, which only a started encoder has. + fn stream(&mut self) -> Result<&mut Stream, Error> { + match &mut self.phase { + Phase::Streaming(stream) => Ok(stream), + Phase::Unstarted => Err(Error::Codec(anyhow::anyhow!( + "the mediafoundation encoder has not started" + ))), + Phase::Ended => Err(Error::Codec(anyhow::anyhow!( + "the mediafoundation encoder has ended its stream and takes no more frames" + ))), + } } fn configure_codec_api(&self) -> Result<(), Error> { @@ -419,7 +453,7 @@ impl MediaFoundation { /// that arrives meanwhile. Runs on the dedicated encode thread (see /// `encode::sink`), so this blocking wait never parks a tokio worker. fn wait_for_input(&mut self, out: &mut Vec) -> Result<(), Error> { - while !self.needs_input { + while !self.stream()?.needs_input { let event = unsafe { self.events .GetEvent(MF_EVENT_FLAG_NONE) @@ -434,14 +468,19 @@ impl MediaFoundation { fn drain_ready(&mut self, out: &mut Vec) -> Result<(), Error> { loop { match unsafe { self.events.GetEvent(MF_EVENT_FLAG_NO_WAIT) } { - Ok(event) => self.handle_event(&event, out)?, + Ok(event) => { + self.handle_event(&event, out)?; + } Err(e) if e.code() == MF_E_NO_EVENTS_AVAILABLE => return Ok(()), Err(e) => return Err(mf_err("GetEvent (drain)", e)), } } } - fn handle_event(&mut self, event: &IMFMediaEvent, out: &mut Vec) -> Result<(), Error> { + /// Act on one event from the MFT, returning whether it reported a drain + /// complete, which is how a drain knows the tail is out rather than still + /// coming. + fn handle_event(&mut self, event: &IMFMediaEvent, out: &mut Vec) -> Result { // Surface an async failure (e.g. an MEError event) instead of looping in // `wait_for_input` forever for an input request that will never come. unsafe { event.GetStatus() } @@ -454,29 +493,30 @@ impl MediaFoundation { const HAVE_OUTPUT: u32 = METransformHaveOutput.0 as u32; const DRAIN_COMPLETE: u32 = METransformDrainComplete.0 as u32; match unsafe { event.GetType().map_err(|e| mf_err("event GetType", e))? } { - NEED_INPUT => self.needs_input = true, + NEED_INPUT => self.stream()?.needs_input = true, HAVE_OUTPUT => { if let Some(packet) = self.process_output()? { out.push(packet); } } - DRAIN_COMPLETE => self.drained = true, + DRAIN_COMPLETE => return Ok(true), _ => {} } - Ok(()) + Ok(false) } /// Pull one encoded access unit, stamped with the timestamp of the frame it was /// encoded from. Returns `None` if the MFT had nothing ready or asked us to /// renegotiate the output type. fn process_output(&mut self) -> Result, Error> { - let provided = if self.provides_samples { - None - } else { - let buffer = unsafe { MFCreateMemoryBuffer(self.output_size).map_err(|e| mf_err("output buffer", e))? }; - let sample = unsafe { MFCreateSample().map_err(|e| mf_err("output sample", e))? }; - unsafe { sample.AddBuffer(&buffer).map_err(|e| mf_err("output AddBuffer", e))? }; - Some(sample) + let provided = match self.stream()?.output { + Output::Provided => None, + Output::Allocated(size) => { + let buffer = unsafe { MFCreateMemoryBuffer(size).map_err(|e| mf_err("output buffer", e))? }; + let sample = unsafe { MFCreateSample().map_err(|e| mf_err("output sample", e))? }; + unsafe { sample.AddBuffer(&buffer).map_err(|e| mf_err("output AddBuffer", e))? }; + Some(sample) + } }; let mut data = [MFT_OUTPUT_DATA_BUFFER { @@ -500,7 +540,7 @@ impl MediaFoundation { // The encoder revised its output format; re-apply ours, refresh the // cached output-buffer info, and retry on the next event. self.set_output_type()?; - self.read_output_info()?; + self.stream()?.output = self.read_output_info()?; return Ok(None); } Err(e) => return Err(mf_err("ProcessOutput", e)), @@ -532,14 +572,6 @@ impl MediaFoundation { /// error here rather than a block later. fn drain(&mut self) -> Result, Error> { let mut out = Vec::new(); - if !self.started { - return Ok(out); - } - - // Cleared here rather than after the wait: a drain we exited early once - // leaves its completion queued, and reading that later would let the next - // drain believe it was already done. - self.drained = false; unsafe { self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0) @@ -548,12 +580,43 @@ impl MediaFoundation { .ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) .map_err(|e| mf_err("drain", e))?; } - while !self.drained { + loop { let event = unsafe { self.events.GetEvent(MF_EVENT_FLAG_NONE) }.map_err(|e| mf_err("GetEvent (drain)", e))?; - self.handle_event(&event, &mut out)?; + if self.handle_event(&event, &mut out)? { + return Ok(out); + } + } + } + + /// Drain the stream and restart it, or end the encoder for good. + /// + /// A drain cut short leaves the MFT somewhere between streams, with its + /// completion perhaps still queued for the next drain to mistake for its own, + /// so nothing after one can be trusted. + fn cycle(&mut self) -> Result, Error> { + let cycled = self.drain().and_then(|out| { + // A drain leaves the MFT stopped: it stops asking for input until a new + // stream starts. + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) + .map_err(|e| mf_err("start of stream", e))?; + } + Ok(out) + }); + match cycled { + Ok(out) => { + let stream = self.stream()?; + stream.needs_input = false; + stream.discontinuity = true; + Ok(out) + } + Err(err) => { + self.phase = Phase::Ended; + Err(err) + } } - Ok(out) } /// The timestamp of the frame this output belongs to, found by the sample time @@ -597,7 +660,7 @@ impl MediaFoundation { impl Backend for MediaFoundation { fn encode(&mut self, frame: &Frame, cut: bool) -> Result, Error> { - if !self.started { + if let Phase::Unstarted = self.phase { self.start(&frame.surface)?; } @@ -609,7 +672,7 @@ impl Backend for MediaFoundation { } let sample = self.build_sample(&frame.surface)?; - if self.discontinuity { + if self.stream()?.discontinuity { // The first picture of a restarted stream is not temporally continuous // with the one before the drain, and only this says so: the encoder would // otherwise be free to predict across the seam. @@ -623,8 +686,9 @@ impl Backend for MediaFoundation { } // Cleared only once the sample is in: a rejected one never carried the mark // anywhere, so the next attempt still owns it. - self.discontinuity = false; - self.needs_input = false; + let stream = self.stream()?; + stream.discontinuity = false; + stream.needs_input = false; // Record the pairing before advancing the index, so this is the sample time // `build_sample` just stamped the sample with. let sample_time = self.sample_time(); @@ -636,23 +700,24 @@ impl Backend for MediaFoundation { } fn flush(&mut self) -> Result, Error> { - let out = self.drain()?; - if self.started { - // A drain leaves the MFT stopped: it stops asking for input until a new - // stream starts. - unsafe { - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) - .map_err(|e| mf_err("start of stream", e))?; - } - self.needs_input = false; - self.discontinuity = true; + if let Phase::Unstarted = self.phase { + return Ok(Vec::new()); } - Ok(out) + // An ended encoder fails here rather than being sent a drain. + self.stream()?; + self.cycle() } fn finish(&mut self) -> Result, Error> { - self.drain() + if let Phase::Unstarted = self.phase { + self.phase = Phase::Ended; + return Ok(Vec::new()); + } + // An ended encoder fails here rather than being sent a drain. + self.stream()?; + let out = self.drain(); + self.phase = Phase::Ended; + out } fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { From cb2aa2b1b975a8a7b60628f330cea775a313c018 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 15:28:09 -0800 Subject: [PATCH 4/6] quest: video codec drains are done Co-Authored-By: Claude Opus 5.5 --- quest/m1/explicit-state/README.md | 1 - quest/m1/explicit-state/video-drain.md | 60 -------------------------- 2 files changed, 61 deletions(-) delete mode 100644 quest/m1/explicit-state/video-drain.md diff --git a/quest/m1/explicit-state/README.md b/quest/m1/explicit-state/README.md index 2850e41ae87..f1811696ca4 100644 --- a/quest/m1/explicit-state/README.md +++ b/quest/m1/explicit-state/README.md @@ -71,7 +71,6 @@ which collapses that state and waits on [Drop set](/quest/m1/drop-set.md). - [Review and agent rule](/quest/m1/explicit-state/rule.md) - REVIEW.md and AGENTS.md ask for one enum per lifecycle, terminal transitions after fallible work, and one lock per value - [moq-boy pause state](/quest/m1/explicit-state/boy-pause.md) - a new viewer always wakes a paused emulator - [Transport close](/quest/m1/explicit-state/transport-close.md) - connection status ends with its error, and every Rust QUIC backend agrees when a connection is closed -- [Video codec drains](/quest/m1/explicit-state/video-drain.md) - a hardware codec whose drain fails is ready for input again or terminally failed, never left stuck - [Watch playback phases](/quest/m1/explicit-state/watch-playback.md) - watch counts a stall only while a rendition plays - [Publish capture phases](/quest/m1/explicit-state/publish-capture.md) - dismissing the screen-share picker turns sharing off - [js/qmux close](/quest/m1/explicit-state/qmux-close.md) - a graceful remote close ends qmux streams cleanly diff --git a/quest/m1/explicit-state/video-drain.md b/quest/m1/explicit-state/video-drain.md deleted file mode 100644 index 929146ceaf3..00000000000 --- a/quest/m1/explicit-state/video-drain.md +++ /dev/null @@ -1,60 +0,0 @@ -# [M] Video codec drains - -## Goal - -When a moq-video decoder's or encoder's flush or drain fails, for example a -drain that times out, the call returns with the codec either ready for input -again or terminally failed. It is never left in end-of-stream or stopped -while the front end keeps feeding it. A decoder opens its keyframe gate only -once a keyframe has actually decoded. This holds for MediaCodec and V4L2 -decode and encode, and for every other backend with the same drain shape. - -## Plan - -The shared decisions are in the [epic](/quest/m1/explicit-state/README.md). - -Real MediaCodec and V4L2 devices aren't in CI, so reproducing these may need -a seam the backends drive the codec through. The `vicodec`/`visl` virtual -V4L2 drivers are worth checking for the V4L2 half. - -The front end retries a failed read on purpose -(`rs/moq-video/src/decode/consumer.rs`: "Left idle, so the next read asks it -again"). That retry turns each hole below into a stuck codec: - -- **MediaCodec decode** (`decode/backend/mediacodec.rs`): - - `flush` returns at `drain_tail()?` before it reaches - `if self.fed { codec.flush() }`, so a drain that never sees EOS leaves the - codec in end-of-stream. - - `fed: bool` can't tell "fed" from "EOS already signalled". The next - `flush` signals EOS again instead of flushing, so the decoder stays stuck. - - Recommended: `enum Input { Idle, Fed, EndSignalled }`, flushing the codec - whenever it is `EndSignalled`. -- **V4L2 decode** (`decode/backend/v4l2.rs`): `V4L2_DEC_CMD_STOP` followed by - a failed `drain_sequence` skips `V4L2_DEC_CMD_START`, so the decoder stays - stopped and nothing records it. -- **V4L2 encode** (`encode/backend/v4l2.rs`): - - It has the same STOP/START hole. - - `start()` can leave the raw queue streaming with the coded queue off. - After that, `encode` never retries and `drain_tail` returns empty. -- **MediaCodec encode** (`encode/backend/mediacodec.rs`): `ended` is set - before the drain loop, and `encode` never checks it. -- **Keyframe gate** (`decode/decoder.rs`): - - `got_keyframe = true` is set before `backend.decode(...)?`. - - A keyframe that fails to decode still opens the gate, and the deltas that - follow reach a backend with no reference. - -Every caller ends the stream on an encoder error today, so the encoder holes -are latent. Fix them with the same state anyway rather than leave the twins -different. - -Also sweep: - -- `encode/backend/mediafoundation.rs`: five bools. -- `decode/backend/videotoolbox.rs`: three `Option`s kept in lockstep, and a - rebuild drops the old session without draining it. -- `encode/backend/v4l2.rs` and `rs/moq-v4l/src/io/*/stream.rs`: `active` plus - `held`, where a half-start leaves buffers queued. -- `vaapi.rs` and `vpx.rs`: the small flag pairs. - -[moq-dev/moq#7916](https://github.com/moq-dev/moq/pull/7916) also edits -`decode/decoder.rs`, so expect a merge conflict. From 301f41bc9ed7c0d61f36fbded22caeb09b3c03a0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 15:35:41 -0800 Subject: [PATCH 5/6] fix(video): stop a V4L2 queue that never streamed, and keep MF timestamps through the drain Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/decode/backend/mediafoundation.rs | 4 +++- rs/moq-video/src/v4l2.rs | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/rs/moq-video/src/decode/backend/mediafoundation.rs b/rs/moq-video/src/decode/backend/mediafoundation.rs index e4420daca06..21458fc0c8a 100644 --- a/rs/moq-video/src/decode/backend/mediafoundation.rs +++ b/rs/moq-video/src/decode/backend/mediafoundation.rs @@ -477,9 +477,9 @@ impl Backend for MediaFoundation { fn flush(&mut self) -> Result, Error> { // Whatever the drain does, the next access unit opens a new stream. - self.pending.clear(); self.discontinuity = true; if !self.output_configured { + self.pending.clear(); return Ok(Vec::new()); } @@ -487,6 +487,8 @@ impl Backend for MediaFoundation { let drained = unsafe { self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) } .map_err(|e| mf_err("drain", e)) .and_then(|()| self.drain_output(&mut out)); + // Only after the drain, which pairs the delayed pictures with these. + self.pending.clear(); if let Err(err) = drained { // A drain cut short leaves the MFT refusing input until it has handed // back everything it holds. Flushing discards that instead, and is what diff --git a/rs/moq-video/src/v4l2.rs b/rs/moq-video/src/v4l2.rs index 999cea0b80c..42de5b21b3c 100644 --- a/rs/moq-video/src/v4l2.rs +++ b/rs/moq-video/src/v4l2.rs @@ -876,11 +876,13 @@ impl Queue { } /// Stop the queue, taking every buffer back from the driver. + /// + /// Issued even on a queue that never streamed: buffers queued ahead of a + /// `STREAMON` that failed are still the driver's, and `STREAMOFF` is what + /// hands them back. pub(crate) fn stop(&mut self, device: &Device) -> Result<(), Error> { - if self.streaming { - device.stream(self.dir, false)?; - self.streaming = false; - } + device.stream(self.dir, false)?; + self.streaming = false; self.free = (0..self.buffers.len() as u32).collect(); Ok(()) } From 9af121882de6d271662af7c9af387d670a74df54 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 22:11:17 -0800 Subject: [PATCH 6/6] fix(video): latch unrecoverable drain failures as CodecGone; keep a held V4L2 buffer on restart Co-Authored-By: Claude Opus 5.5 --- rs/moq-v4l/src/io/mmap/stream.rs | 6 +++++- rs/moq-v4l/src/io/userptr/stream.rs | 6 +++++- rs/moq-video/src/decode/backend/mediacodec.rs | 10 ++++++---- rs/moq-video/src/decode/backend/mediafoundation.rs | 2 +- rs/moq-video/src/decode/backend/openh264.rs | 2 +- rs/moq-video/src/decode/backend/v4l2.rs | 4 +--- rs/moq-video/src/encode/backend/mediacodec.rs | 4 ++++ 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/rs/moq-v4l/src/io/mmap/stream.rs b/rs/moq-v4l/src/io/mmap/stream.rs index 68c10047aa5..10715ab3d45 100644 --- a/rs/moq-v4l/src/io/mmap/stream.rs +++ b/rs/moq-v4l/src/io/mmap/stream.rs @@ -120,7 +120,11 @@ impl<'a> StreamTrait for Stream<'a> { )?; } - self.state = State::Streaming { held: false }; + // STREAMON on a streaming queue does nothing, and leaves a dequeued + // buffer with userspace. + if let State::Stopped = self.state { + self.state = State::Streaming { held: false }; + } Ok(()) } diff --git a/rs/moq-v4l/src/io/userptr/stream.rs b/rs/moq-v4l/src/io/userptr/stream.rs index 4a6bcbe76d9..f598e2f47f9 100644 --- a/rs/moq-v4l/src/io/userptr/stream.rs +++ b/rs/moq-v4l/src/io/userptr/stream.rs @@ -120,7 +120,11 @@ impl StreamTrait for Stream { )?; } - self.state = State::Streaming { held: false }; + // STREAMON on a streaming queue does nothing, and leaves a dequeued + // buffer with userspace. + if let State::Stopped = self.state { + self.state = State::Streaming { held: false }; + } Ok(()) } diff --git a/rs/moq-video/src/decode/backend/mediacodec.rs b/rs/moq-video/src/decode/backend/mediacodec.rs index 703598341c2..7aa475e53d5 100644 --- a/rs/moq-video/src/decode/backend/mediacodec.rs +++ b/rs/moq-video/src/decode/backend/mediacodec.rs @@ -139,9 +139,9 @@ impl Input { /// Fails once the codec can take no more input. fn check(self) -> Result<(), Error> { match self { - Input::Failed => Err(Error::Codec(anyhow::anyhow!( - "the MediaCodec decoder failed to flush after a drain and takes no more input" - ))), + Input::Failed => Err(Error::CodecGone( + "the MediaCodec decoder failed a drain and takes no more input".into(), + )), Input::Idle | Input::Fed => Ok(()), } } @@ -492,7 +492,9 @@ impl Backend for MediaCodec { self.input = Input::Failed; return Err(codec_err("flush", err)); } - self.input = Input::Idle; + // A failed drain may leave a render queued on the reader, which the next + // stream would take for its own, so the decoder is not reused. + self.input = if tail.is_ok() { Input::Idle } else { Input::Failed }; tail } diff --git a/rs/moq-video/src/decode/backend/mediafoundation.rs b/rs/moq-video/src/decode/backend/mediafoundation.rs index 21458fc0c8a..a91288015a3 100644 --- a/rs/moq-video/src/decode/backend/mediafoundation.rs +++ b/rs/moq-video/src/decode/backend/mediafoundation.rs @@ -494,7 +494,7 @@ impl Backend for MediaFoundation { // back everything it holds. Flushing discards that instead, and is what // a failure here leaves the decoder ready for the next keyframe with. unsafe { self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) } - .map_err(|e| mf_err("flush after a failed drain", e))?; + .map_err(|e| Error::CodecGone(format!("flush after a failed drain: {e}")))?; return Err(err); } // A synchronous MFT may accept input immediately after NEED_MORE_INPUT. diff --git a/rs/moq-video/src/decode/backend/openh264.rs b/rs/moq-video/src/decode/backend/openh264.rs index 004b133f2dd..989ddf7e6b5 100644 --- a/rs/moq-video/src/decode/backend/openh264.rs +++ b/rs/moq-video/src/decode/backend/openh264.rs @@ -304,7 +304,7 @@ impl Backend for Openh264 { // Replaced whatever the drain did, so a failed one does not carry its // state into the next stream. `FlushFrame` also leaks a single-threaded // decoder's picture-pool slots, so a drained decoder is never reused. - self.decoder = new_decoder()?; + self.decoder = new_decoder().map_err(|err| Error::CodecGone(format!("openh264 decoder replacement: {err}")))?; tail } diff --git a/rs/moq-video/src/decode/backend/v4l2.rs b/rs/moq-video/src/decode/backend/v4l2.rs index b80b01dcdf4..059d678f562 100644 --- a/rs/moq-video/src/decode/backend/v4l2.rs +++ b/rs/moq-video/src/decode/backend/v4l2.rs @@ -129,9 +129,7 @@ enum Phase { /// The error every call reports once the decoder has failed. fn failed() -> Error { - Error::Codec(anyhow::anyhow!( - "the V4L2 decoder could not be restarted after a drain and takes no more input" - )) + Error::CodecGone("the V4L2 decoder could not be restarted after a drain and takes no more input".into()) } /// The CAPTURE queue plus where a picture sits in one of its buffers. diff --git a/rs/moq-video/src/encode/backend/mediacodec.rs b/rs/moq-video/src/encode/backend/mediacodec.rs index 19c360f079e..936ff161699 100644 --- a/rs/moq-video/src/encode/backend/mediacodec.rs +++ b/rs/moq-video/src/encode/backend/mediacodec.rs @@ -425,6 +425,10 @@ impl Backend for MediaCodec { // A flush drops the reference frames, so the next picture has to be an // IDR. Asked for rather than assumed: not every encoder inserts one. self.keyframe_pending = true; + // Android does not promise a flush restores an encoder whose drain failed. + if tail.is_err() { + self.input = Input::Ended; + } tail }