Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 0 additions & 34 deletions quest/m1/precut-fixes/video-decode-lifecycle.md

This file was deleted.

8 changes: 7 additions & 1 deletion rs/moq-video/src/decode/backend/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,12 @@ const NAMED_ONLY: &[Candidate] = &[
open: probe::Buffered::open,
drives: Drives::Own,
},
Candidate {
name: probe::FAILING_FLUSH_NAME,
supports: |c| matches!(c, Codec::H264),
open: probe::Buffered::open_failing_flush,
drives: Drives::Own,
},
Candidate {
name: probe::REORDER_NAME,
supports: |c| matches!(c, Codec::H264),
Expand All @@ -305,7 +311,7 @@ const NAMED_ONLY: &[Candidate] = &[
},
Candidate {
name: probe::HOLDING_NAME,
supports: |c| matches!(c, Codec::H264),
supports: |c| matches!(c, Codec::H264 | Codec::H265),
open: probe::Reorder::open_holding,
drives: Drives::Own,
},
Expand Down
19 changes: 18 additions & 1 deletion rs/moq-video/src/decode/backend/probe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ pub(crate) const NAME: &str = "probe";
/// A test decoder that holds one picture until the next call or a flush.
pub(crate) const BUFFERED_NAME: &str = "probe-buffered";

/// A test decoder that holds one picture like [`BUFFERED_NAME`], whose flush
/// readies it and then fails, as a codec that errors partway through a drain.
pub(crate) const FAILING_FLUSH_NAME: &str = "probe-failing-flush";

/// A test decoder that reorders like B-frames do: it holds two pictures and
/// returns the earliest once it has a third. It hands pictures back at
/// nanosecond scale, as MediaCodec does, whatever scale the access unit came in.
Expand Down Expand Up @@ -148,6 +152,7 @@ pub(crate) const SIZE: Size = Size {
pub(crate) struct Probe(VecDeque<Frame>);

pub(crate) struct Buffered {
name: &'static str,
held: Option<Frame>,
ready: VecDeque<Frame>,
}
Expand Down Expand Up @@ -180,6 +185,15 @@ impl Probe {
impl Buffered {
pub(crate) fn open(_stream: &Stream, _config: &Config) -> Result<Box<dyn Backend>, Error> {
Ok(Box::new(Self {
name: BUFFERED_NAME,
held: None,
ready: VecDeque::new(),
}))
}

pub(crate) fn open_failing_flush(_stream: &Stream, _config: &Config) -> Result<Box<dyn Backend>, Error> {
Ok(Box::new(Self {
name: FAILING_FLUSH_NAME,
held: None,
ready: VecDeque::new(),
}))
Expand Down Expand Up @@ -381,11 +395,14 @@ impl Backend for Buffered {

fn flush(&mut self) -> Result<(), Error> {
self.ready.extend(self.held.take());
if self.name == FAILING_FLUSH_NAME {
return Err(anyhow::anyhow!("probe: flush failed").into());
}
Ok(())
}

fn name(&self) -> &str {
BUFFERED_NAME
self.name
}
}

Expand Down
168 changes: 152 additions & 16 deletions rs/moq-video/src/decode/consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -350,16 +350,20 @@ impl Reader {
// Drained so the pictures from before the boundary all come out ahead of it,
// rather than whenever the media after it pushes them through. The decoder
// stays usable and restarts on the next group's keyframe.
// The marker is consumed, so a failed drain still queues it and arms recovery
// before reporting, or the next read would never see it.
Some(Encoded::Gap { timestamp, duration }) => {
self.drain().await?;
let drained = self.drain().await;
self.pending.push_back((Read::Gap { timestamp, duration }, None));
self.recover = true;
drained?;
continue;
}
// A fresh encoder has no preceding GOP to reference, so no recovery.
Some(Encoded::End { end, .. }) => {
self.drain().await?;
let drained = self.drain().await;
self.pending.push_back((Read::End { end }, None));
drained?;
continue;
}
None => None,
Expand All @@ -370,21 +374,15 @@ impl Reader {
// dropped before the drain ran retries it rather than reporting
// an end the stream has not reached. Flushing twice is safe: the
// second hands back nothing.
let tail = self.decoder.flush().await;
let drained = self.drain().await;
// Set before the error is returned, not after. A flush that
// failed once fails the same way every time, and the track has
// ended either way, so leaving the flag down turns one bad
// drain into a caller that reads, fails, and reads again with
// nothing in between to wait on. A caller that treats a codec
// error as one lost picture and carries on then spins.
self.drained = true;
// Queue what the flush readied before reporting its error, so a
// later read still hands those pictures over.
let received = self.receive();
// Whatever the flush did not return never comes back.
self.arrivals.clear();
tail?;
received?;
drained?;
continue;
};

Expand Down Expand Up @@ -452,15 +450,14 @@ impl Reader {
Ok(())
}

/// Queue every picture the decoder still holds.
/// Queue every picture the decoder still holds. A flush that fails still
/// queues what it readied, so a later read hands those pictures over.
async fn drain(&mut self) -> Result<(), Error> {
let drained = match self.decoder.flush().await {
Ok(()) => self.receive(),
Err(err) => Err(err),
};
let flushed = self.decoder.flush().await;
let received = self.receive();
// Whatever the flush did not return never comes back.
self.arrivals.clear();
drained
flushed.and(received)
}
}

Expand Down Expand Up @@ -1144,6 +1141,145 @@ mod tests {
assert_eq!(reads, ["frame 100000", "end 133332", "frame 200000", "none"]);
}

/// Every read until the track is done, as text: a codec error is one read too.
async fn read_all(consumer: &mut Consumer) -> Vec<String> {
let mut reads = Vec::new();
loop {
reads.push(match consumer.read().await {
Ok(Some(Read::Frame(frame))) => format!("frame {}", frame.timestamp.as_micros().unwrap()),
Ok(Some(Read::Gap { timestamp, .. })) => format!("gap {}", timestamp.as_micros().unwrap()),
Ok(Some(Read::End { end })) => format!("end {}", end.as_micros().unwrap()),
Ok(None) => return reads,
Err(_) => "error".to_string(),
});
}
}

/// A drain that fails at an end still delivers the pictures it readied ahead
/// of the end, and the end itself, after reporting the error.
#[tokio::test]
async fn a_failed_drain_keeps_its_end() {
let broadcast = moq_net::broadcast::Info::new().produce();
let track = broadcast
.create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video))
.unwrap();
let subscriber = broadcast.consume();
let mut producer = moq_mux::container::Producer::new(
track,
moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video),
);
let picture = |micros: u64| moq_mux::container::Frame {
timestamp: Timestamp::from_micros(micros).unwrap(),
duration: Some(Timestamp::from_micros(33_333).unwrap()),
payload: Bytes::from_static(b"access unit"),
keyframe: true,
};
producer.write(picture(100_000)).unwrap();
producer.cut(None).unwrap();
producer.discontinuity().unwrap();
producer.write(picture(200_000)).unwrap();
let _finished = producer.finish().unwrap();

let catalog = VideoConfig::new(hang::catalog::H264 {
inline: true,
profile: 0x42,
constraints: 0,
level: 30,
});
let mut consumer = Consumer::new(
&subscriber,
&catalog,
"video",
Options {
decoder: Config {
kind: Kind::Named(probe::FAILING_FLUSH_NAME.into()),
..Config::new()
},
delay: Delay {
max: Some(std::time::Duration::from_secs(10)),
..Delay::default()
},
..Options::new()
},
)
.await
.unwrap();

assert_eq!(
read_all(&mut consumer).await,
["error", "frame 100000", "end 133332", "error", "frame 200000"]
);
}

/// A drain that fails at a gap still delivers the gap after the pictures it
/// readied, and still trims the leading pictures of the recovery group.
#[tokio::test(start_paused = true)]
async fn a_failed_drain_keeps_its_gap_and_recovery() {
let broadcast = moq_net::broadcast::Info::new().produce();
let track = broadcast
.create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video))
.unwrap();
let catalog = VideoConfig::new(hang::catalog::H264 {
inline: true,
profile: 0x42,
constraints: 0,
level: 30,
});
let mut consumer = Consumer::new(
&broadcast.consume(),
&catalog,
"video",
Options {
decoder: Config {
kind: Kind::Named(probe::FAILING_FLUSH_NAME.into()),
..Config::new()
},
delay: Delay {
max: Some(std::time::Duration::from_secs(10)),
..Delay::default()
},
..Options::new()
},
)
.await
.unwrap();
let wire = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video);
let write = |sequence, stamps: &[u64]| {
let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap();
let frames: Vec<_> = stamps
.iter()
.enumerate()
.map(|(index, stamp)| moq_mux::container::Frame {
timestamp: Timestamp::from_micros(*stamp).unwrap(),
duration: None,
payload: Bytes::from_static(b"access unit"),
keyframe: index == 0,
})
.collect();
wire.write(&mut group, &frames).unwrap();
group.finish().unwrap();
};
write(10, &[1000, 1100]);
let first = consumer.next_frame().await.unwrap().unwrap();
assert_eq!(first.timestamp.as_micros().unwrap(), 1000);

consumer.reader().track.set_delay(Delay::default());
write(11, &[2000]);
write(12, &[3000, 2800, 2900, 3100]);
let _track = track.finish().unwrap();
let reads = read_all(&mut consumer).await;
assert_eq!(
reads[..3],
["error", "frame 1100", "gap 1100"],
"the drained picture, then its gap: {reads:?}"
);
assert_eq!(
reads[3..],
["frame 3000", "error", "frame 3100"],
"recovery trims the leading pictures: {reads:?}"
);
}

/// A read dropped while a threaded flush is in flight loses nothing: the
/// next read finishes that same flush rather than issuing another into a
/// poisoned sink, and only then reports the end.
Expand Down
Loading