diff --git a/rs/moq-video/src/decode/backend/nvdec.rs b/rs/moq-video/src/decode/backend/nvdec.rs index 71a4ad6727e..f7d0753ace4 100644 --- a/rs/moq-video/src/decode/backend/nvdec.rs +++ b/rs/moq-video/src/decode/backend/nvdec.rs @@ -17,7 +17,8 @@ //! The cuvid parser is driven synchronously: callbacks (sequence / decode / //! display) fire inside `cuvidParseVideoData` on the calling thread. Each access //! unit uses `CUVID_PKT_ENDOFPICTURE`, and an explicit end-of-stream packet drains -//! any pictures the display queue still holds. Timestamps are threaded through +//! any pictures the display queue still holds. A parser decodes nothing after +//! end-of-stream, so each drain replaces it. Timestamps are threaded through //! the parser (`ulClockRate` is set to microseconds), so output frames keep //! correct presentation times even across reordering. @@ -48,12 +49,12 @@ fn codec_err(msg: String) -> Error { } pub(crate) struct Nvdec { - parser: CUvideoparser, + /// `None` once a drain could not replace it: the decoder has failed for good. + parser: Option, + codec: cudaVideoCodec, /// Boxed so its address is stable: the parser holds a raw pointer to it for /// the lifetime of the parser (callbacks dereference it during parse). state: Box, - /// Mark the first access unit after a drain as a new parser epoch. - discontinuity: bool, } /// State shared with the parser's C callbacks via the user-data pointer. @@ -86,6 +87,9 @@ struct Decoder { /// Output size: cuvid scales to this while writing the output surface. width: u32, height: u32, + /// Decode surfaces it was created with; the parser may index any of the + /// count it reports, so a larger requirement needs a new decoder. + surfaces: c_int, } impl Drop for Decoder { @@ -142,35 +146,12 @@ impl Nvdec { error: None, }); - let mut params = CUVIDPARSERPARAMS { - CodecType: cuda_codec, - // Placeholder until the sequence callback reports the real minimum. - ulMaxNumDecodeSurfaces: 1, - // Packet timestamps are in microseconds (default is a 10 MHz clock). - ulClockRate: 1_000_000, - // Emit each picture as soon as it decodes (live path, no lookahead). - ulMaxDisplayDelay: 0, - pUserData: &mut *state as *mut State as *mut c_void, - pfnSequenceCallback: Some(sequence_callback), - pfnDecodePicture: Some(decode_callback), - pfnDisplayPicture: Some(display_callback), - ..Default::default() - }; - - let mut parser: CUvideoparser = ptr::null_mut(); - // SAFETY: params points at a fully-initialized struct and outlives the - // call; the user-data pointer stays valid because `state` is boxed and - // owned by the returned backend alongside the parser. - let result = unsafe { (api.create_video_parser)(&mut parser, &mut params) }; - if result != cudarc::driver::sys::CUresult::CUDA_SUCCESS { - return Err(codec_err(format!("cuvidCreateVideoParser: {result:?}"))); - } - + let parser = create_parser(cuda_codec, &mut state)?; tracing::info!(decoder = NAME, codec = ?codec, resize = ?config.scale_hint, "opened video decoder"); Ok(Box::new(Self { - parser, + parser: Some(parser), + codec: cuda_codec, state, - discontinuity: false, })) } @@ -184,9 +165,12 @@ impl Nvdec { .bind_to_thread() .map_err(|e| codec_err(format!("CUDA bind: {e:?}")))?; + let parser = self + .parser + .ok_or_else(|| Error::CodecGone("NVDEC parser replacement failed after a drain".into()))?; // SAFETY: parser and packet are valid. A non-null payload is owned by the // caller for the duration of this synchronous call. - let result = unsafe { (self.state.api.parse_video_data)(self.parser, packet) }; + let result = unsafe { (self.state.api.parse_video_data)(parser, packet) }; if let Some(error) = self.state.error.take() { self.state.ready.clear(); return Err(error); @@ -203,27 +187,25 @@ impl Nvdec { impl Backend for Nvdec { fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - let mut flags = (CUvideopacketflags::CUVID_PKT_TIMESTAMP as c_ulong) - | (CUvideopacketflags::CUVID_PKT_ENDOFPICTURE as c_ulong); - if self.discontinuity { - flags |= CUvideopacketflags::CUVID_PKT_DISCONTINUITY as c_ulong; - } let mut packet = CUVIDSOURCEDATAPACKET { // ENDOFPICTURE: each payload is one complete access unit, so the // parser sees the picture boundary without waiting for the next AU. - flags, + flags: (CUvideopacketflags::CUVID_PKT_TIMESTAMP as c_ulong) + | (CUvideopacketflags::CUVID_PKT_ENDOFPICTURE as c_ulong), payload_size: access_unit.len() as c_ulong, payload: access_unit.as_ptr(), // cuvid's clock rate is microseconds (set at parser creation). timestamp: timestamp.as_micros() as i64, }; - let frames = self.parse(&mut packet)?; - self.discontinuity = false; - Ok(frames) + self.parse(&mut packet) } fn flush(&mut self) -> Result, Error> { + if self.parser.is_none() { + // A failed replacement ended the session; never install another parser. + return Err(Error::CodecGone("NVDEC parser replacement failed after a drain".into())); + } let mut packet = CUVIDSOURCEDATAPACKET { // NVIDIA requires an explicit EOS packet to release every picture still // waiting in display order, even with zero display delay. @@ -232,10 +214,16 @@ impl Backend for Nvdec { payload: ptr::null(), timestamp: 0, }; - // Set ahead of the parse: whatever it did, the next access unit opens a - // new stream. - self.discontinuity = true; - self.parse(&mut packet) + let frames = self.parse(&mut packet); + // Past end-of-stream the parser accepts data but never decodes it again + // (CUVID_PKT_DISCONTINUITY does not reopen it), so whatever the drain did, + // the next access unit goes to a new parser. The cuvid decoder is kept: the + // new parser's sequence callback reuses it while the format holds. + self.destroy_parser(); + let parser = create_parser(self.codec, &mut self.state) + .map_err(|err| Error::CodecGone(format!("NVDEC parser replacement: {err}")))?; + self.parser = Some(parser); + frames } fn name(&self) -> &str { @@ -243,14 +231,50 @@ impl Backend for Nvdec { } } -impl Drop for Nvdec { - fn drop(&mut self) { +impl Nvdec { + fn destroy_parser(&mut self) { + let Some(parser) = self.parser.take() else { return }; // Drop may run on a different thread than decode; destroying the parser // needs the context current (the decoder binds its own). let _ = self.state.ctx.bind_to_thread(); // SAFETY: the parser is valid and no parse call is in flight (&mut self). - unsafe { (self.state.api.destroy_video_parser)(self.parser) }; + unsafe { (self.state.api.destroy_video_parser)(parser) }; + } +} + +impl Drop for Nvdec { + fn drop(&mut self) { + self.destroy_parser(); + } +} + +/// A parser for `codec` whose callbacks feed `state`. +fn create_parser(codec: cudaVideoCodec, state: &mut State) -> Result { + let api = state.api; + let mut params = CUVIDPARSERPARAMS { + CodecType: codec, + // Placeholder until the sequence callback reports the real minimum. + ulMaxNumDecodeSurfaces: 1, + // Packet timestamps are in microseconds (default is a 10 MHz clock). + ulClockRate: 1_000_000, + // Emit each picture as soon as it decodes (live path, no lookahead). + ulMaxDisplayDelay: 0, + pUserData: ptr::from_mut(state).cast(), + pfnSequenceCallback: Some(sequence_callback), + pfnDecodePicture: Some(decode_callback), + pfnDisplayPicture: Some(display_callback), + ..Default::default() + }; + + let mut parser: CUvideoparser = ptr::null_mut(); + // SAFETY: params points at a fully-initialized struct and outlives the call; + // the user-data pointer stays valid because `state` is boxed and owned by the + // backend alongside the parser. + let result = unsafe { (api.create_video_parser)(&mut parser, &mut params) }; + if result != cudarc::driver::sys::CUresult::CUDA_SUCCESS { + return Err(codec_err(format!("cuvidCreateVideoParser: {result:?}"))); } + Ok(parser) } impl State { @@ -300,8 +324,10 @@ impl State { && decoder.coded == coded && decoder.display_area == display_area && (decoder.width, decoder.height) == (width, height) + && decoder.surfaces >= surfaces { - return Ok(surfaces); + // The parser indexes up to this count, so report what was allocated. + return Ok(decoder.surfaces); } self.decoder = None; @@ -357,6 +383,7 @@ impl State { display_area, width, height, + surfaces, }); Ok(surfaces) } @@ -648,6 +675,48 @@ mod tests { out } + /// The transcode feed drains at every group end, so a drain must leave the + /// decoder decoding the next keyframe and everything behind it. + #[test] + fn nvdec_decodes_after_a_flush() { + if !hw_available() { + return; + } + let (w, h) = (320u32, 240u32); + let mut encoder = Encoder::new(&EncodeConfig { + kind: EncodeKind::Software, + ..EncodeConfig::new(w, h, crate::FrameRate::new(30, 1).unwrap()) + }) + .unwrap(); + let mut decoder = Nvdec::open(&Codec::H264.into(), &decode_config(None)).expect("NVDEC H.264 decoder"); + let rgba = gradient_rgba(w, h); + + let mut decoded = Vec::new(); + for group in 0..3u64 { + let mut timestamps = Vec::new(); + encoder.cut().unwrap(); + for i in 0..10u64 { + for encoded in encoder.encode(&gradient_frame(&rgba, w, h, group * 10 + i)).unwrap() { + let frames = decoder.decode(encoded.payload, encoded.timestamp, i == 0).unwrap(); + timestamps.extend(frames.iter().map(|frame| frame.timestamp.as_micros() as u64)); + } + } + let frames = decoder.flush().unwrap(); + timestamps.extend(frames.iter().map(|frame| frame.timestamp.as_micros() as u64)); + // A second flush lands on a parser that has seen no packet yet. + assert!( + decoder.flush().unwrap().is_empty(), + "a repeated flush hands back nothing" + ); + decoded.push(timestamps); + } + + for (group, timestamps) in decoded.iter().enumerate() { + let expected: Vec = (0..10).map(|i| (group as u64 * 10 + i) * 33_333).collect(); + assert_eq!(timestamps, &expected, "group {group} after {group} flushes"); + } + } + /// H.264 through the real hardware: openh264 encodes a gradient (Annex-B, /// inline SPS/PPS) and NVDEC decodes it. Asserts the downloaded NV12 -> I420 /// picture matches the input (a layout bug shears a plane and blows the MAE)