diff --git a/quest/m1/README.md b/quest/m1/README.md index e657b29d324..7822de6576b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -81,7 +81,6 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [C++ through moq-ffi](/quest/m1/cpp/README.md) - generated C++ over moq-ffi with futures and expected-style errors, shipped as a tarball, vcpkg, and Conan, and adopted by the OBS plugin - [Generated C bindings](/quest/m1/c/README.md) - C generated from moq-ffi ships as `moq-c` 0.8.0 and replaces the hand-written libmoq - [OBS native codecs](/quest/m1/obs-moq-video/README.md) - replace FFmpeg video and audio decoding with moq-video and moq-audio, deliver GPU frames, and use native audio/video encoders -- [Opus concealment](/quest/m1/opus-conceal.md) - a lost Opus packet conceals the last packet's length, not 120 ms - [Audio codecs](/quest/m1/audio-codecs/README.md) - platform audio codecs, explicit unsupported cases, and channel layouts up to 7.1 - [Opus mapping family](/quest/m1/opus-mapping-family.md) - on dev, the Opus head config keeps its mapping family only in `mapping` - [mp4-atom dOps mapping](/quest/m1/mp4-atom-dops-mapping.md) - a released mp4-atom reads and writes any `dOps` channel mapping family and table diff --git a/quest/m1/opus-conceal.md b/quest/m1/opus-conceal.md deleted file mode 100644 index cac6f8255ed..00000000000 --- a/quest/m1/opus-conceal.md +++ /dev/null @@ -1,33 +0,0 @@ -# [XS] Opus concealment length - -## Goal - -`moq_audio` `Decoder::decode(&[])` conceals one Opus packet, the length of the -last real one, instead of 120 ms. The API is unchanged and lands on main. - -## Plan - -- `rs/moq-audio/src/decode/decoder.rs` passes `max_frame_size` (120 ms) as - libopus's `frame_size` on every call, and libopus conceals exactly that many - samples. Record the last packet's sample count (`opus_packet_get_nb_samples`) - and pass it for an empty packet; it is always a multiple of 2.5 ms. -- The audio-codecs line branch moves this code to - `rs/moq-audio/src/decode/backend/libopus.rs` (same `max_frame_size`); - land the fix wherever the code lives when this starts, and port it on the - line's next merge from main otherwise. -- Refuse loss before any packet has decoded, rather than surfacing libopus's - `BUFFER_TOO_SMALL`. -- Document on `decode` how much an empty packet conceals. -- Regression from the issue's repro: one lost 20 ms packet yields 20 ms. -- A `conceal(duration)` method and `decode(Option<&[u8]>)` were rejected: no - consumer needs a caller-chosen length, and the latter breaks the common case. - How platform backends signal loss belongs to the decode-backend seam. - -## Closes - -- [#4248](https://github.com/moq-dev/moq/issues/4248) - `Decoder::decode(&[])` conceals 120 ms of Opus audio for one lost packet - -## Related - -- [Audio loss recovery](/quest/m2/audio-loss-recovery.md) - the FEC versus concealment policy study -- [Decode backend](/quest/m1/audio-codecs/decode-backend.md) - where platform decoders signal loss diff --git a/rs/moq-audio/src/decode/decoder.rs b/rs/moq-audio/src/decode/decoder.rs index 3d813c40c65..0267cd417f9 100644 --- a/rs/moq-audio/src/decode/decoder.rs +++ b/rs/moq-audio/src/decode/decoder.rs @@ -73,6 +73,9 @@ struct Opus { inner: *mut OpusDecoder, pre_skip_remaining: usize, max_frame_size: usize, + /// Frames per channel in the last packet that decoded, which is what a lost + /// packet most likely carried. `None` until one has. + last_frame_size: Option, in_dtx: bool, } @@ -135,6 +138,7 @@ impl Decoder { inner, pre_skip_remaining: head.pre_skip as usize, max_frame_size: (opus::DECODE_RATE as usize * MAX_FRAME_MS) / 1000, + last_frame_size: None, in_dtx: false, }; @@ -281,13 +285,26 @@ impl Decoder { /// Decode one packet into interleaved `f32` PCM and report its codec activity. /// - /// Empty Opus packets invoke packet-loss concealment. Loss during DTX remains - /// classified as DTX, while loss during active audio remains active. + /// An empty Opus packet marks one lost packet and conceals as much audio as the + /// last packet that decoded held, so a lost 20 ms packet yields 20 ms. It is + /// refused with [`Error::Decode`] before any packet has decoded, since there is + /// no length to conceal. Loss during DTX remains classified as DTX, while loss + /// during active audio remains active. pub fn decode(&mut self, packet: &[u8]) -> Result { match &mut self.backend { Backend::Opus(opus) => { let channels = self.layout.channels() as usize; - let mut out = vec![0.0f32; opus.max_frame_size * channels]; + // libopus conceals exactly `frame_size` samples for a lost packet, and + // the decoder can't know how long it was, so assume the last one's length. + // A real packet decodes into the most it can hold. + let frame_size = if packet.is_empty() { + opus.last_frame_size.ok_or_else(|| { + Error::Decode("opus packet lost before any packet decoded, so no length to conceal".into()) + })? + } else { + opus.max_frame_size + }; + let mut out = vec![0.0f32; frame_size * channels]; // SAFETY: `inner` owns a live OpusDecoder; packet/out slices are // bounded by the lengths we pass. let samples = unsafe { @@ -296,13 +313,16 @@ impl Decoder { packet.as_ptr(), packet.len() as i32, out.as_mut_ptr(), - opus.max_frame_size as i32, + frame_size as i32, 0, ) }; if samples < 0 { return Err(crate::opus::decode_error(samples)); } + if !packet.is_empty() { + opus.last_frame_size = Some(samples as usize); + } out.truncate(samples as usize * channels); let trim_frames = opus.pre_skip_remaining.min(samples as usize); if trim_frames > 0 { @@ -504,6 +524,29 @@ pub(crate) mod tests { } } + /// One lost packet conceals one packet's worth of audio, not the 120 ms + /// libopus would fill given the whole buffer. + #[test] + fn opus_conceals_the_length_of_the_last_packet() { + let packets = opus_packets(3); + let mut decoder = Decoder::new( + &opus_catalog(moq_mux::codec::opus::Config::new(48_000, 1)), + &Config::default(), + ) + .unwrap(); + + // Nothing decoded yet, so there is no length to conceal. + assert!(matches!(decoder.decode(&[]), Err(Error::Decode(_)))); + + for packet in &packets { + decoder.decode(packet).unwrap(); + } + assert_eq!(decoder.decode(&[]).unwrap().samples.len(), 960); + + // Concealment doesn't change the length: the next loss is the same size. + assert_eq!(decoder.decode(&[]).unwrap().samples.len(), 960); + } + #[test] fn opus_applies_the_declared_gain() { let packets = opus_packets(5);