From 509516dbeca1ea2e3eaa735fc9ec559019e21d16 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 08:42:21 -0800 Subject: [PATCH 01/14] chore(quest): claim quest/m2/video-producer.md Quest-Claim: 051eecdefa28bf09 From aaad5db6be50981052ce40b607b74352ffb8f605 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 08:50:56 -0800 Subject: [PATCH 02/14] refactor(moq-video)!: the encoded-frame publisher is encode::Publisher The frame-taking producer takes encode::Producer next. Mechanical rename of the type and its file; behaviour unchanged. Co-Authored-By: Claude Opus 5.5 --- rs/moq-boy/src/video.rs | 4 +- rs/moq-c/src/video.rs | 16 +++---- rs/moq-ffi/src/video.rs | 12 ++--- rs/moq-video/README.md | 2 +- rs/moq-video/src/capture/wgc.rs | 4 +- rs/moq-video/src/decode/consumer.rs | 4 +- rs/moq-video/src/encode/encoder.rs | 4 +- rs/moq-video/src/encode/mod.rs | 12 ++--- .../src/encode/{producer.rs => publisher.rs} | 44 +++++++++---------- rs/moq-video/src/lib.rs | 2 +- 10 files changed, 52 insertions(+), 52 deletions(-) rename rs/moq-video/src/encode/{producer.rs => publisher.rs} (98%) diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index 9b164a7ec7e..a6020632321 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -62,7 +62,7 @@ impl VideoEncoder { .unique_track(".avc3", reserved.track_info(hang::catalog::PRIORITY.video)) .expect("failed to create the avc3 track"); let producer = - moq_video::encode::Producer::new(track, reserved, rendition).expect("failed to create avc3 producer"); + moq_video::encode::Publisher::new(track, reserved, rendition).expect("failed to create avc3 producer"); let demand = producer.demand(); let force_keyframe = Arc::new(AtomicBool::new(false)); @@ -117,7 +117,7 @@ impl VideoEncoder { fn encoder_thread( mut rx: tokio::sync::mpsc::Receiver, - mut producer: moq_video::encode::Producer, + mut producer: moq_video::encode::Publisher, config: moq_video::encode::Config, force_keyframe: Arc, encode_duration: Arc, diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index 3634e5ddbfe..96a0444cde0 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -403,7 +403,7 @@ fn retune( pub(crate) struct VideoEncoder { /// `None` while the rendition is disabled for lack of bandwidth. encoder: Option, - producer: moq_video::encode::Producer, + producer: moq_video::encode::Publisher, /// The encoder settings, to reopen at [`ceiling`](Self::ceiling) when a disabled /// rendition resumes. config: moq_video::encode::Config, @@ -539,7 +539,7 @@ fn surface(_frame: &moq_video::Frame) -> *const c_void { /// anyway would leave a truncated stream indistinguishable from a complete one, /// and only the local caller would ever learn otherwise. fn finalize( - mut producer: moq_video::encode::Producer, + mut producer: moq_video::encode::Publisher, drained: Result<(), moq_video::Error>, children: &crate::publish::Children, child: crate::publish::Child, @@ -769,7 +769,7 @@ impl VideoEncoder { /// Publish every access unit `encoder` has ready. fn publish( encoder: &mut moq_video::encode::Sink, - producer: &mut moq_video::encode::Producer, + producer: &mut moq_video::encode::Publisher, ) -> Result<(), moq_video::Error> { while let Some(encoded) = encoder.receive()? { producer.publish(&encoded)?; @@ -780,7 +780,7 @@ fn publish( /// Flush `encoder` and publish its tail, closing it. fn drain( mut encoder: moq_video::encode::Sink, - producer: &mut moq_video::encode::Producer, + producer: &mut moq_video::encode::Publisher, ) -> Result<(), moq_video::Error> { block_on(encoder.flush())?; publish(&mut encoder, producer) @@ -834,7 +834,7 @@ impl Video { } }; let track = target.producer.unique_track(suffix, reserved.track_info(priority))?; - let producer = moq_video::encode::Producer::new(track, reserved, rendition)?; + let producer = moq_video::encode::Publisher::new(track, reserved, rendition)?; let encoder = Shared::new(VideoEncoder { encoder: Some(encoder), producer, @@ -1460,7 +1460,7 @@ mod tests { /// A video track wired up without an encoder, plus a subscriber on it: enough /// to pin what [`finalize`] shows the far end. - async fn track_under_test() -> (moq_video::encode::Producer, moq_net::track::Subscriber) { + async fn track_under_test() -> (moq_video::encode::Publisher, moq_net::track::Subscriber) { let mut broadcast = moq_net::broadcast::Info::new().produce(); let config = moq_mux::catalog::Config::default() .with_catalog(moq_mux::catalog::hang::Catalog::::default()); @@ -1476,7 +1476,7 @@ mod tests { let track = broadcast .unique_track(".avc3", reserved.track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let producer = moq_video::encode::Producer::new(track, reserved, rendition).unwrap(); + let producer = moq_video::encode::Publisher::new(track, reserved, rendition).unwrap(); let name = producer.demand().name().to_string(); let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); @@ -1571,7 +1571,7 @@ mod tests { let track = broadcast .create_track("camera", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let producer = moq_video::encode::Producer::new(track.clone(), catalog.reserve(), rendition).unwrap(); + let producer = moq_video::encode::Publisher::new(track.clone(), catalog.reserve(), rendition).unwrap(); let shared = Shared::new(VideoEncoder { encoder: Some(moq_video::encode::Sink::open(&encode).await.unwrap()), producer, diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 51026fe02bd..cf92787a6a5 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -285,7 +285,7 @@ impl MoqVideoEncoder { /// Publish every access unit `encoder` has ready. fn publish( encoder: &mut moq_video::encode::Sink, - producer: &mut moq_video::encode::Producer, + producer: &mut moq_video::encode::Publisher, ) -> Result<(), moq_video::Error> { while let Some(encoded) = encoder.receive()? { producer.publish(&encoded)?; @@ -296,7 +296,7 @@ fn publish( /// Flush `encoder` and publish its tail, closing it. fn drain( mut encoder: moq_video::encode::Sink, - producer: &mut moq_video::encode::Producer, + producer: &mut moq_video::encode::Publisher, ) -> Result<(), moq_video::Error> { block_on(encoder.flush())?; publish(&mut encoder, producer) @@ -309,7 +309,7 @@ fn drain( /// anyway would leave a truncated stream indistinguishable from a complete one, /// and only the local caller would ever learn otherwise. fn finalize( - mut producer: moq_video::encode::Producer, + mut producer: moq_video::encode::Publisher, drained: Result<(), moq_video::Error>, children: &crate::producer::Children, ) -> Result<(), MoqError> { @@ -350,7 +350,7 @@ fn block_on(future: impl std::future::Future) -> T { struct VideoProducer { /// `None` while the rendition is disabled for lack of bandwidth. encoder: Option, - producer: moq_video::encode::Producer, + producer: moq_video::encode::Publisher, /// The encoder settings, to reopen at [`ceiling`](Self::ceiling) when a disabled /// rendition resumes. config: moq_video::encode::Config, @@ -870,7 +870,7 @@ impl MoqVideoProducer { Some(name) => state.broadcast.create_track(name, info)?, None => state.broadcast.unique_track(suffix, info)?, }; - Ok(moq_video::encode::Producer::new( + Ok(moq_video::encode::Publisher::new( track, state.catalog.reserve().with_interval(join), rendition, @@ -1695,7 +1695,7 @@ mod finish_tests { let track = broadcast .create_track("camera", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let producer = moq_video::encode::Producer::new(track.clone(), catalog.reserve(), rendition).unwrap(); + let producer = moq_video::encode::Publisher::new(track.clone(), catalog.reserve(), rendition).unwrap(); let encoder = moq_video::encode::Sink::open(&encode).await.unwrap(); let mut video = VideoProducer { encoder: Some(encoder), diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index 74680066d75..00216fef50f 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -137,7 +137,7 @@ Public entry points: the track and catalog are advertised up front, and the camera stays open whether or not anyone is watching. With `Capture::on_demand`, the camera opens only while a subscriber is watching and is released when the last one leaves. -- `encode::Producer` publishes frames you encoded yourself (`publish(&[Encoded])`), +- `encode::Publisher` publishes frames you encoded yourself (`publish(&Encoded)`), handling the catalog and framing. Each is published at its own timestamp. The default features are `nvidia`, `v4l2`, `vaapi`, and `mediacodec`, so a default diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index 9aa0a303c88..eeccf1665f1 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -300,7 +300,7 @@ mod tests { #[tokio::test] #[cfg(feature = "openh264")] async fn unchanged_frames_reach_an_active_subscriber() { - use crate::encode::{Config, Encoder, Kind, Producer}; + use crate::encode::{Config, Encoder, Kind, Publisher}; let now = Instant::now(); let interval = Duration::from_millis(20); @@ -325,7 +325,7 @@ mod tests { let container = moq_mux::catalog::hang::Container::try_from(&rendition).unwrap(); let mut subscriber = moq_mux::container::Consumer::new(subscriber, container); let reserved = catalog.reserve().with_interval(config.effective_join()); - let mut producer = Producer::new(track, reserved, rendition).unwrap(); + let mut producer = Publisher::new(track, reserved, rendition).unwrap(); assert!(producer.demand().wanted().is_some()); // Only one captured picture arrives; more than three intervals of paced diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index f5dcd8f44ce..2bd17b26ea7 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -540,7 +540,7 @@ mod tests { use super::*; use crate::decode::Kind; use crate::decode::backend::probe; - use crate::encode::{Config as EncodeConfig, Encoder, Kind as EncodeKind, Producer as EncodeProducer}; + use crate::encode::{Config as EncodeConfig, Encoder, Kind as EncodeKind, Publisher as EncodePublisher}; use moq_mux::container::Container as _; #[tokio::test] @@ -559,7 +559,7 @@ mod tests { let track = source_broadcast .unique_track(".avc3", reserved.track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let mut producer = EncodeProducer::new(track, reserved, rendition).unwrap(); + let mut producer = EncodePublisher::new(track, reserved, rendition).unwrap(); let mut encoder = Encoder::new(&config).unwrap(); let rgba = vec![0x80u8; 320 * 240 * 4]; for index in 0..2 { diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index 0728ffbfe26..673ecd0ff73 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -366,7 +366,7 @@ pub(crate) fn default_bitrate(size: Size, framerate: FrameRate) -> moq_net::band /// Video encoder. Build one with [`Encoder::new`], submit raw [`Frame`]s via /// [`encode`](Self::encode), [`receive`](Self::receive) the resulting /// [`Encoded`] access units until it returns `None`, and publish them through a -/// [`Producer`](super::Producer) built for the same [`Codec`]. +/// [`Publisher`](super::Publisher) built for the same [`Codec`]. /// /// An encoder is bound to the thread that opens it. Use [`Sink`](super::Sink) /// when the owner can move between threads. @@ -478,7 +478,7 @@ impl Encoder { Ok(()) } - /// The codec this encoder emits. A [`Producer`](super::Producer) must be + /// The codec this encoder emits. A [`Publisher`](super::Publisher) must be /// built for the same codec to publish its packets. pub fn codec(&self) -> Codec { self.codec diff --git a/rs/moq-video/src/encode/mod.rs b/rs/moq-video/src/encode/mod.rs index a7d9c1eb8a8..22aad18f88a 100644 --- a/rs/moq-video/src/encode/mod.rs +++ b/rs/moq-video/src/encode/mod.rs @@ -8,14 +8,14 @@ //! `Publication` that reports its state and the `Driver` that runs it. //! Requires the `capture` feature. //! - [`Encoder`] encodes raw [`Frame`](crate::Frame)s you supply into -//! [`Encoded`] access units, and [`Producer`] publishes those (bring your own +//! [`Encoded`] access units, and [`Publisher`] publishes those (bring your own //! frames). Build both for the same [`Codec`]. //! - [`Sink`] is an [`Encoder`] that owns the thread it runs on, for an encoder //! outliving a single thread's stack (a shared object, an FFI handle, a task //! that migrates between workers). -//! - [`Producer`] alone publishes frames you already encoded. +//! - [`Publisher`] alone publishes frames you already encoded. //! -//! A [`Producer`] advertises its catalog rendition as soon as the track exists, +//! A [`Publisher`] advertises its catalog rendition as soon as the track exists, //! resolved from the encoder itself via [`Config::probe`], so a subscriber can //! discover a track nothing has encoded yet. That's what lets an on-demand //! capture wait for its first viewer at all. @@ -40,15 +40,15 @@ pub(crate) mod backend; mod cuts; mod encoded; mod encoder; -mod producer; +mod publisher; mod sink; pub use backend::NAMES; pub use encoded::Encoded; pub use encoder::{Applied, Codec, Config, Encoder, Kind, Preset}; -pub use producer::Producer; +pub use publisher::Publisher; #[cfg(feature = "capture")] -pub use producer::{Capture, Driver, Publication, State, Status, publish_capture}; +pub use publisher::{Capture, Driver, Publication, State, Status, publish_capture}; pub use sink::Sink; #[cfg(test)] diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/publisher.rs similarity index 98% rename from rs/moq-video/src/encode/producer.rs rename to rs/moq-video/src/encode/publisher.rs index f9de903fd9a..913744b28fb 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/publisher.rs @@ -89,11 +89,11 @@ enum Codecs { /// registered) before the camera opens; this is what lets a subscriber /// trigger capture on demand. The `moq_mux::codec` importer for the codec /// handles catalog registration and framing. -pub struct Producer { +pub struct Publisher { codecs: Codecs, } -impl Producer { +impl Publisher { /// Publish `rendition` on `track`, registering it under `reserved`, a /// [`catalog::Producer::reserve`](moq_mux::catalog::Producer::reserve) /// that carries any per-track hint such as the encoder's GOP duration @@ -512,7 +512,7 @@ enum Owned { /// Registered, until the probe knows its rendition. Track(moq_net::track::Producer), /// Probed, and wrapped in the importer that publishes it. - Producer(Box), + Publisher(Box), /// Ended or cancelled, so nothing is left to end. Done, } @@ -524,7 +524,7 @@ impl Owned { let ended = match self { // The caller is done with the track, so the finished handle goes with it. Self::Track(track) => track.finish().map(drop).map_err(Error::from), - Self::Producer(mut producer) => producer.finish(), + Self::Publisher(mut producer) => producer.finish(), Self::Done => Ok(()), }; if let Err(err) = ended { @@ -538,7 +538,7 @@ impl Owned { Self::Track(track) => { let _ = track.abort(err); } - Self::Producer(producer) => producer.abort(err), + Self::Publisher(producer) => producer.abort(err), Self::Done => {} } } @@ -629,9 +629,9 @@ impl Driver { let Owned::Track(track) = &self.owned else { unreachable!("a driver runs once") }; - let producer = Producer::new(track.clone(), self.catalog.reserve().with_interval(join), rendition)?; - self.owned = Owned::Producer(Box::new(producer)); - let Owned::Producer(producer) = &mut self.owned else { + let producer = Publisher::new(track.clone(), self.catalog.reserve().with_interval(join), rendition)?; + self.owned = Owned::Publisher(Box::new(producer)); + let Owned::Publisher(producer) = &mut self.owned else { unreachable!("just set") }; // Capturing from the start, the loop keeps the probe's camera rather than reopening it, @@ -786,7 +786,7 @@ fn log_track_ended(err: moq_net::Error) { } #[cfg(any(feature = "capture", all(test, feature = "openh264")))] -fn capture_stopped(producer: &mut Producer) -> Result<(), Error> { +fn capture_stopped(producer: &mut Publisher) -> Result<(), Error> { // The shared clock keeps advancing while capture is stopped. Mark the break before waiting // for demand again so the next timestamp does not stretch the previous frame across the gap. producer.discontinuity() @@ -866,7 +866,7 @@ async fn rest( /// itself wedged. #[cfg(feature = "capture")] async fn capture_loop( - producer: &mut Producer, + producer: &mut Publisher, source: &mut S, options: &Capture, clock: &moq_mux::Clock, @@ -1084,7 +1084,7 @@ mod tests { } /// Encode a handful of synthetic frames for `codec` and publish them through a real - /// [`Producer`], returning the catalog rendition's track name and config. + /// [`Publisher`], returning the catalog rendition's track name and config. /// /// Asserts the property the whole design rests on: the rendition published before anything is /// encoded is the one the first keyframe resolves. A guessed codec string would be corrected @@ -1100,7 +1100,7 @@ mod tests { config.codec = codec; config.kind = kind; - let mut producer = Producer::new( + let mut producer = Publisher::new( broadcast .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(), @@ -1179,7 +1179,7 @@ mod tests { let mut config = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); config.kind = encoder::Kind::Software; - let mut producer = Producer::new( + let mut producer = Publisher::new( track, catalog.reserve().with_interval(config.effective_join()), config.probe().await.unwrap(), @@ -1211,7 +1211,7 @@ mod tests { let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); let mut initial = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); initial.kind = encoder::Kind::Software; - let mut producer = Producer::new( + let mut producer = Publisher::new( broadcast .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(), @@ -1266,7 +1266,7 @@ mod tests { let track = broadcast .unique_track(".avc3", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let _producer = Producer::new(track, catalog.reserve(), selected).unwrap(); + let _producer = Publisher::new(track, catalog.reserve(), selected).unwrap(); let (_, published) = rendition(&catalog).expect("the rendition publishes before any frame"); assert_eq!(published.container, hang::catalog::Container::Loc); @@ -1291,7 +1291,7 @@ mod tests { config.bitrate = Some(moq_net::bandwidth::Rate::from_mbps(6)); // Software (openh264) so the test is deterministic and never touches a hardware backend. config.kind = encoder::Kind::Software; - let _producer = Producer::new( + let _producer = Publisher::new( broadcast .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(), @@ -1338,7 +1338,7 @@ mod tests { .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(); let mut subscriber = track.subscribe(None); - let mut producer = Producer::new( + let mut producer = Publisher::new( track, catalog.reserve().with_interval(config.effective_join()), config.probe().await.unwrap(), @@ -1451,7 +1451,7 @@ mod tests { let mut probe = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); probe.kind = encoder::Kind::Software; - let mut producer = Producer::new( + let mut producer = Publisher::new( track, catalog.reserve().with_interval(probe.effective_join()), probe.probe().await.unwrap(), @@ -2249,7 +2249,7 @@ mod tests { let watched = track.clone(); let mut rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); rendition.kind = encoder::Kind::Software; - let mut producer = Producer::new( + let mut producer = Publisher::new( track, catalog.reserve().with_interval(rendition.effective_join()), rendition.probe().await.unwrap(), @@ -2306,7 +2306,7 @@ mod tests { let _subscriber = track.subscribe(None); let mut rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); rendition.kind = encoder::Kind::Software; - let mut producer = Producer::new( + let mut producer = Publisher::new( track, catalog.reserve().with_interval(rendition.effective_join()), rendition.probe().await.unwrap(), @@ -2391,7 +2391,7 @@ mod tests { } } -/// A finished [`Producer`] keeps its track and rendition published, but not its splitter. +/// A finished [`Publisher`] keeps its track and rendition published, but not its splitter. /// /// Bindings keep finished video producers until their broadcast ends, so a long-lived broadcast /// that retires many encoders would otherwise hold each one's largest access unit. The counting @@ -2481,7 +2481,7 @@ mod finished { let track = broadcast .unique_track(".avc3", catalog.track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let mut producer = Producer::new(track, catalog.reserve(), rendition.clone()).unwrap(); + let mut producer = Publisher::new(track, catalog.reserve(), rendition.clone()).unwrap(); for pts in [0, 33_333] { let pts = moq_net::Timestamp::from_micros(pts).unwrap(); producer.publish(&Encoded::new(keyframe(), pts, true)).unwrap(); diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 195dce50964..87442150ccc 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -33,7 +33,7 @@ //! limits it to while a subscriber is. Requires the `capture` feature. //! - [`encode::Encoder`] encodes [`Frame`]s you supply (from capture, a //! decoder, or your own pixels via [`Surface::rgba`]) and -//! [`encode::Producer`] publishes the results. +//! [`encode::Publisher`] publishes the results. //! - [`decode`] subscribes to an H.264, H.265, or AV1 track and decodes it to //! raw frames with a native backend (VideoToolbox on macOS and iOS, Media Foundation / //! DXVA on Windows, NVDEC, VAAPI, or an ARM SoC's V4L2 M2M decoder on Linux, From 219f410f5a6a82237636559b2baedbcccd0f7bb1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 09:11:10 -0800 Subject: [PATCH 03/14] feat(moq-video)!: encode::Producer takes raw frames and follows the grant The frame-taking producer owns its encoder, registers its rendition up front, and follows the bandwidth grant between frames, like moq_audio's. A lower rendition makes it a simulcast rung that disables and resumes. The capture driver now runs on it, so its rate control lives in one place. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-video.md | 9 +- rs/moq-video/README.md | 7 + rs/moq-video/src/encode/backend/probe.rs | 11 +- rs/moq-video/src/encode/capture.rs | 1659 +++++++++++++++++++ rs/moq-video/src/encode/mod.rs | 44 +- rs/moq-video/src/encode/producer.rs | 953 +++++++++++ rs/moq-video/src/encode/publisher.rs | 1899 +--------------------- rs/moq-video/src/lib.rs | 8 +- 8 files changed, 2695 insertions(+), 1895 deletions(-) create mode 100644 rs/moq-video/src/encode/capture.rs create mode 100644 rs/moq-video/src/encode/producer.rs diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 956747e5956..ac59ccc7061 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -40,7 +40,14 @@ build with the `v4l2` feature keeps every `Auto` or `Hardware` selection to its GOP, whichever encoder it lands on, since a V4L2 driver may refuse the request. Backends that can change bitrate live follow the connection's send estimate while someone -watches, and return to the configured bitrate while nobody does. Where the platform +watches, and return to the configured bitrate while nobody does. + +To publish frames from your own source, `encode::Producer` takes raw frames, +owns the encoder, and follows the same send estimate between frames, like +`moq_audio::encode::Producer`. Chain renditions through `Options::lower` for +simulcast: a higher rung stops encoding and advertises itself disabled while the +connection cannot fund it, then resumes on a keyframe. `encode::Publisher` is +the layer below, for frames you encoded yourself. Where the platform allows it, frames stay on the GPU between capture, decode, encode, and render. Opt-in features: `capture`, `pipewire` (Linux portal capture and PipeWire diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index 00216fef50f..5cf21a32661 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -137,6 +137,13 @@ Public entry points: the track and catalog are advertised up front, and the camera stays open whether or not anyone is watching. With `Capture::on_demand`, the camera opens only while a subscriber is watching and is released when the last one leaves. +- `encode::Producer` encodes raw frames you supply and publishes them + (`write(frame)`), registering the catalog rendition up front. Given the + connection's bandwidth allocator, it retunes the encoder to its grant between + frames; given the next lower rendition as `Options::lower`, it is a simulcast + rung that stops encoding while the grant cannot fund it and resumes on a + keyframe. `finish()` drains the encoder and returns a `Finished` that keeps the + track published. - `encode::Publisher` publishes frames you encoded yourself (`publish(&Encoded)`), handling the catalog and framing. Each is published at its own timestamp. diff --git a/rs/moq-video/src/encode/backend/probe.rs b/rs/moq-video/src/encode/backend/probe.rs index fd214fb1f81..90eec4fc5f1 100644 --- a/rs/moq-video/src/encode/backend/probe.rs +++ b/rs/moq-video/src/encode/backend/probe.rs @@ -6,6 +6,7 @@ //! //! It also holds each frame back by one, like the Media Foundation MFT, so a //! drain that loses the tail shows up as a missing frame rather than passing. +//! Every bitrate it accepts is logged too, so a test can follow rate control. use std::collections::VecDeque; use std::sync::Mutex; @@ -71,13 +72,11 @@ pub(crate) fn take() -> Vec { } /// Empty the accepted bitrates and hand them back. -#[cfg(feature = "capture")] pub(crate) fn bitrates() -> Vec { std::mem::take(&mut BITRATES.lock().unwrap()) } /// Fail the first `set_bitrate` asking for `bitrate`, then accept it again. -#[cfg(feature = "capture")] pub(crate) fn refuse_once(bitrate: u64) { *REFUSE.lock().unwrap() = Some(bitrate); } @@ -120,8 +119,12 @@ impl Backend for Probe { record("cut"); } // The payload is the frame's timestamp, so a test can tell which frame a - // packet came from independently of what it's stamped with. - let payload = bytes::Bytes::from(frame.timestamp.as_micros()?.to_string()); + // packet came from independently of what it's stamped with. It rides in an + // H.264 filler NAL, which a publisher parses and never emits as a frame, so a + // test can publish what the probe encodes without a real codec. + let mut payload = vec![0, 0, 0, 1, 0x0c]; + payload.extend_from_slice(frame.timestamp.as_micros()?.to_string().as_bytes()); + let payload = bytes::Bytes::from(payload); let previous = self.held.replace(Encoded::new(payload, frame.timestamp, cut)); self.ready.extend(previous); Ok(()) diff --git a/rs/moq-video/src/encode/capture.rs b/rs/moq-video/src/encode/capture.rs new file mode 100644 index 00000000000..153f59eff05 --- /dev/null +++ b/rs/moq-video/src/encode/capture.rs @@ -0,0 +1,1659 @@ +//! Capture a source, encode it, and publish it as a moq video track. +//! +//! The track and its catalog rendition are advertised immediately (the rendition is probed from +//! the encoder, since nothing has been encoded yet). Capture opens its camera at startup to learn +//! the mode it negotiates, then keeps it open and encodes whether or not anyone is watching, so a +//! late subscriber can reach the recent past within its `max_delay`. With +//! `Capture::on_demand` it instead closes the camera between viewers, encoding only while +//! [`Demand::wanted`](moq_net::track::Demand::wanted) is `Some`: a subscriber is watching. +//! +//! Capture advertises [`track::Info::refresh`](moq_net::track::Info::refresh) when its encoder +//! is sure to honor a cut, and forces a keyframe whenever a joining subscriber asks for a group +//! past the largest. The encoder's GOP stays the upper bound between keyframes. +//! +//! The capture loop feeds a [`Producer`], which owns the encoder and follows the grant, the same +//! one a caller with its own frames writes to. + +use moq_mux::catalog::hang::CatalogExt; +use moq_net::Timestamp; + +use crate::{Error, FrameRate, capture}; + +use super::cuts::Cuts; +use super::encoder::{self, Codec}; +use super::producer::{Producer, suffix}; + +/// Last-resort framerate when neither the caller nor the camera reports one. +const DEFAULT_FRAMERATE: FrameRate = FrameRate::integer(30); + +/// The [`track::Info::refresh`](moq_net::track::Info::refresh) ratio a capture advertises: a +/// joiner forces a keyframe once a group's deltas outweigh twice its keyframe, so later joiners +/// start near one. The joiner that asks still reads the cached group meanwhile. +/// +/// Measured on realtime CBR encodes (two-second GOPs at 2 to 4 Mbps), where a GOP's deltas run +/// three to five keyframes: one join a second halves the mean joiner download for 10 to 13% +/// more bitrate, and one every ten seconds costs about 1%. A ratio scales with the keyframe, +/// so it holds for every rendition. +const REFRESH: u32 = 2; + +/// Capture and encode settings for [`publish_capture`]. +/// +/// The encoded geometry (width / height / framerate) comes from the capture +/// source, not the caller. For the bring-your-own-frames +/// [`Encoder`](super::Encoder) path, where you specify geometry, use +/// [`Config`](super::Config) instead. +/// +/// Frames are stamped on the catalog's [`clock`](moq_mux::catalog::Producer::clock), the one +/// its consumers are told about, so a concurrent audio capture on the same catalog stays aligned. +/// +/// `#[non_exhaustive]`: construct via [`Capture::default`] and set fields, so +/// new settings can be added without breaking callers. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct Capture { + /// The source to capture. + pub capture: capture::Config, + /// Target bitrate; `None` derives one from the resolution. + /// + /// This is a ceiling, not a fixed rate: with [`bandwidth`](Self::bandwidth) + /// set, the encoder backs off below it while the uplink is congested and + /// climbs back afterwards, but never exceeds it. + pub bitrate: Option, + /// Output codec. Defaults to [`Codec::H264`]. + pub codec: Codec, + /// Encoder implementation preference. + pub kind: encoder::Kind, + /// The connection's bandwidth, as an allocator over + /// [`Session::send_bandwidth`](moq_net::Session::send_bandwidth) (or + /// `moq_tokio::Connection::send_bandwidth`, which survives reconnects). + /// + /// Set it and the encoder reserves this track's ceiling, then tracks its share of + /// the estimate per the default [`moq_mux::rate::Policy`], so a closing + /// uplink gets a softer picture instead of a stalled one. Pass the same allocator + /// to every sender on the connection, including the audio side: that's what keeps + /// their bitrates summing to the uplink instead of each matching it. + /// + /// Defaults to [`Allocator::unlimited`](moq_net::bandwidth::Allocator::unlimited), + /// which holds [`bitrate`](Self::bitrate) regardless of congestion. That's what you + /// want when the estimate isn't meaningful (a local file, a test harness) or + /// unavailable (a publisher that only accepts inbound sessions). + pub bandwidth: moq_net::bandwidth::Allocator, + /// Open the source and encode only while a subscriber wants the track, closing it between + /// viewers. + /// + /// Off by default: capture runs for the whole publish, so a late subscriber or fetch finds + /// the recent past in the track's cache. On demand saves the capture and encode while nobody + /// watches, and leaves nothing to fetch from before the first viewer arrived. + pub on_demand: bool, + /// The VA-API render node the encoder opens; see [`Config::vaapi`](encoder::Config::vaapi). + #[cfg(all(target_os = "linux", feature = "vaapi"))] + pub vaapi: Option, + /// The V4L2 M2M node the encoder opens; see [`Config::v4l2`](encoder::Config::v4l2). + #[cfg(all(target_os = "linux", feature = "v4l2"))] + pub v4l2: Option, +} + +impl Capture { + /// The encoder for a `width` x `height` source at `framerate`, with these options. + fn encoder(&self, width: u32, height: u32, framerate: FrameRate) -> encoder::Config { + let mut config = encoder::Config::new(width, height, framerate); + config.bitrate = self.bitrate; + config.codec = self.codec; + config.kind = self.kind.clone(); + #[cfg(all(target_os = "linux", feature = "vaapi"))] + { + config.vaapi = self.vaapi.clone(); + } + #[cfg(all(target_os = "linux", feature = "v4l2"))] + { + config.v4l2 = self.v4l2.clone(); + } + config + } +} + +/// The capture publication's current lifecycle state. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum Status { + /// On demand, the source is closed until a subscriber wants the track. + Waiting, + /// The source and its encoder are opening. + Starting, + /// Frames are being captured, encoded, and published. + Live, + /// The capture, encoder, or track failed with this error, which [`Driver::run`] returns. + Failed(Error), + /// The MoQ track ended, so the driver has stopped. + Ended, + /// The [`Driver`] or its [`run`](Driver::run) future was dropped before it + /// finished, cancelling the capture. + Cancelled, +} + +/// A snapshot of a capture publication's lifecycle. +#[derive(Clone, Debug)] +pub struct State { + status: Status, +} + +impl State { + /// The publication's current lifecycle state. + pub fn status(&self) -> &Status { + &self.status + } +} + +#[derive(Clone, Debug)] +struct Published { + state: State, + revision: u64, +} + +/// A new publication's state, before its driver runs. +fn starting() -> kio::Producer { + kio::Producer::new(Published { + state: State { + status: Status::Starting, + }, + revision: 0, + }) +} + +/// Publish a lifecycle transition, skipping one that changes nothing. +fn publish_status(state: &kio::Producer, status: Status) { + let Ok(mut published) = state.write() else { return }; + // A repeated status changes nothing an observer can see; a failure always carries news. + if matches!(status, Status::Failed(_)) + || std::mem::discriminant(&published.state.status) != std::mem::discriminant(&status) + { + published.state.status = status; + published.revision = published.revision.wrapping_add(1); + } +} + +/// Capture a webcam and publish it as a video track. +/// +/// Registers the track before anything about its source is known, and returns a +/// handle that observes the publication plus the [`Driver`] that captures it. The +/// driver opens the source to probe the mode it negotiates, so the catalog +/// rendition is exact before a frame is published, then captures and encodes +/// whether or not anyone is watching. Set [`Capture::on_demand`] to capture only +/// while a subscriber is watching instead. +pub fn publish_capture( + broadcast: moq_net::broadcast::Producer, + catalog: moq_mux::catalog::Producer, + options: Capture, +) -> Result<(Publication, Driver), Error> { + let track = capture_track(broadcast, &catalog, &options)?; + let state = starting(); + let publication = Publication { + state: state.consume(), + observed: 0, + }; + let driver = Driver { + owned: Owned::Track(track), + catalog, + options, + state, + }; + Ok((publication, driver)) +} + +/// A handle that observes a capture publication. +/// +/// Clones observe the same publication. Dropping it changes nothing: the +/// [`Driver`] owns the publication. +pub struct Publication { + state: kio::Consumer, + observed: u64, +} + +impl Clone for Publication { + fn clone(&self) -> Self { + Self { + state: self.state.clone(), + observed: self.state.read().revision, + } + } +} + +impl std::fmt::Debug for Publication { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Publication") + .field("state", &self.state.read().state) + .finish_non_exhaustive() + } +} + +impl Publication { + /// Return the latest state without waiting. + pub fn state(&self) -> State { + self.state.read().state.clone() + } + + /// Wait for a lifecycle change, or return `None` after the driver exits. + pub async fn changed(&mut self) -> Option { + let observed = self.observed; + let published = self + .state + .wait(move |published| { + if published.revision != observed { + std::task::Poll::Ready((**published).clone()) + } else { + std::task::Poll::Pending + } + }) + .await + .ok()?; + self.observed = published.revision; + Some(published.state) + } + + /// Whether the driver has exited. + pub fn is_finished(&self) -> bool { + self.state.is_closed() + } +} + +/// The task that captures, encodes, and publishes one video track. +/// +/// Dropping it, or its [`run`](Self::run) future, stops the capture, which also +/// abandons a camera open or permission prompt still in flight. +pub struct Driver { + owned: Owned, + catalog: moq_mux::catalog::Producer, + options: Capture, + state: kio::Producer, +} + +/// What a [`Driver`] owns of its track, one phase at a time, held in the driver so a +/// dropped [`run`](Driver::run) future still cancels it. +enum Owned { + /// Registered, until the probe knows its rendition. + Track(moq_net::track::Producer), + /// Probed, and wrapped in the producer that encodes and publishes it. + Producer(Box), + /// Ended or cancelled, so nothing is left to end. + Done, +} + +impl Owned { + /// End the track cleanly, best effort: it is usually already going away. + async fn finish(self) { + let ended = match self { + // The caller is done with the track, so the finished handle goes with it. + Self::Track(track) => track.finish().map(drop).map_err(Error::from), + // The capture is gone, so its encoder's tail has nothing to follow. Released, the encoder + // leaves nothing to drain, so the finish completes without waiting on it. + Self::Producer(mut producer) => match producer.release() { + Ok(()) => producer.finish().await.map(drop), + Err(err) => Err(err), + }, + Self::Done => Ok(()), + }; + if let Err(err) = ended { + tracing::debug!(error = %err, "video track finish after capture ended"); + } + } + + /// End the track with `err`. A track that already ended has nobody left to tell. + fn abort(self, err: moq_net::Error) { + match self { + Self::Track(track) => { + let _ = track.abort(err); + } + Self::Producer(producer) => producer.abort(err), + Self::Done => {} + } + } +} + +impl std::fmt::Debug for Driver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Driver").finish_non_exhaustive() + } +} + +impl Drop for Driver { + fn drop(&mut self) { + // Dropped before `run` finished: say so rather than `Dropped`. + std::mem::replace(&mut self.owned, Owned::Done).abort(moq_net::Error::Cancel); + // A no-op once `run` closed the state on its own terminal status. + publish_status(&self.state, Status::Cancelled); + } +} + +impl Driver { + /// Capture until the capture fails or its track ends. Closing or dropping the broadcast + /// leaves the track running; drop the [`Driver`] or this future to stop it. + /// + /// Frames are stamped on the catalog's clock, so a concurrent audio publish into the + /// same catalog stays aligned. Off macOS and iOS the future is `Send` and can be + /// spawned; on macOS the capture session is `!Send`, so await it on a local task there. + pub async fn run(self) -> Result<(), Error> { + self.run_from(DeviceSource).await + } + + /// [`run`](Self::run) from `source`, so the fixtures can drive it from a synthetic or + /// stalled one. + async fn run_from(mut self, source: S) -> Result<(), Error> { + let result = self.capture(source).await; + // This runs only when the capture ends on its own; dropping the future leaves the + // track to `Drop` instead, since async `Drop` can't finalize it. + let owned = std::mem::replace(&mut self.owned, Owned::Done); + match &result { + Ok(()) => owned.finish().await, + // Abort with the real cause so subscribers see it. + Err(err) => owned.abort(moq_net::Error::Transport(err.to_string())), + } + publish_status( + &self.state, + match &result { + Ok(()) => Status::Ended, + Err(err) => Status::Failed(err.clone()), + }, + ); + let _ = self.state.close(); + result + } + + /// Probe `source`, wrap the track in a producer for its rendition, then run the capture loop. + async fn capture(&mut self, mut source: S) -> Result<(), Error> { + let clock = self.catalog.clock(); + let options = &self.options; + + // Open the camera to find out what it actually negotiated, since a requested size is only a + // hint (macOS ignores it outright) and the encoder is built from the mode, not the request. + // This buys a rendition that says exactly what the stream will carry, rather than one every + // consumer has to treat as provisional, and lets a consumer size itself against it without + // waiting for an encoder that may not run yet. + let (rendition, config, camera) = async { + let camera = source.open(&options.capture).await?; + let mut probe_config = options.encoder( + camera.width(), + camera.height(), + options + .capture + .framerate + .or_else(|| camera.framerate()) + .unwrap_or(DEFAULT_FRAMERATE), + ); + probe_config.color = camera.color(); + let rendition = probe_config.probe().await?; + // The encoder is built from the same defaults, so the probe's join is the one it emits, + // and its ceiling the one the track first claims. + Ok::<_, Error>((rendition, probe_config, camera)) + } + .await?; + + // The producer takes a clone, so a setup failure leaves the registered track for `run` + // to abort, and success drops ours so it never outlives the producer's. + let Owned::Track(track) = &self.owned else { + unreachable!("a driver runs once") + }; + let producer = Producer::with_track( + track.clone(), + &self.catalog, + config, + rendition, + options.bandwidth.clone(), + None, + )?; + self.owned = Owned::Producer(Box::new(producer)); + let Owned::Producer(producer) = &mut self.owned else { + unreachable!("just set") + }; + // Capturing from the start, the loop keeps the probe's camera rather than reopening it, + // which could mean a second permission prompt or picker. On demand it closes until a viewer. + let mut source = Opened { + stream: (!options.on_demand).then_some(camera), + source, + }; + capture_loop(producer, &mut source, options, &clock, &self.state).await + } +} + +/// Register the capture's video track, before anything about its source is known. +/// +/// Takes `broadcast` only to mint the track, which outlives it. +fn capture_track( + broadcast: moq_net::broadcast::Producer, + catalog: &moq_mux::catalog::Producer, + options: &Capture, +) -> Result { + // Advertised only when every backend the encoder may open honors a cut, since the info + // is fixed before the capture learns which one it got. + let refresh = super::backend::always_cuts(options.codec, &options.kind).then_some(REFRESH); + let info = catalog.track_info(hang::catalog::PRIORITY.video).with_refresh(refresh); + Ok(broadcast.unique_track(suffix(options.codec), info)?) +} + +/// Off macOS and iOS, [`Driver::run`]'s future must stay `Send` so a server can +/// `tokio::spawn` it: the encoder runs on its own thread and the capture guard +/// is `Send` there. This is never called; it exists only to fail compilation if +/// the future ever regains a `!Send` component. Apple platforms are exempt (the inline encoder and objc +/// capture session are `!Send`). +#[cfg(not(apple))] +#[allow(dead_code, reason = "a compile-time `Send` assertion, never called")] +fn assert_driver_send(driver: Driver) { + fn is_send(_: &T) {} + is_send(&driver.run()); +} + +/// Where the capture opens its camera. Kept apart from the device backends so the +/// fixtures can drive the real capture from a synthetic or stalled source. +trait CaptureSource { + async fn open(&mut self, config: &capture::Config) -> Result; +} + +struct DeviceSource; + +/// Hands out a stream that is already open before opening `source` again. +struct Opened { + stream: Option, + source: S, +} + +impl CaptureSource for Opened { + async fn open(&mut self, config: &capture::Config) -> Result { + match self.stream.take() { + Some(stream) => Ok(stream), + None => self.source.open(config).await, + } + } +} + +impl CaptureSource for DeviceSource { + async fn open(&mut self, config: &capture::Config) -> Result { + capture::open(config).await + } +} + +/// An application dropping or closing its producer ends publishing normally; +/// a specific abort (such as a transport reset) stays visible in diagnostics. +fn log_track_ended(err: moq_net::Error) { + if matches!(err, moq_net::Error::Dropped | moq_net::Error::Closed) { + tracing::debug!("video track no longer announced; stopping capture"); + } else { + tracing::warn!(error = %err, "video track aborted; stopping capture"); + } +} + +/// Wait until capture should run (`run`) or release its source (`!run`). On demand it runs +/// while some viewer wants the track; otherwise it runs, and never releases, while the track +/// is open. Errors once the track closes. +async fn until_capturing(demand: &moq_net::track::Demand, on_demand: bool, run: bool) -> Result<(), moq_net::Error> { + let mut demand = demand.clone(); + loop { + let wanted = demand.wanted_changed().await?.is_some(); + if (wanted || !on_demand) == run { + return Ok(()); + } + } +} + +// Race source setup or an encode against capture releasing its source, so an on-demand track +// its last viewer left, or a track that ended, stops waiting promptly. `None` when it released; +// the next `until_capturing` reports a track that ended. +async fn wait_capture( + demand: &moq_net::track::Demand, + on_demand: bool, + work: impl std::future::Future>, +) -> Result, Error> { + tokio::select! { + biased; + _ = until_capturing(demand, on_demand, false) => Ok(None), + res = work => res.map(Some), + } +} + +/// Async capture/encode loop. Opens the camera and keeps it open, or +/// [`on_demand`](Capture::on_demand), opens it while at least one viewer is +/// watching and releases it when the last one leaves. It never returns on its +/// own; dropping its future stops it. +/// +/// The [`Producer`] holds the encoder, the track's claim on the connection, and +/// rate control, so they outlive a closed camera: a reopen at another mode moves +/// the claim's ceiling rather than lapsing it. +/// +/// Cancel safety: every wait here is a real `.await` (a frame read, a demand +/// transition, or an encode), so dropping this future (e.g. on Ctrl+C) drops +/// `camera`, which releases the device (LED off); the encode thread is joined +/// once the producer goes. Both the capture and encode threads sit idle between +/// frames, so their joins return promptly unless the underlying device or encoder +/// is itself wedged. +async fn capture_loop( + producer: &mut Producer, + source: &mut S, + options: &Capture, + clock: &moq_mux::Clock, + state: &kio::Producer, +) -> Result<(), Error> { + let capture = &options.capture; + let on_demand = options.on_demand; + // The published track's demand, followed onto the next track when a frame moves the rendition. + let mut demand = producer.demand(); + // The last timestamp sent to an encoder, which a reopened camera's frames must exceed. + let mut last: Option = None; + + loop { + // On demand, idle until a viewer subscribes. The catalog rendition was published when + // the track was created, so a subscriber can get here without a frame ever having been + // encoded. Either way the track ending is a clean exit. + if on_demand { + publish_status(state, Status::Waiting); + } + if let Err(err) = until_capturing(&demand, on_demand, true).await { + log_track_ended(err); + return Ok(()); + } + publish_status(state, Status::Starting); + + // Open the camera and an encoder sized to its negotiated mode. + let Some(mut camera) = wait_capture(&demand, on_demand, source.open(capture)).await? else { + continue; + }; + // Capture timestamps use a private monotonic timeline. Sample both clocks + // once at open so every queued frame maps to the shared broadcast epoch + // without mistaking dequeue time for acquisition time. + let mut capture_epoch = + u64::try_from(clock.now().as_micros()?.saturating_sub(camera.now().as_micros()?)).unwrap_or(u64::MAX); + // Prefer an explicit --fps, otherwise the camera's reported rate, falling + // back only if the backend doesn't expose one. + let framerate = capture + .framerate + .or_else(|| camera.framerate()) + .unwrap_or(DEFAULT_FRAMERATE); + let mut encoder_config = options.encoder(camera.width(), camera.height(), framerate); + encoder_config.color = camera.color(); + // Off macOS and iOS this opens the encoder on a dedicated thread; see `sink`. + // No cut on reopen: a fresh encoder opens with a keyframe on every backend, + // so the viewer whose subscription reopened the camera can decode from the + // first frame regardless, and a backend that cannot cut still captures. + if wait_capture(&demand, on_demand, producer.open(encoder_config)) + .await? + .is_none() + { + continue; + } + // A reopen can land on another backend than the probe did, so ask this one. + let cuts = producer.applied().is_some_and(|applied| applied.cut); + tracing::info!(encoder = producer.name(), device = camera.label(), "capturing"); + publish_status(state, Status::Live); + + // Per encoder, since a fresh one opens with a keyframe that serves every earlier ask. + let mut forced = Cuts::default(); + let mut watch = demand.clone(); + // The camera's skips already accounted for, from its first read on: frames it replaced + // while this encoder was still opening are startup, not frames the encoder fell behind on. + let mut reported = None; + + loop { + // Race the next frame against demand, so an on-demand capture releases the + // camera promptly when the last viewer leaves. `biased` checks demand first + // so an unwatched on-demand track stops before reading another frame. + let frame = tokio::select! { + biased; + wanted = watch.wanted_changed() => match wanted { + Ok(wanted) if on_demand && wanted.is_none() => break, // no viewers: release the camera + Ok(_) => continue, + Err(err) => { + log_track_ended(err); + return Ok(()); + } + }, + // A read error is terminal for this selection (the source is gone + // or was refused); `None` just ends the stream, so reopen below. + frame = camera.read() => frame?, + }; + + let Some(mut frame) = frame else { break }; + // Frames the camera replaced while this loop was encoding or retuning. + let skipped = camera.skipped(); + if let Some(reported) = reported { + producer.report_skipped(skipped - reported); + } + reported = Some(skipped); + let mut mapped = map_capture_timestamp(capture_epoch, frame.timestamp)?; + // A camera never rewinds its own stream, but a reopened one maps from the shared clock, + // which the previous stream can lead: a device re-anchor floors just past its last + // timestamp, ahead of arrival. Shift the new stream past it, keeping its spacing. + if let Some(last) = last + && mapped <= last + { + let behind = u64::try_from(last.as_micros()? - mapped.as_micros()?).unwrap_or(u64::MAX); + capture_epoch = capture_epoch.saturating_add(behind).saturating_add(1); + mapped = map_capture_timestamp(capture_epoch, frame.timestamp)?; + } + last = Some(mapped); + frame.timestamp = mapped; + // A backend that cannot cut leaves every group to its cadence. + let wanted = cuts.then(|| producer.wanted()).flatten(); + if forced.due(wanted) { + producer.cut().await?; + } + // Released mid-encode, the encoder refuses the next call, so the release below drops it. + if wait_capture(&demand, on_demand, producer.write(frame)).await?.is_none() { + break; + } + // A frame whose decoder identity differs from the probed one (a reopen onto another + // mode or backend) publishes under a new name and ends this track, and the producer's + // claim follows it there. Nobody watches the new track yet: its viewers find it in the + // catalog, so on demand the camera closes until one subscribes there. + if producer.demand().name() != demand.name() { + demand = producer.demand(); + tracing::info!(track = %demand.name(), "capture moved to a new track"); + if on_demand { + break; + } + watch = demand.clone(); + } + } + + // Frames the camera replaced after the last read, before demand or the stream ended + // the loop: no read follows to report them. + if let Some(reported) = reported { + producer.report_skipped(camera.skipped() - reported); + } + // Drop the camera (LED off) and encoder before opening another. The shared clock keeps + // advancing while capture is stopped, so mark the break before waiting for demand again, + // so the next timestamp does not stretch the previous frame across the gap. + drop(camera); + producer.release()?; + tracing::info!("capture stopped; released source"); + } +} + +fn map_capture_timestamp(epoch_micros: u64, timestamp: Timestamp) -> Result { + let capture_micros = u64::try_from(timestamp.as_micros()?).unwrap_or(u64::MAX); + Ok(Timestamp::from_micros(epoch_micros.saturating_add(capture_micros))?) +} + +#[cfg(test)] +mod tests { + #![cfg_attr( + not(feature = "openh264"), + allow(dead_code, unused_imports, reason = "only the `openh264` feature uses it") + )] + + use moq_mux::catalog::Stream as _; + + use super::*; + use crate::encode::Config; + + /// A producer for `config` on `track`, publishing the rendition the software encoder probes, + /// with no encoder open until the capture opens one. + async fn producer( + track: moq_net::track::Producer, + catalog: &moq_mux::catalog::Producer, + config: Config, + bandwidth: moq_net::bandwidth::Allocator, + ) -> Producer { + let mut probe = config.clone(); + probe.kind = encoder::Kind::Software; + let rendition = probe.probe().await.unwrap(); + Producer::with_track(track, catalog, config, rendition, bandwidth, None).unwrap() + } + + #[test] + fn capture_clock_mapping_is_monotonic() { + let first = map_capture_timestamp(10_000, Timestamp::from_micros(2_000).unwrap()).unwrap(); + let second = map_capture_timestamp(10_000, Timestamp::from_micros(2_001).unwrap()).unwrap(); + assert!(second > first); + } + + /// Clock fixtures: the real capture loop, fed by a synthetic camera against a pinned + /// broadcast clock, graded on the timestamps a subscriber reads back. The fixture installs a + /// manual clock, since paused tokio time would skip ahead while the encoder thread works: + /// time passes only where a test advances it, so a slow runner moves nothing. + /// + /// Each expectation is the acquisition instant measured on the broadcast clock. The loop + /// samples the broadcast clock and then the camera's timeline when it opens a camera, so a + /// published timestamp may land up to `SAMPLING` early, never late. + #[cfg(feature = "openh264")] + mod clock { + use kio::clock::Wall; + use std::time::Duration; + + use kio::clock::Instant; + + use super::*; + use crate::capture::Synthetic; + + /// How early a mapped timestamp may land: the gap between the loop's two clock samples. + const SAMPLING: Duration = Duration::from_millis(250); + /// Rounding slack on the late side: each clock reading truncates to a microsecond. + const ROUNDING: u64 = 2; + /// Retain every fixture group, so a slow runner never evicts one before it is read. + const RETAIN: Duration = Duration::from_secs(600); + + /// Hands the loop one fixture-supplied stream per camera open. + struct Opens(tokio::sync::mpsc::UnboundedReceiver); + + impl CaptureSource for Opens { + async fn open(&mut self, _config: &capture::Config) -> Result { + self.0 + .recv() + .await + .ok_or_else(|| Error::SourceUnavailable("the fixture stopped opening cameras".to_string())) + } + } + + struct Fixture { + time: kio::clock::Installed, + epoch: Instant, + clock: moq_mux::Clock, + catalog: moq_mux::catalog::Producer, + consumer: moq_net::broadcast::Consumer, + _broadcast: moq_net::broadcast::Producer, + opens: tokio::sync::mpsc::UnboundedSender, + stop: Option>, + task: tokio::task::JoinHandle>, + } + + impl Fixture { + /// Start the capture loop on a broadcast whose clock began `behind` ago, at `wall`. + async fn start(behind: Duration, wall: Wall) -> Self { + let time = kio::clock::Manual::new().install(&tokio::runtime::Handle::current()); + time.advance(behind).unwrap(); + let epoch = kio::clock::now() + .checked_sub(behind) + .expect("a monotonic clock that far back"); + let clock = moq_mux::Clock::at(epoch, wall).unwrap(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let config = moq_mux::catalog::Config::default() + .with_clock(clock.clone()) + .with_max_age(RETAIN); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); + let track = broadcast + .create_track( + "video", + catalog.track_info(hang::catalog::PRIORITY.video).with_max_age(RETAIN), + ) + .unwrap(); + + let mut probe = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); + probe.kind = encoder::Kind::Software; + let mut producer = + super::producer(track, &catalog, probe, moq_net::bandwidth::Allocator::unlimited()).await; + + let (opens, rx) = tokio::sync::mpsc::unbounded_channel(); + let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); + // Local: a capture stream is `!Send` on macOS, so each test runs on a `LocalSet`. + let capturing = clock.clone(); + let task = tokio::task::spawn_local(async move { + let mut source = Opens(rx); + // Each subscription opens the camera, so a fixture can idle and restart it. + let options = Capture { + kind: encoder::Kind::Software, + on_demand: true, + ..Capture::default() + }; + let state = starting(); + tokio::select! { + res = capture_loop(&mut producer, &mut source, &options, &capturing, &state) => res?, + _ = stopped => {} + } + producer.finish().await.map(drop) + }); + + Self { + time, + epoch, + clock, + catalog, + consumer, + _broadcast: broadcast, + opens, + stop: Some(stop), + task, + } + } + + /// Subscribe to the video track, which is what opens the camera. + async fn subscribe(&self) -> moq_mux::container::Consumer { + let snapshot = self.catalog.snapshot(); + let (name, rendition) = snapshot.video.renditions.iter().next().expect("the probed rendition"); + let container = moq_mux::catalog::hang::Container::try_from(rendition).unwrap(); + let track = self + .consumer + .track(name) + .unwrap() + .subscribe( + moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(RETAIN)), + ) + .await + .unwrap(); + moq_mux::container::Consumer::new(track, container) + } + + /// Plug in the camera the loop opens next. + fn camera(&self) -> Synthetic { + self.camera_sized(320, 240) + } + + /// Plug in a camera of another mode for the loop to open next. + fn camera_sized(&self, width: u32, height: u32) -> Synthetic { + let (camera, stream) = + Synthetic::open(crate::Size::new(width, height), crate::FrameRate::new(30, 1).unwrap()); + self.opens.send(stream).unwrap(); + camera + } + + /// `instant` on the broadcast clock, in microseconds. + fn at(&self, instant: Instant) -> u64 { + u64::try_from(instant.duration_since(self.epoch).as_micros()).unwrap() + } + + /// Stop the loop and finalize the track, as a clean end of capture does. + async fn finish(mut self) -> (moq_mux::catalog::Producer, moq_net::broadcast::Consumer) { + let _ = self.stop.take().expect("finished once").send(()); + self.task.await.unwrap().unwrap(); + (self.catalog, self.consumer) + } + + /// A frame acquired at `captured` publishes at that instant on the broadcast clock. + fn assert_acquired(&self, published: u64, captured: Instant) { + let exact = self.at(captured); + let early = u64::try_from(SAMPLING.as_micros()).unwrap(); + assert!( + published + early >= exact && published <= exact + ROUNDING, + "published {published}us, acquired at {exact}us on the broadcast clock" + ); + } + } + + pub(super) fn surface() -> crate::frame::Surface { + surface_sized(320, 240) + } + + fn surface_sized(width: u32, height: u32) -> crate::frame::Surface { + crate::frame::Surface::I420(crate::frame::I420 { + width, + height, + data: vec![0x80; usize::try_from(width * height * 3 / 2).unwrap()], + color: None, + }) + } + + pub(super) fn us(micros: u64) -> Timestamp { + Timestamp::from_micros(micros).unwrap() + } + + async fn read(track: &mut moq_mux::container::Consumer) -> u64 { + loop { + let read = track.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return u64::try_from(frame.timestamp.as_micros().unwrap()).unwrap(); + } + } + } + + /// Read the next frame not already in `seen`: a resubscription replays retained groups first. + async fn read_new( + track: &mut moq_mux::container::Consumer, + seen: &[u64], + ) -> u64 { + loop { + let timestamp = read(track).await; + if !seen.contains(×tamp) { + return timestamp; + } + } + } + + /// A camera whose first frame arrives long after the broadcast began stamps it at its + /// acquisition: not zero, and not the later instant the loop dequeued it. + #[tokio::test] + async fn a_late_first_frame_publishes_its_acquisition() { + tokio::task::LocalSet::new() + .run_until(async { + let fixture = Fixture::start(Duration::from_secs(5), kio::clock::wall()).await; + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + + let captured = Instant::now(); + // Delivered well after acquisition: dequeue time must not leak into the timestamp. + fixture.time.advance(Duration::from_millis(50)).unwrap(); + camera.push_at(surface(), captured); + let published = read(&mut track).await; + + assert!(published >= 4_000_000, "{published}us restarted the broadcast at zero"); + fixture.assert_acquired(published, captured); + fixture.finish().await; + }) + .await + } + + /// The catalog's stats carry the encoder's target and every frame the camera replaced + /// before the loop read it, once the loop has read its first. + #[tokio::test] + async fn stats_report_the_target_and_skipped_frames() { + tokio::task::LocalSet::new() + .run_until(async { + let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + + // Replaced before the first read, while the encoder was still opening: not a skip. + camera.push_at(surface(), Instant::now()); + camera.push_at(surface(), Instant::now()); + read(&mut track).await; + // Back to back once encoding, so the loop only ever sees the second. + camera.push_at(surface(), Instant::now()); + camera.push_at(surface(), Instant::now()); + read(&mut track).await; + + let stats = fixture.catalog.stats().remove("video").expect("the video rendition"); + assert_eq!(stats.skipped, 1); + let ceiling = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()).resolved_bitrate(); + assert_eq!(stats.bitrate, Some(ceiling.as_bps())); + fixture.finish().await; + }) + .await + } + + /// Frames the camera replaced after the last read still count when the last viewer + /// leaves before another read. + #[tokio::test] + async fn stats_report_skips_left_when_the_viewer_leaves() { + tokio::task::LocalSet::new() + .run_until(async { + let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + camera.push_at(surface(), Instant::now()); + read(&mut track).await; + + // The viewer leaves, then the camera replaces a frame before the loop runs again. + drop(track); + camera.push_at(surface(), Instant::now()); + camera.push_at(surface(), Instant::now()); + while !camera.released() { + tokio::task::yield_now().await; + } + + let stats = fixture.catalog.stats().remove("video").expect("the video rendition"); + assert_eq!(stats.skipped, 1); + fixture.finish().await; + }) + .await + } + + /// A device clock that restarts at zero, mid-stream or across a reopen, continues the + /// broadcast forward with the device's spacing instead of rewinding it. + #[tokio::test] + async fn a_device_clock_restart_continues_forward() { + tokio::task::LocalSet::new() + .run_until(async { + let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + + // The device numbers from zero, and the clock keeps pace with it. + camera.push_native(surface(), us(0)); + let first = read(&mut track).await; + fixture.time.advance(Duration::from_millis(40)).unwrap(); + camera.push_native(surface(), us(40_000)); + let second = read(&mut track).await; + assert_eq!(second - first, 40_000, "the device's spacing survives"); + + // The device restarts its clock without the stream ending, re-anchoring just past + // the last timestamp and so ahead of the clock. + camera.push_native(surface(), us(0)); + let restarted = read(&mut track).await; + assert!(restarted >= second, "{restarted}us rewound behind {second}us"); + fixture.time.advance(Duration::from_millis(40)).unwrap(); + camera.push_native(surface(), us(40_000)); + let resumed = read(&mut track).await; + assert_eq!(resumed - restarted, 40_000, "the device's spacing resumes"); + + // The device goes away and comes back numbering from zero again. + camera.close(); + let camera = fixture.camera(); + let pushed = Instant::now(); + camera.push_native(surface(), us(0)); + let reopened = read(&mut track).await; + let arrived = fixture.at(Instant::now()); + assert!(reopened >= resumed, "{reopened}us rewound across the reopen"); + let early = u64::try_from(SAMPLING.as_micros()).unwrap(); + assert!(reopened + early >= fixture.at(pushed) && reopened <= arrived + ROUNDING); + fixture.finish().await; + }) + .await + } + + /// Releasing the camera while nobody watches keeps the broadcast clock running: the + /// frame after a resume lands after the idle gap, at its own acquisition. + #[tokio::test] + async fn a_restart_after_idle_keeps_the_gap() { + tokio::task::LocalSet::new() + .run_until(async { + let idle = Duration::from_millis(300); + let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; + + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + let captured = Instant::now(); + camera.push_at(surface(), captured); + let before = read(&mut track).await; + fixture.assert_acquired(before, captured); + drop(track); + // The loop releases the camera before the idle gap starts. + while !camera.released() { + tokio::task::yield_now().await; + } + drop(camera); + + fixture.time.advance(idle).unwrap(); + + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + let captured = Instant::now(); + camera.push_at(surface(), captured); + let after = read_new(&mut track, &[before]).await; + fixture.assert_acquired(after, captured); + assert!( + after - before >= u64::try_from(idle.as_micros()).unwrap(), + "the {idle:?} idle gap collapsed to {}us", + after - before + ); + fixture.finish().await; + }) + .await + } + + /// A reopen onto a mode the probed level cannot carry is a new codec string, so it moves + /// the rendition to a new track: the old one ends, and a viewer who follows the catalog + /// there reopens the camera on it. + #[tokio::test] + async fn a_reopen_onto_a_new_level_follows_the_new_track() { + tokio::task::LocalSet::new() + .run_until(async { + let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; + let probed = fixture + .catalog + .snapshot() + .video + .renditions + .keys() + .next() + .unwrap() + .clone(); + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + camera.push_native(surface(), us(0)); + read(&mut track).await; + + // The device goes away and comes back in a mode four times the size. + camera.close(); + // Reopening takes the device a while, so its first frame is newer than the last. + fixture.time.advance(Duration::from_millis(40)).unwrap(); + let camera = fixture.camera_sized(640, 480); + camera.push_native(surface_sized(640, 480), us(40_000)); + // The probed track ends without the reopened camera's frames. + while let Some(read) = track.read().await.unwrap() { + assert!( + !matches!(read, moq_mux::container::Read::Frame { .. }), + "the reopened camera publishes on the probed track" + ); + } + + let mut moved = fixture.subscribe().await; + let name = fixture + .catalog + .snapshot() + .video + .renditions + .keys() + .next() + .unwrap() + .clone(); + assert_eq!(name, format!("{probed}/1")); + // Another reopen, another while. + fixture.time.advance(Duration::from_millis(40)).unwrap(); + let camera = fixture.camera_sized(640, 480); + camera.push_native(surface_sized(640, 480), us(80_000)); + let first = read(&mut moved).await; + let second = read_new(&mut moved, &[first]).await; + assert!(second > first, "the new track carries the reopened camera"); + fixture.finish().await; + }) + .await + } + + /// The wall mapping is pinned when the broadcast clock is built. A system clock stepped + /// an hour since then retimes neither the published timestamps nor the advertised mapping. + #[tokio::test] + async fn a_system_wall_adjustment_retimes_nothing() { + tokio::task::LocalSet::new() + .run_until(async { + // Whole seconds, so the advertised mapping holds it exactly. + let now = kio::clock::wall().checked_duration_since(Wall::UNIX_EPOCH).unwrap(); + let wall = Wall::UNIX_EPOCH + Duration::from_secs(now.as_secs() - 3600); + let fixture = Fixture::start(Duration::from_secs(1), wall).await; + let advertised = fixture.catalog.snapshot().media.clock; + assert_eq!(advertised, fixture.clock.wall()); + + let mut track = fixture.subscribe().await; + let camera = fixture.camera(); + let captured = Instant::now(); + camera.push_at(surface(), captured); + let published = read(&mut track).await; + + // Timestamps follow the monotonic epoch and map to walls under the pinned mapping. + fixture.assert_acquired(published, captured); + let mapped = advertised.unwrap().wall_clock(us(published)).unwrap(); + assert_eq!(mapped, wall + Duration::from_micros(published)); + assert_eq!(fixture.catalog.snapshot().media.clock, advertised); + fixture.finish().await; + }) + .await + } + } + + /// Publish fixtures: the real capture publish against sources that answer, or never do, on cue. + #[cfg(feature = "openh264")] + mod publish { + use std::collections::VecDeque; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use super::clock::{surface, us}; + use super::*; + use crate::capture::Synthetic; + use crate::encode::backend::probe; + + /// Hands out one queued stream per open, then stalls like a permission prompt nobody answers. + struct Streams { + queued: VecDeque, + opens: Arc, + } + + impl Streams { + fn new(queued: impl IntoIterator) -> Self { + Self { + queued: queued.into_iter().collect(), + opens: Arc::default(), + } + } + } + + impl CaptureSource for Streams { + async fn open(&mut self, _config: &capture::Config) -> Result { + self.opens.fetch_add(1, Ordering::SeqCst); + match self.queued.pop_front() { + Some(stream) => Ok(stream), + None => std::future::pending().await, + } + } + } + + fn camera() -> (Synthetic, capture::Stream) { + Synthetic::open(crate::Size::new(320, 240), crate::FrameRate::new(30, 1).unwrap()) + } + + async fn until(done: impl Fn() -> bool) { + while !done() { + tokio::task::yield_now().await; + } + } + + /// [`publish_capture`] from `source` rather than a device. + async fn publish_from( + broadcast: moq_net::broadcast::Producer, + catalog: moq_mux::catalog::Producer, + options: Capture, + source: impl CaptureSource, + ) -> Result<(), Error> { + let (_, driver) = publish_capture(broadcast, catalog, options)?; + driver.run_from(source).await + } + + async fn status(publication: &mut Publication, reached: impl Fn(&Status) -> bool) { + while !reached(publication.state().status()) { + publication.changed().await.expect("driver still alive"); + } + } + + /// What the capture is fed next, one at a time. + enum Step { + /// A frame 10ms after the previous one, waited on until the first subscriber has it. + Frame(crate::frame::Surface), + /// Another subscriber joining at the live edge, held until the end. + Join, + } + + /// A frame of seeded noise, which no encoder can predict from the one before it. + fn noise(seed: u32) -> crate::frame::Surface { + let mut state = seed.wrapping_mul(2_654_435_761) | 1; + let data = (0..320 * 240 * 3 / 2) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + 120 + (state as u8 >> 4) + }) + .collect(); + crate::frame::Surface::I420(crate::frame::I420 { + width: 320, + height: 240, + data, + color: None, + }) + } + + /// Drive the real capture publish through `steps`, returning the info the track advertised and the + /// payload size of every frame the first subscriber read, grouped as it read them. + async fn capture_groups(steps: Vec) -> (moq_net::track::Info, Vec>) { + use moq_mux::container::Container as _; + + let started = kio::clock::Instant::now(); + let result = local(async { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let options = Capture { + kind: encoder::Kind::Software, + // Room for noise, which rate control would otherwise skip frames of. + bitrate: Some(moq_net::bandwidth::Rate::from_mbps(4)), + ..Capture::default() + }; + // Ten nominal fps makes the default two-second GOP twenty frames, which every + // script below stays inside. The capture keeps the probe's camera. + let rate = crate::FrameRate::new(10, 1).unwrap(); + let (camera, stream) = Synthetic::open(crate::Size::new(320, 240), rate); + let run = tokio::task::spawn_local(publish_from( + broadcast.clone(), + catalog, + options, + Streams::new([stream]), + )); + let mut catalog = + moq_mux::catalog::Consumer::<()>::new(&consumer, moq_mux::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let (name, mut subscriber, format) = loop { + let snapshot = catalog.next().await.unwrap().expect("a catalog"); + if let Some((name, rendition)) = snapshot.video.renditions.iter().next() { + break ( + name.clone(), + consumer.track(name).unwrap().subscribe(None).await.unwrap(), + moq_mux::catalog::hang::Container::try_from(rendition).unwrap(), + ); + } + }; + let info = subscriber.info().clone(); + + // Each picture the subscriber reads, with the group it arrived in. + let (published, mut received) = tokio::sync::mpsc::unbounded_channel(); + let collect = tokio::task::spawn_local(async move { + while let Some(mut group) = subscriber.recv_group().await.unwrap() { + while let Some(wire) = group.read_frame().await.unwrap() { + for frame in format.decode(wire).unwrap() { + // A duration marker closes the group but is not an encoded picture. + if format.end(&frame).is_none() { + published.send((group.start, frame.payload.len())).unwrap(); + } + } + } + } + }); + + let mut groups: Vec> = Vec::new(); + let mut sequence = None; + let mut joined = Vec::new(); + let mut millis = 0; + for step in steps { + match step { + Step::Frame(surface) => { + camera.push_native(surface, us(millis * 1000)); + millis += 10; + // Wait for delivery before the next step, which also keeps the + // capture's latest slot from being replaced. + let (group, len) = received.recv().await.expect("the frame reached the subscriber"); + if sequence != Some(group) { + sequence = Some(group); + groups.push(Vec::new()); + } + groups.last_mut().unwrap().push(len); + } + Step::Join => joined.push(consumer.track(&name).unwrap().subscribe(None).await.unwrap()), + } + } + assert!(!run.is_finished(), "the capture ended early: {:?}", run.await); + run.abort(); + collect.abort(); + (info, groups) + }) + .await; + assert_eq!(started.elapsed(), Duration::ZERO); + result + } + + /// A capture whose encoder always cuts advertises the refresh ratio, and a join opens a + /// group at the next frame only once the deltas outweigh it, well within 500ms of the + /// keyframe before. + #[tokio::test(start_paused = true)] + async fn a_join_over_the_refresh_ratio_forces_a_keyframe() { + let mut steps = vec![ + Step::Frame(noise(0)), + Step::Frame(surface()), + Step::Frame(surface()), + // Two flat deltas cost next to nothing against a noise keyframe. + Step::Join, + Step::Frame(surface()), + ]; + steps.extend((1..=10).map(|seed| Step::Frame(noise(seed)))); + steps.extend([Step::Join, Step::Frame(surface()), Step::Frame(surface())]); + let (info, groups) = capture_groups(steps).await; + assert_eq!(info.refresh, Some(REFRESH)); + + let lengths: Vec<_> = groups.iter().map(Vec::len).collect(); + assert_eq!(lengths, [14, 2], "only the second join cut, at the frame after it"); + let ratio = usize::try_from(REFRESH).unwrap(); + let first = &groups[0]; + assert!( + first[1..3].iter().sum::() <= ratio * first[0], + "the first join was under the ratio: {first:?}" + ); + assert!( + first[1..].iter().sum::() > ratio * first[0], + "the second join was over the ratio: {first:?}" + ); + } + + /// Refresh is advertised only when every backend the encoder may open honors a cut, since + /// the track info is fixed before the capture learns which one it got. + #[test] + fn a_backend_that_may_refuse_cuts_advertises_no_refresh() { + for (kind, refresh) in [ + (encoder::Kind::Software, Some(REFRESH)), + (encoder::Kind::Named(probe::NO_CUT.into()), None), + ] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let options = Capture { + kind, + ..Capture::default() + }; + let track = capture_track(broadcast.clone(), &catalog, &options).unwrap(); + assert_eq!(track.info().refresh, refresh); + } + } + + /// Run `test` on a `LocalSet`, since capture streams are `!Send` on macOS. + /// + /// Off Apple, encoding waits on an OS thread, so a blocking task keeps Tokio from + /// auto-advancing the paused clock to a timer deadline while that thread answers. + async fn local(test: impl std::future::Future) -> T { + let (hold, released) = tokio::sync::oneshot::channel::<()>(); + let clock = tokio::task::spawn_blocking(move || { + let _ = released.blocking_recv(); + }); + let result = tokio::task::LocalSet::new().run_until(test).await; + drop(hold); + clock.await.unwrap(); + result + } + + /// The name of the first video rendition `consumer`'s catalog publishes. + async fn rendition_name(consumer: &moq_net::broadcast::Consumer) -> String { + let mut catalog = moq_mux::catalog::Consumer::<()>::new(consumer, moq_mux::catalog::CatalogFormat::Hang) + .await + .unwrap(); + loop { + let snapshot = catalog.next().await.unwrap().expect("a catalog"); + if let Some(name) = snapshot.video.renditions.keys().next() { + return name.clone(); + } + } + } + + /// Nobody watching, the capture still encodes, from the probe's camera rather than a + /// second open, and a subscriber arriving later reads what it wrote meanwhile. + #[tokio::test(start_paused = true)] + async fn an_unwatched_capture_keeps_its_past() { + local(async { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut options = Capture { + kind: encoder::Kind::Software, + ..Capture::default() + }; + // Half a frame per second rounds the default two-second GOP down to one frame, so + // every frame opens a group. + options.capture.framerate = Some(crate::FrameRate::new(1, 2).unwrap()); + let (camera, stream) = camera(); + let source = Streams::new([stream]); + let opens = source.opens.clone(); + let run = tokio::task::spawn_local(publish_from(broadcast.clone(), catalog, options, source)); + + let name = rendition_name(&consumer).await; + let mut track = consumer.track(&name).unwrap(); + // Each frame opens a group of its own, starting at the frame's pts. + camera.push_native(surface(), us(0)); + let first = track.largest().await.expect("the track is live").group; + camera.push_native(surface(), us(2_000_000)); + while track.largest().await.expect("the track is live").group < first + 2_000_000 {} + assert_eq!( + opens.load(Ordering::SeqCst), + 1, + "the capture reopened the probe's camera" + ); + assert!(!camera.released(), "the capture released its camera"); + + let mut subscriber = consumer + .track(&name) + .unwrap() + .subscribe( + moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(5))), + ) + .await + .unwrap(); + // The groups start their frames' pts apart, wherever the capture's timeline begins. + let first = subscriber.recv_group().await.unwrap().expect("a cached group").start; + let second = subscriber.recv_group().await.unwrap().expect("a cached group").start; + assert_eq!(second - first, 2_000_000); + + assert!(!run.is_finished(), "the capture ended early: {:?}", run.await); + run.abort(); + }) + .await + } + + /// On demand, the probe's camera closes at once, and only a viewer opens it again. + #[tokio::test(start_paused = true)] + async fn on_demand_opens_the_camera_for_a_viewer() { + local(async { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let options = Capture { + kind: encoder::Kind::Software, + on_demand: true, + ..Capture::default() + }; + let (probed, probe) = camera(); + let (camera, stream) = camera(); + let source = Streams::new([probe, stream]); + let opens = source.opens.clone(); + let (mut publication, driver) = publish_capture(broadcast.clone(), catalog, options).unwrap(); + assert!(matches!(publication.state().status(), Status::Starting)); + let run = tokio::task::spawn_local(driver.run_from(source)); + + let name = rendition_name(&consumer).await; + until(|| probed.released()).await; + assert_eq!(opens.load(Ordering::SeqCst), 1, "only the probe opened the camera"); + status(&mut publication, |status| matches!(status, Status::Waiting)).await; + + let mut subscriber = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); + camera.push_native(surface(), us(0)); + subscriber.recv_group().await.unwrap().expect("a group"); + assert_eq!(opens.load(Ordering::SeqCst), 2, "the viewer opened the camera"); + assert!(matches!(publication.state().status(), Status::Live)); + + drop(subscriber); + until(|| camera.released()).await; + status(&mut publication, |status| matches!(status, Status::Waiting)).await; + assert!(!run.is_finished(), "the capture ended early: {:?}", run.await); + let mut subscriber = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); + run.abort(); + assert!(run.await.unwrap_err().is_cancelled()); + // Dropping the driver's future cancels the publication and its track. + let error = loop { + match subscriber.recv_group().await { + Ok(Some(_)) => {} + Ok(None) => panic!("the cancelled track ended cleanly"), + Err(error) => break error, + } + }; + assert!(format!("{error:?}").contains("Cancel"), "{error:?}"); + let mut last = None; + while let Some(state) = publication.changed().await { + last = Some(state); + } + assert!(matches!(last.as_ref().map(State::status), Some(Status::Cancelled))); + assert!(publication.is_finished()); + }) + .await + } + + /// Feed `camera` a frame at a time and drive `run` until the bitrates the probe encoder + /// accepted satisfy `done`, returning them. The producer follows its grant between frames. + async fn settle( + mut run: std::pin::Pin<&mut impl std::future::Future>>, + camera: &Synthetic, + micros: &mut u64, + done: impl Fn(&[u64]) -> bool, + ) -> Vec { + let mut accepted = Vec::new(); + loop { + accepted.extend(probe::bitrates()); + if done(&accepted) { + return accepted; + } + camera.push_native(surface(), us(*micros)); + *micros += 33_333; + tokio::select! { + res = &mut run => panic!("the capture ended: {res:?}"), + () = tokio::task::yield_now() => {} + } + } + } + + /// A watched capture follows its grant. Once nobody watches, it returns to its ceiling + /// and ignores the grant, and the next viewer's grant applies afresh. + #[tokio::test(start_paused = true)] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn an_unwatched_capture_rests_at_its_ceiling() { + use moq_net::bandwidth::Rate; + + let _probe = probe::exclusive(); + let ceiling = Rate::from_mbps(4); + let estimate = moq_net::bandwidth::Producer::new(); + estimate.set(Some(Rate::from_mbps(1))).unwrap(); + let bandwidth = moq_net::bandwidth::Allocator::new(estimate.consume()); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let watched = track.clone(); + let mut rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); + rendition.bitrate = Some(ceiling); + let mut producer = producer(track, &catalog, rendition, bandwidth.clone()).await; + + let (camera, stream) = camera(); + let mut source = Streams::new([stream]); + let options = Capture { + kind: encoder::Kind::Named(probe::NAME.into()), + bitrate: Some(ceiling), + bandwidth, + ..Capture::default() + }; + let clock = catalog.clock(); + let state = starting(); + let mut run = std::pin::pin!(capture_loop(&mut producer, &mut source, &options, &clock, &state)); + let mut micros = 0; + + let viewer = watched.subscribe(None); + let followed = settle(run.as_mut(), &camera, &mut micros, |set| !set.is_empty()).await; + assert!(followed.iter().all(|&bps| bps < ceiling.as_bps()), "{followed:?}"); + + drop(viewer); + let rested = settle(run.as_mut(), &camera, &mut micros, |set| { + set.contains(&ceiling.as_bps()) + }) + .await; + assert_eq!(rested, [ceiling.as_bps()]); + + // Unwatched, a grant change retunes nothing, however long it holds and however many + // frames are written meanwhile. + estimate.set(Some(Rate::from_kbps(500))).unwrap(); + for _ in 0..3 { + camera.push_native(surface(), us(micros)); + micros += 33_333; + tokio::select! { + res = &mut run => panic!("the capture ended: {res:?}"), + () = kio::clock::sleep(Duration::from_secs(20)) => {} + } + } + assert_eq!(probe::bitrates(), Vec::::new()); + + let _viewer = watched.subscribe(None); + let resumed = settle(run.as_mut(), &camera, &mut micros, |set| !set.is_empty()).await; + assert!(resumed.iter().all(|&bps| bps <= 500_000), "{resumed:?}"); + } + + /// A backend that cannot cut still captures, leaving every group to its cadence. + #[tokio::test] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_backend_that_cannot_cut_still_captures() { + let _probe = probe::exclusive(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let _subscriber = track.subscribe(None); + let rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); + let mut producer = producer(track, &catalog, rendition, moq_net::bandwidth::Allocator::unlimited()).await; + + let (camera, stream) = camera(); + let mut source = Streams::new([stream]); + let options = Capture { + kind: encoder::Kind::Named(probe::NO_CUT.into()), + ..Capture::default() + }; + let clock = catalog.clock(); + let state = starting(); + let mut run = std::pin::pin!(capture_loop(&mut producer, &mut source, &options, &clock, &state)); + + // The probe holds its first frame back, so this one reaches the codec but not the track. + camera.push_native(surface(), us(0)); + tokio::select! { + res = &mut run => panic!("the capture ended: {res:?}"), + () = until(|| probe::take().iter().any(|(event, _)| *event == "encode")) => {} + } + } + } +} diff --git a/rs/moq-video/src/encode/mod.rs b/rs/moq-video/src/encode/mod.rs index 22aad18f88a..f1de611e639 100644 --- a/rs/moq-video/src/encode/mod.rs +++ b/rs/moq-video/src/encode/mod.rs @@ -7,29 +7,33 @@ //! - `publish_capture` captures and publishes a webcam (turnkey), returning a //! `Publication` that reports its state and the `Driver` that runs it. //! Requires the `capture` feature. -//! - [`Encoder`] encodes raw [`Frame`](crate::Frame)s you supply into -//! [`Encoded`] access units, and [`Publisher`] publishes those (bring your own -//! frames). Build both for the same [`Codec`]. -//! - [`Sink`] is an [`Encoder`] that owns the thread it runs on, for an encoder -//! outliving a single thread's stack (a shared object, an FFI handle, a task -//! that migrates between workers). -//! - [`Publisher`] alone publishes frames you already encoded. +//! - [`Producer`] encodes raw [`Frame`](crate::Frame)s you supply and publishes +//! them (bring your own frames), following the uplink's bandwidth grant between +//! frames. The counterpart of `moq_audio::encode::Producer`; the capture driver +//! runs on it. +//! - [`Encoder`] encodes raw frames into [`Encoded`] access units without +//! publishing them. [`Sink`] is an [`Encoder`] that owns the thread it runs on, +//! for an encoder outliving a single thread's stack (a shared object, an FFI +//! handle, a task that migrates between workers). +//! - [`Publisher`] publishes frames you already encoded, with no rate control. //! -//! A [`Publisher`] advertises its catalog rendition as soon as the track exists, -//! resolved from the encoder itself via [`Config::probe`], so a subscriber can -//! discover a track nothing has encoded yet. That's what lets an on-demand -//! capture wait for its first viewer at all. +//! Both [`Producer`] and [`Publisher`] advertise their catalog rendition as soon +//! as the track exists, resolved from the encoder itself via [`Config::probe`], so +//! a subscriber can discover a track nothing has encoded yet. That's what lets an +//! on-demand capture wait for its first viewer at all. //! -//! `Capture` (with `capture`) / [`Kind`] / [`Config`] configure them. -//! [`Config::preset`] picks a [`Preset`], how the encoder trades latency for -//! compression at the configured bitrate. The decode/consume -//! counterpart (mirror of `moq-audio`'s consumer) lives in the sibling +//! [`Options`] / `Capture` (with `capture`) / [`Kind`] / [`Config`] configure +//! them. [`Config::preset`] picks a [`Preset`], how the encoder trades latency +//! for compression at the configured bitrate. The decode/consume counterpart +//! (mirror of `moq-audio`'s consumer) lives in the sibling //! [`decode`](crate::decode) module. //! //! The shared [`moq_mux::rate`] policy maps a congestion-control bandwidth -//! estimate onto the encoder's bitrate, which `publish_capture` drives for you. +//! estimate onto the encoder's bitrate, which [`Producer`] drives for you. pub(crate) mod backend; +#[cfg(feature = "capture")] +mod capture; #[cfg_attr( not(feature = "capture"), allow( @@ -40,15 +44,17 @@ pub(crate) mod backend; mod cuts; mod encoded; mod encoder; +mod producer; mod publisher; mod sink; pub use backend::NAMES; +#[cfg(feature = "capture")] +pub use capture::{Capture, Driver, Publication, State, Status, publish_capture}; pub use encoded::Encoded; pub use encoder::{Applied, Codec, Config, Encoder, Kind, Preset}; -pub use publisher::Publisher; -#[cfg(feature = "capture")] -pub use publisher::{Capture, Driver, Publication, State, Status, publish_capture}; +pub use producer::{Options, Producer, Rung}; +pub use publisher::{Finished, Publisher}; pub use sink::Sink; #[cfg(test)] diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs new file mode 100644 index 00000000000..7ad32b14822 --- /dev/null +++ b/rs/moq-video/src/encode/producer.rs @@ -0,0 +1,953 @@ +//! Encode raw frames and publish them as a moq video track, following the uplink's grant. + +use std::sync::Arc; +use std::task::Poll; + +use moq_mux::catalog::hang::CatalogExt; +use moq_mux::rate; +use moq_net::bandwidth::Rate; + +use super::Sink; +use super::encoder::{Codec, Config}; +use super::publisher::{Finished, Publisher}; +use crate::{Error, Frame}; + +/// Encode and publication policy for [`Producer`]. +/// +/// `#[non_exhaustive]`: construct via [`Options::new`] and set fields, so new knobs can be added +/// without breaking callers. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Options { + /// Track name to publish under. `None` derives a unique one from the codec (`0.avc3`, then + /// `1.avc3`, ...). Subscribers find it through the catalog either way. + pub track: Option, + /// The encoder: the geometry every written frame has, the codec, the backend, and the + /// [`bitrate`](Config::bitrate), which is the ceiling the grant never raises it past. + pub config: Config, + /// The connection's bandwidth, as an allocator over + /// [`Session::send_bandwidth`](moq_net::Session::send_bandwidth) (or + /// `moq_tokio::Connection::send_bandwidth`, which survives reconnects). + /// + /// The track reserves its ceiling against it and, while someone watches, retunes the encoder + /// to its share between frames, per the default [`rate::Policy`], so a closing uplink gets a + /// softer picture instead of a stalled one. Pass the same allocator to every sender on the + /// connection, audio included, so their bitrates sum to the uplink instead of each matching it. + /// + /// Defaults to [`Allocator::unlimited`](moq_net::bandwidth::Allocator::unlimited), which holds + /// the ceiling regardless of congestion. + pub bandwidth: moq_net::bandwidth::Allocator, + /// The next lower rendition of the same source, when this one is encoded beside it. + /// + /// Makes this rendition a simulcast rung: while the grant can no longer fund it, it stops + /// encoding, drops what is written to it, and advertises itself disabled, so the lower + /// renditions keep their share, and it resumes on a keyframe once the grant recovers. It takes + /// the track priority one below the lower rendition's, so the allocator fills that one first. + /// Needs a [`bandwidth`](Self::bandwidth) to follow, and a ceiling above the lower one's. + pub lower: Option, +} + +impl Options { + /// Encode with `config`, holding its bitrate. + pub fn new(config: Config) -> Self { + Self { + track: None, + config, + bandwidth: moq_net::bandwidth::Allocator::unlimited(), + lower: None, + } + } +} + +/// A rendition's place in a simulcast ladder, from [`Producer::rung`], which the next +/// rendition up takes as its [`Options::lower`]. +/// +/// Watches the rendition without keeping it publishing: once it finishes or drops, the rung +/// above has nothing to fall back to and never disables. +#[derive(Clone)] +pub struct Rung { + ceiling: kio::Consumer, + priority: i8, +} + +impl std::fmt::Debug for Rung { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Rung") + .field("ceiling", &*self.ceiling.read()) + .field("priority", &self.priority) + .finish() + } +} + +impl Rung { + /// The rendition's current ceiling, or `None` once it ended. Arms `waiter` for either change. + fn ceiling(&self, waiter: &kio::Waiter) -> Option { + match self.ceiling.poll(waiter, |_| Poll::<()>::Pending) { + Poll::Pending => Some(*self.ceiling.read()), + // The closure never completes, so this is the channel closing. + Poll::Ready(_) => None, + } + } +} + +/// What a [`Producer`]'s encoder is doing. +enum Encoding { + /// Encoding every frame written. + Open(Sink), + /// Stopped by a [`discontinuity`](Producer::discontinuity) or a capture between sources: the + /// next frame written reopens it. + Idle, + /// Stopped while the grant cannot fund this rung: frames written are dropped until it resumes. + Disabled, +} + +/// Encode raw [`Frame`]s and publish them as a moq video track (avc3 / hev1 by codec). +/// +/// The counterpart of `moq_audio::encode::Producer`: it owns its encoder, registers the catalog +/// rendition at construction, probed from the encoder so a subscriber can discover the track +/// before a frame is written, and follows the [`bandwidth`](Options::bandwidth) grant between +/// frames. [`Publisher`] is the layer below, for frames you encoded yourself. +/// +/// While nobody watches, a rendition with no [`lower`](Options::lower) one encodes at its +/// ceiling and ignores the grant: its groups wait in the cache rather than being paced to a +/// subscriber, and the next viewer's grant applies afresh. A rung keeps following the grant, +/// which reports the slice it would get back, so it resumes after its viewers left. +/// +/// The `async` methods are not cancel-safe, as [`Sink`]'s are not: a cancelled one leaves the +/// encoder refusing every later call, so drop the producer instead. +pub struct Producer { + publisher: Publisher, + encoding: Encoding, + /// The encoder settings, whose bitrate is the ceiling, to reopen with. + config: Config, + /// The ceiling, which caps every grant. A channel, so a rung above this rendition wakes when + /// it moves and when this rendition ends. + ceiling: kio::Producer, + /// The track priority, which a higher rung takes one below. + priority: i8, + lower: Option, + /// Reserves against the track a decoder identity change moves the rendition to. + allocator: moq_net::bandwidth::Allocator, + /// This rendition's claim on the track named by `demand`. + reservation: Arc, + /// The track `reservation` claims on, which stays the published one until a frame moves it. + demand: moq_net::track::Demand, + /// `None` once retired: the grant ended for good, so the encoder holds its rate. + grant: Option, + /// What the encoder produces, or reopens at. + target: rate::Target, + /// Whether someone watched at the last write, so a rendition that never disables rests at + /// its ceiling once nobody does. + watched: bool, + /// Whether nothing has been published since the last break marker, which a second break + /// would only repeat. + marked: bool, +} + +impl Producer { + /// Open the encoder and publish a track for it in `broadcast`, registering its rendition in + /// `catalog` immediately. + /// + /// The rendition comes from [`Config::probe`], so it says what the first keyframe will. An + /// unsupported codec, resolution, or backend fails here, before the track exists. + pub async fn new( + broadcast: &moq_net::broadcast::Producer, + catalog: moq_mux::catalog::Producer, + options: Options, + ) -> Result { + let Options { + track, + config, + bandwidth, + lower, + } = options; + let ceiling = config.resolved_bitrate(); + if let Some(below) = lower.as_ref().and_then(|lower| lower.ceiling(&kio::Waiter::noop())) + && below >= ceiling + { + return Err(Error::UnsupportedConfig(format!( + "the lower rendition's {} bps is not below this one's {} bps", + below.as_bps(), + ceiling.as_bps() + ))); + } + // Both before the track exists: a config this machine can't encode fails without leaving + // a track advertised that will never carry frames. The probe closes its encoder before this + // one opens, so only one codec session is live. + let rendition = config.probe().await?; + let sink = Sink::open(&config).await?; + + let priority = lower + .as_ref() + .map_or(hang::catalog::PRIORITY.video, |lower| lower.priority.saturating_sub(1)); + let info = catalog.track_info(priority); + let track = match track { + Some(name) => broadcast.create_track(name, info)?, + None => broadcast.unique_track(suffix(config.codec), info)?, + }; + let mut producer = Self::with_track(track, &catalog, config, rendition, bandwidth, lower)?; + producer.encoding = Encoding::Open(sink); + Ok(producer) + } + + /// Publish `rendition` on `track` with no encoder open yet: the next frame written, or + /// [`open`](Self::open), opens one. + pub(crate) fn with_track( + track: moq_net::track::Producer, + catalog: &moq_mux::catalog::Producer, + mut config: Config, + rendition: hang::catalog::VideoConfig, + allocator: moq_net::bandwidth::Allocator, + lower: Option, + ) -> Result { + let priority = track.info().priority; + let publisher = Publisher::new( + track, + catalog.reserve().with_interval(config.effective_join()), + rendition, + )?; + let ceiling = config.resolved_bitrate(); + config.bitrate = Some(ceiling); + let demand = publisher.demand(); + let reservation = Arc::new(allocator.reserve(&demand, ceiling)); + let grant = rate::Grant::new(reservation.consume(), policy(ceiling, lower.as_ref())); + publisher.report_bitrate(ceiling); + Ok(Self { + publisher, + encoding: Encoding::Idle, + config, + ceiling: kio::Producer::new(ceiling), + priority, + lower, + allocator, + reservation, + demand, + grant: Some(grant), + target: rate::Target::Enabled(ceiling), + watched: false, + marked: false, + }) + } + + /// A watch-only handle to the published track's subscriber demand. A frame that changes the + /// decoder identity, such as a resume at a ceiling past the declared level, moves the + /// rendition to a newly named track, so read this again after [`write`](Self::write). + pub fn demand(&self) -> moq_net::track::Demand { + self.publisher.demand() + } + + /// This rendition's place in a simulcast ladder, for the next rendition up to take as its + /// [`Options::lower`]. + pub fn rung(&self) -> Rung { + Rung { + ceiling: self.ceiling.consume(), + priority: self.priority, + } + } + + /// This track's claim on the [`bandwidth`](Options::bandwidth) allocator, whose + /// [`rate`](moq_net::bandwidth::Reservation::rate) is the grant the encoder follows. A frame + /// that moves the rendition to a new track claims on that one instead, so read this again + /// after [`write`](Self::write). + pub fn reservation(&self) -> &Arc { + &self.reservation + } + + /// The bitrate the encoder produces at, or `None` while a rung is disabled. + pub fn bitrate(&self) -> Option { + match self.target { + rate::Target::Enabled(bitrate) => Some(bitrate), + rate::Target::Disabled => None, + } + } + + /// Encode and publish one frame at its own timestamp, at the configured size. + /// + /// Follows any grant change first, so the encoder retunes between frames rather than mid- + /// encode. A backend that pipelines publishes an earlier frame's output here, so a call that + /// publishes nothing is normal. While a rung is disabled, the frame is dropped unencoded. + /// + /// Takes the frame as anything that can become an [`Arc`], like [`Sink::encode`], so a + /// source fanning one frame out to a ladder hands each rung a clone of the handle. + pub async fn write(&mut self, frame: impl Into>) -> Result<(), Error> { + self.follow().await; + if let Encoding::Idle = self.encoding { + let wanted = self.bitrate().unwrap_or(*self.ceiling.read()); + let sink = self.reopen(wanted).await?; + self.encoding = Encoding::Open(sink); + } + let Encoding::Open(sink) = &mut self.encoding else { + return Ok(()); + }; + sink.encode(frame).await?; + self.publish_ready() + } + + /// Cut a new group at the next written frame. + /// + /// Optional: the encoder already keyframes every [`join`](Config::join), and each of those cuts + /// a group. Reach for it to place a boundary the encoder cannot see, such as a scene change. + /// Fails when the encoder cannot force a keyframe, leaving groups to that interval. A stopped + /// encoder opens with a keyframe anyway, so this does nothing then. + pub async fn cut(&mut self) -> Result<(), Error> { + match &mut self.encoding { + Encoding::Open(sink) => sink.cut().await, + Encoding::Idle | Encoding::Disabled => Ok(()), + } + } + + /// Set the ceiling to `bitrate`, retuning the encoder to it from roughly the next frame. No + /// keyframe is forced. The reservation moves with it, so a later grant never retunes above it. + /// + /// Some backends cannot raise past the bitrate they opened at (openh264), so configure the + /// highest you will ask for and adapt downwards. Errors, changing nothing, when the encoder + /// cannot retune while running; it keeps encoding at its current rate. + pub async fn set_bitrate(&mut self, bitrate: Rate) -> Result<(), Error> { + if let Encoding::Open(sink) = &mut self.encoding { + sink.set_bitrate(bitrate).await?; + } + if !matches!(self.encoding, Encoding::Disabled) { + self.target = rate::Target::Enabled(bitrate); + self.publisher.report_bitrate(bitrate); + } + self.config.bitrate = Some(bitrate); + if let Ok(mut ceiling) = self.ceiling.write() { + *ceiling = bitrate; + } + self.reservation.update(bitrate); + // A new ceiling starts the grant over from it, since the encoder applied it itself. + if let Some(grant) = self.grant.as_mut() { + grant.set_policy(policy(bitrate, self.lower.as_ref())); + } + Ok(()) + } + + /// Mark a break in the published timeline: what is written next does not continue what came + /// before. + /// + /// Call this when the source stops rather than merely pausing between frames, such as a + /// paused game or a source switch. The encoder drains into the track and closes, so what + /// resumes opens a group with a keyframe from a fresh one. Errors when the drain fails, after + /// the break is marked anyway: only those last frames are lost. + pub async fn discontinuity(&mut self) -> Result<(), Error> { + let drained = match std::mem::replace(&mut self.encoding, Encoding::Idle) { + Encoding::Open(sink) => self.drain(sink).await, + Encoding::Idle => Ok(()), + // Stays disabled, so what is written is still dropped until the grant resumes it. + Encoding::Disabled => { + self.encoding = Encoding::Disabled; + Ok(()) + } + }; + self.mark()?; + drained + } + + /// Drain the encoder into the track and finish it. + /// + /// Consumes the producer, releasing the encoder and the bandwidth claim. The returned + /// [`Finished`] keeps the track and its catalog rendition published, and can still + /// [`abort`](Finished::abort) them. A drain that fails ends the track with an error instead, + /// since a clean end promises a subscriber everything the publisher produced. + pub async fn finish(self) -> Result { + let Self { + mut publisher, + encoding, + ceiling, + reservation, + .. + } = self; + // The claim goes before the drain, so siblings can take the room while the last frames go + // out, and the ceiling closes, so a rung above stops counting on this one now. + drop((ceiling, reservation)); + let drained = match encoding { + Encoding::Open(mut sink) => { + async { + sink.flush().await?; + while let Some(encoded) = sink.receive()? { + publisher.publish(&encoded)?; + } + Ok::<_, Error>(()) + } + .await + } + Encoding::Idle | Encoding::Disabled => Ok(()), + }; + match drained { + Ok(()) => Finished::new(publisher), + Err(err) => { + publisher.abort(moq_net::Error::Transport(err.to_string())); + Err(err) + } + } + } + + /// Abort the track with `err` instead of finishing it, so subscribers see the real cause rather + /// than [`moq_net::Error::Dropped`]. + pub fn abort(self, err: moq_net::Error) { + self.publisher.abort(err); + } +} + +/// What the capture driver needs beyond the public API: it opens and releases the encoder with +/// its camera rather than on the first frame and on a drain. +#[cfg(feature = "capture")] +impl Producer { + /// Open an encoder for `config`, a capture's source reopened at whatever mode it negotiated. + /// + /// The ceiling and its claim follow the mode, and rate control starts over from the ceiling, + /// since what the link did while the source was closed says little about now. + pub(crate) async fn open(&mut self, mut config: Config) -> Result<(), Error> { + let sink = Sink::open(&config).await?; + let ceiling = config.resolved_bitrate(); + config.bitrate = Some(ceiling); + self.config = config; + if let Ok(mut current) = self.ceiling.write() { + *current = ceiling; + } + self.reservation.update(ceiling); + if self.grant.is_some() { + self.grant = Some(rate::Grant::new( + self.reservation.consume(), + policy(ceiling, self.lower.as_ref()), + )); + } + self.target = rate::Target::Enabled(ceiling); + self.watched = false; + self.publisher.report_bitrate(ceiling); + self.encoding = Encoding::Open(sink); + Ok(()) + } + + /// Close the encoder without draining it and mark the break, for a capture whose source went + /// away, possibly mid-encode: a cancelled call leaves the encoder refusing the next. + pub(crate) fn release(&mut self) -> Result<(), Error> { + if let Encoding::Open(_) = self.encoding { + self.encoding = Encoding::Idle; + } + self.mark() + } + + /// The open encoder's backend name. + pub(crate) fn name(&self) -> Option<&str> { + match &self.encoding { + Encoding::Open(sink) => Some(sink.name()), + Encoding::Idle | Encoding::Disabled => None, + } + } + + /// What the open encoder's backend applied. + pub(crate) fn applied(&self) -> Option { + match &self.encoding { + Encoding::Open(sink) => Some(sink.applied()), + Encoding::Idle | Encoding::Disabled => None, + } + } + + /// The group a subscriber wants past the largest published; see [`Publisher::wanted`]. + pub(crate) fn wanted(&self) -> Option { + self.publisher.wanted() + } + + /// Report source frames the encoder never saw, in the catalog's stats. + pub(crate) fn report_skipped(&self, frames: u64) { + self.publisher.report_skipped(frames); + } +} + +impl Producer { + /// Publish every access unit the encoder has ready, then move the claim to the track the + /// rendition is on now, even when publishing failed, so it never stays on an ended track. + fn publish_ready(&mut self) -> Result<(), Error> { + let published = match &mut self.encoding { + Encoding::Open(sink) => Self::publish_from(sink, &mut self.publisher, &mut self.marked), + Encoding::Idle | Encoding::Disabled => Ok(()), + }; + self.follow_track(); + published + } + + fn publish_from(sink: &mut Sink, publisher: &mut Publisher, marked: &mut bool) -> Result<(), Error> { + while let Some(encoded) = sink.receive()? { + publisher.publish(&encoded)?; + *marked = false; + } + Ok(()) + } + + /// Flush `sink` and publish its tail, closing it. + async fn drain(&mut self, mut sink: Sink) -> Result<(), Error> { + sink.flush().await?; + let published = Self::publish_from(&mut sink, &mut self.publisher, &mut self.marked); + self.follow_track(); + published + } + + /// Publish the break marker, unless nothing was published since the last one. + fn mark(&mut self) -> Result<(), Error> { + if !self.marked { + self.publisher.discontinuity()?; + self.marked = true; + } + Ok(()) + } + + /// A frame whose decoder identity differs from the declared one publishes under a new name + /// and ends the old track. The claim moves there, following on from the current target. + fn follow_track(&mut self) { + let demand = self.publisher.demand(); + if demand.name() == self.demand.name() { + return; + } + tracing::info!(track = %demand.name(), "video moved to a new track"); + self.reservation = Arc::new(self.allocator.reserve(&demand, *self.ceiling.read())); + self.demand = demand; + if let Some(grant) = self.grant.as_mut() { + let mut moved = rate::Grant::new(self.reservation.consume(), grant.policy().clone()); + moved.assume(self.target); + *grant = moved; + } + // The new track's stats start empty. + if let rate::Target::Enabled(bitrate) = self.target { + self.publisher.report_bitrate(bitrate); + } + } + + /// Apply every target the grant has due, or rest at the ceiling while nobody watches a + /// rendition that never disables. + /// + /// A target that fails to apply is taken back and asked for again a ramp step later, so an + /// encoder that cannot retune at all costs one attempt per step while the grant moves. + async fn follow(&mut self) { + if self.lower.is_none() { + let watched = self.demand.wanted().is_some(); + let rested = self.watched && !watched; + self.watched = watched; + if rested { + self.rest().await; + } + if !watched { + return; + } + } + + let noop = kio::Waiter::noop(); + // Each target is one the grant would not hand out again without a new grant, a deadline, + // or a failure, all of which leave it pending until the next write. + loop { + let policy = policy(*self.ceiling.read(), self.lower.as_ref()); + let Some(grant) = self.grant.as_mut() else { return }; + grant.set_policy(policy); + let target = match grant.poll_next(&noop) { + Poll::Pending => return, + Poll::Ready(Ok(target)) => target, + Poll::Ready(Err(_)) => { + tracing::debug!("bandwidth grant ended; holding the current encoder bitrate"); + self.grant = None; + return; + } + }; + if let Err(err) = self.apply(target).await { + tracing::warn!(error = %err, ?target, "failed to follow the bandwidth grant"); + if let Some(grant) = self.grant.as_mut() { + grant.failed(); + } + return; + } + tracing::debug!(?target, "followed the bandwidth grant"); + } + } + + /// Return an encoder nobody watches to its ceiling, and start its rate control over from + /// there for the next viewer. + async fn rest(&mut self) { + // Retired: the grant is gone, and the encoder holds its rate. + if self.grant.is_none() { + return; + } + let ceiling = *self.ceiling.read(); + if let Err(err) = self.apply(rate::Target::Enabled(ceiling)).await { + // The grant still matches the rate the encoder kept, so the next viewer follows on from it. + tracing::warn!(error = %err, bitrate = ceiling.as_bps(), "failed to rest the encoder bitrate"); + return; + } + self.grant = Some(rate::Grant::new(self.reservation.consume(), policy(ceiling, None))); + } + + /// Apply a rate-control target, leaving the rendition as it was when that fails. + /// + /// A disable drains the encoder into the track and closes it; a resume reopens one, whose + /// first frame is a keyframe on every backend. + async fn apply(&mut self, target: rate::Target) -> Result<(), Error> { + match (target, &mut self.encoding) { + (rate::Target::Disabled, Encoding::Disabled) => Ok(()), + (rate::Target::Disabled, _) => { + // The catalog first, the only step that leaves the rendition as it was when it fails. + // After it the encoder goes whatever the drain reports, so the catalog never + // advertises a rendition whose frames are dropped. + self.publisher.set_enabled(false)?; + self.target = rate::Target::Disabled; + let drained = match std::mem::replace(&mut self.encoding, Encoding::Disabled) { + Encoding::Open(sink) => self.drain(sink).await, + Encoding::Idle | Encoding::Disabled => Ok(()), + }; + // The encoder is gone either way, so a lost tail only costs those frames. + if let Err(err) = drained { + tracing::warn!(error = %err, "failed to drain a disabled rendition's encoder"); + } + self.mark() + } + (rate::Target::Enabled(bitrate), Encoding::Open(sink)) => { + sink.set_bitrate(bitrate).await?; + self.target = target; + self.publisher.report_bitrate(bitrate); + Ok(()) + } + // Applied when the next frame reopens the encoder. + (rate::Target::Enabled(bitrate), Encoding::Idle) => { + self.target = target; + self.publisher.report_bitrate(bitrate); + Ok(()) + } + (rate::Target::Enabled(bitrate), Encoding::Disabled) => { + let sink = self.reopen(bitrate).await?; + if let Err(err) = self.publisher.set_enabled(true) { + self.target = rate::Target::Disabled; + return Err(err); + } + self.encoding = Encoding::Open(sink); + Ok(()) + } + } + } + + /// Open an encoder at the ceiling and retune it to `wanted`, which becomes the target once it + /// opens. A failed open changes nothing. + /// + /// One that cannot retune still opens, at the ceiling, which the grant then follows on from. + async fn reopen(&mut self, wanted: Rate) -> Result { + let mut sink = Sink::open(&self.config).await?; + let ceiling = *self.ceiling.read(); + let wanted = wanted.min(ceiling); + let applied = match wanted < ceiling { + true => match sink.set_bitrate(wanted).await { + Ok(()) => wanted, + Err(err) => { + tracing::warn!(error = %err, bitrate = wanted.as_bps(), "reopened encoder kept its ceiling"); + ceiling + } + }, + false => ceiling, + }; + self.target = rate::Target::Enabled(applied); + if applied != wanted + && let Some(grant) = self.grant.as_mut() + { + grant.assume(self.target); + } + self.publisher.report_bitrate(applied); + Ok(sink) + } +} + +/// The track name suffix for `codec`: avc3 or hev1, which carry their parameter sets in band. +pub(crate) fn suffix(codec: Codec) -> &'static str { + match codec { + Codec::H264 => ".avc3", + Codec::H265 => ".hev1", + } +} + +/// The policy a rendition follows: its `ceiling`, above the `lower` rendition's current ceiling. +fn policy(ceiling: Rate, lower: Option<&Rung>) -> rate::Policy { + let mut policy = rate::Policy::new(ceiling); + policy.lower = lower.and_then(|lower| lower.ceiling(&kio::Waiter::noop())); + policy +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use moq_net::Timestamp; + + use super::*; + use crate::encode::backend::probe; + use crate::encode::encoder; + + fn config(kind: encoder::Kind, ceiling: Rate) -> Config { + let mut config = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); + config.kind = kind; + config.bitrate = Some(ceiling); + config + } + + /// The rendition the probe backend would declare, if it emitted H.264: it emits the frame's + /// timestamp instead, which publishes nothing, so the declared shape never gets corrected. + fn rendition() -> hang::catalog::VideoConfig { + hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::H264(hang::catalog::H264 { + inline: true, + profile: 0x64, + constraints: 0, + level: 0x1f, + })) + } + + /// A producer on the probe backend, publishing `name` with no encoder open yet. + fn probed( + broadcast: &moq_net::broadcast::Producer, + catalog: &moq_mux::catalog::Producer, + name: &str, + ceiling: Rate, + bandwidth: moq_net::bandwidth::Allocator, + lower: Option, + ) -> Producer { + let track = broadcast + .create_track(name, catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let config = config(encoder::Kind::Named(probe::NAME.into()), ceiling); + Producer::with_track(track, catalog, config, rendition(), bandwidth, lower).unwrap() + } + + /// A mid-gray frame at `micros`, advancing it by one frame at 30 fps. + fn frame(micros: &mut u64) -> Frame { + let size = crate::Size::new(320, 240); + let i420 = crate::I420::new(size, vec![0x80; crate::I420::len(size).unwrap()]).unwrap(); + let frame = Frame::new(crate::Surface::I420(i420), Timestamp::from_micros(*micros).unwrap()); + *micros += 33_333; + frame + } + + fn enabled(catalog: &moq_mux::catalog::Producer, name: &str) -> bool { + catalog.snapshot().video.renditions[name].enabled + } + + fn events(name: &str) -> usize { + probe::take().iter().filter(|(event, _)| *event == name).count() + } + + /// A watched producer retunes to a falling grant between frames, and climbs back to its + /// ceiling after congestion clears although the grant never changes again, retrying a step + /// the encoder refused rather than waiting for the next change. + #[tokio::test(start_paused = true)] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn follows_a_falling_and_recovering_grant() { + let _probe = probe::exclusive(); + let ceiling = Rate::from_mbps(4); + let estimate = moq_net::bandwidth::Producer::new(); + estimate.set(Some(ceiling)).unwrap(); + let bandwidth = moq_net::bandwidth::Allocator::new(estimate.consume()); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut producer = probed(&broadcast, &catalog, "video", ceiling, bandwidth, None); + let _viewer = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + assert_eq!(producer.bitrate(), Some(ceiling)); + + estimate.set(Some(Rate::from_mbps(1))).unwrap(); + for _ in 0..3 { + tokio::time::advance(Duration::from_millis(33)).await; + producer.write(frame(&mut micros)).await.unwrap(); + } + let fallen = probe::bitrates(); + assert!(!fallen.is_empty(), "the encoder never followed the grant down"); + assert!(fallen.iter().all(|&bps| bps <= 1_000_000), "{fallen:?}"); + assert!(producer.bitrate().unwrap() <= Rate::from_mbps(1)); + + // The last grant change: the link recovers a second later. + tokio::time::advance(Duration::from_secs(1)).await; + estimate.set(Some(ceiling)).unwrap(); + probe::refuse_once(ceiling.as_bps()); + let _ = probe::take(); + + // A minute of frames is far past anything the ramp still needs. + for _ in 0..600 { + tokio::time::advance(Duration::from_millis(100)).await; + producer.write(frame(&mut micros)).await.unwrap(); + } + let recovered = probe::bitrates(); + assert_eq!( + recovered.last(), + Some(&ceiling.as_bps()), + "the ramp stalled: {recovered:?}" + ); + assert_eq!(producer.bitrate(), Some(ceiling)); + assert!( + events("set_bitrate") > recovered.len(), + "the refused step was never retried: {recovered:?}" + ); + + let _finished = producer.finish().await.unwrap(); + } + + /// A rung the grant cannot fund stops encoding, drops what is written, and advertises itself + /// disabled; resumed, it opens a fresh encoder and advertises itself again. + #[tokio::test] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn disables_and_reenables_its_rendition() { + let _probe = probe::exclusive(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let unlimited = moq_net::bandwidth::Allocator::unlimited(); + let low = probed( + &broadcast, + &catalog, + "low", + Rate::from_kbps(500), + unlimited.clone(), + None, + ); + let ceiling = Rate::from_mbps(2); + let mut high = probed(&broadcast, &catalog, "high", ceiling, unlimited, Some(low.rung())); + let mut micros = 0; + + high.write(frame(&mut micros)).await.unwrap(); + assert!(enabled(&catalog, "high")); + let _ = probe::take(); + + high.apply(rate::Target::Disabled).await.unwrap(); + assert!(!enabled(&catalog, "high")); + assert_eq!(high.bitrate(), None); + let disabled = probe::take(); + assert!(disabled.iter().any(|(event, _)| *event == "flush"), "{disabled:?}"); + assert!(disabled.iter().any(|(event, _)| *event == "drop"), "{disabled:?}"); + + high.write(frame(&mut micros)).await.unwrap(); + assert_eq!(events("encode"), 0, "a disabled rung encodes nothing"); + + let resumed = Rate::from_mbps(1); + high.apply(rate::Target::Enabled(resumed)).await.unwrap(); + assert!(enabled(&catalog, "high")); + assert_eq!(high.bitrate(), Some(resumed)); + assert_eq!(probe::bitrates(), [resumed.as_bps()]); + assert_eq!(events("open"), 1); + + high.write(frame(&mut micros)).await.unwrap(); + assert_eq!(events("encode"), 1); + + let _high = high.finish().await.unwrap(); + let _low = low.finish().await.unwrap(); + } + + /// A rung must sit above the rendition it falls back to: one that does not fails before its + /// track exists. + #[tokio::test] + async fn a_rung_at_or_below_its_lower_rendition_is_refused() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let unlimited = moq_net::bandwidth::Allocator::unlimited(); + let lower = probed(&broadcast, &catalog, "lower", Rate::from_mbps(2), unlimited, None); + + let mut options = Options::new(config(encoder::Kind::Named(probe::NAME.into()), Rate::from_mbps(2))); + options.lower = Some(lower.rung()); + assert!(Producer::new(&broadcast, catalog.clone(), options).await.is_err()); + assert_eq!(catalog.snapshot().video.renditions.len(), 1); + } + + /// A rung takes the priority below the rendition it falls back to, so the allocator funds + /// that one first. + #[tokio::test] + #[cfg(feature = "openh264")] + async fn a_rung_takes_the_priority_below_its_lower_rendition() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let unlimited = moq_net::bandwidth::Allocator::unlimited(); + let lower = probed(&broadcast, &catalog, "lower", Rate::from_kbps(500), unlimited, None); + + let mut options = Options::new(config(encoder::Kind::Software, Rate::from_mbps(1))); + options.lower = Some(lower.rung()); + let higher = Producer::new(&broadcast, catalog, options).await.unwrap(); + // Read back from the track it created. + assert_eq!(higher.priority, lower.priority - 1); + } + + /// Subscribe to `video`, skipping no group however far behind the reader falls. + #[cfg(feature = "openh264")] + async fn subscribe(broadcast: &moq_net::broadcast::Producer) -> moq_net::track::Subscriber { + let history = moq_net::track::Delay::default().with_max(None); + broadcast + .consume() + .track("video") + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_delay(history)) + .await + .unwrap() + } + + /// The frames in each group `consumer` reads, until the track ends. + #[cfg(feature = "openh264")] + async fn read(mut consumer: moq_net::track::Subscriber) -> Vec { + let mut groups = Vec::new(); + while let Some(mut group) = consumer.recv_group().await.unwrap() { + let mut frames = 0; + while group.next_frame().await.unwrap().is_some() { + frames += 1; + } + groups.push(frames); + } + groups + } + + /// A discontinuity drains the encoder and marks the break once, however often it is called, + /// and the next frame opens a fresh encoder on a keyframe. + #[tokio::test] + #[cfg(feature = "openh264")] + async fn a_discontinuity_marks_the_break_once() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut options = Options::new(config(encoder::Kind::Software, Rate::from_mbps(1))); + options.track = Some("video".into()); + let mut producer = Producer::new(&broadcast, catalog, options).await.unwrap(); + let consumer = subscribe(&broadcast).await; + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + producer.discontinuity().await.unwrap(); + producer.discontinuity().await.unwrap(); + producer.write(frame(&mut micros)).await.unwrap(); + let _finished = producer.finish().await.unwrap(); + + assert_eq!(read(consumer).await, [1, 1, 1]); + } + + /// A disabled rung's last frames reach the track before its break marker, and it resumes on + /// a keyframe, so a subscriber reads the gap as a break rather than a stall. + #[tokio::test] + #[cfg(feature = "openh264")] + async fn a_disabled_rung_resumes_on_a_new_group() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut options = Options::new(config(encoder::Kind::Software, Rate::from_mbps(1))); + options.track = Some("video".into()); + let mut producer = Producer::new(&broadcast, catalog, options).await.unwrap(); + let consumer = subscribe(&broadcast).await; + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + producer.apply(rate::Target::Disabled).await.unwrap(); + producer.write(frame(&mut micros)).await.unwrap(); + producer + .apply(rate::Target::Enabled(Rate::from_kbps(500))) + .await + .unwrap(); + producer.write(frame(&mut micros)).await.unwrap(); + let _finished = producer.finish().await.unwrap(); + + assert_eq!(read(consumer).await, [1, 1, 1]); + } +} diff --git a/rs/moq-video/src/encode/publisher.rs b/rs/moq-video/src/encode/publisher.rs index 913744b28fb..0acdfb35941 100644 --- a/rs/moq-video/src/encode/publisher.rs +++ b/rs/moq-video/src/encode/publisher.rs @@ -1,52 +1,12 @@ -//! Publish encoded video frames as a moq video track, with optional capture. -//! -//! The track and its catalog rendition are advertised immediately (the rendition is probed from -//! the encoder, since nothing has been encoded yet). Capture opens its camera at startup to learn -//! the mode it negotiates, then keeps it open and encodes whether or not anyone is watching, so a -//! late subscriber can reach the recent past within its `max_delay`. With -//! `Capture::on_demand` it instead closes the camera between viewers, encoding only while -//! [`Demand::wanted`](moq_net::track::Demand::wanted) is `Some`: a subscriber is watching. -//! -//! Capture advertises [`track::Info::refresh`](moq_net::track::Info::refresh) when its encoder -//! is sure to honor a cut, and forces a keyframe whenever a joining subscriber asks for a group -//! past the largest. The encoder's GOP stays the upper bound between keyframes. +//! Publish already-encoded video frames as a moq video track. use moq_mux::catalog::hang::CatalogExt; -#[cfg(feature = "capture")] -use moq_mux::rate; -#[cfg(any(feature = "capture", test))] -use moq_net::Timestamp; use crate::Error; -#[cfg(feature = "capture")] -use crate::FrameRate; -#[cfg(feature = "capture")] -use crate::capture; use super::Encoded; -#[cfg(feature = "capture")] -use super::Sink; -#[cfg(feature = "capture")] -use super::cuts::Cuts; -#[cfg(any(feature = "capture", test))] +#[cfg(test)] use super::encoder; -#[cfg(feature = "capture")] -use super::encoder::Codec; - -/// Last-resort framerate when neither the caller nor the camera reports one. -#[cfg(feature = "capture")] -const DEFAULT_FRAMERATE: FrameRate = FrameRate::integer(30); - -/// The [`track::Info::refresh`](moq_net::track::Info::refresh) ratio a capture advertises: a -/// joiner forces a keyframe once a group's deltas outweigh twice its keyframe, so later joiners -/// start near one. The joiner that asks still reads the cached group meanwhile. -/// -/// Measured on realtime CBR encodes (two-second GOPs at 2 to 4 Mbps), where a GOP's deltas run -/// three to five keyframes: one join a second halves the mean joiner download for 10 to 13% -/// more bitrate, and one every ten seconds costs about 1%. A ratio scales with the keyframe, -/// so it holds for every rendition. -#[cfg(feature = "capture")] -const REFRESH: u32 = 2; /// Convert the probed rendition into the importer hint published before the first frame. fn rendition_hint(rendition: hang::catalog::VideoConfig) -> moq_mux::catalog::VideoHint { @@ -82,13 +42,11 @@ enum Codecs { }, } -/// Publishes encoded video frames as a moq track (avc3 / hev1 depending on the -/// codec). +/// Publishes already-encoded video frames as a moq track (avc3 / hev1 depending on the +/// codec), for a caller that runs its own encoder. /// -/// Built on the async side so the track is advertised (and the catalog -/// registered) before the camera opens; this is what lets a subscriber -/// trigger capture on demand. The `moq_mux::codec` importer for the codec -/// handles catalog registration and framing. +/// The `moq_mux::codec` importer for the codec handles catalog registration and framing. +/// [`Producer`](super::Producer) runs the encoder too, and follows the uplink's grant. pub struct Publisher { codecs: Codecs, } @@ -150,8 +108,8 @@ impl Publisher { /// The group a subscriber wants past the largest published, which the next frame should /// open with a keyframe, or `None` while the largest serves. - #[cfg(feature = "capture")] - fn wanted(&self) -> Option { + #[cfg_attr(not(feature = "capture"), allow(dead_code, reason = "only capture forces keyframes"))] + pub(crate) fn wanted(&self) -> Option { let track = self.track(); let wanted = track.demand().wanted()?; let largest = track.largest().map(|largest| largest.group); @@ -211,8 +169,8 @@ impl Publisher { } /// Report capture frames the encoder never saw, in the catalog's stats. - #[cfg(feature = "capture")] - fn report_skipped(&self, frames: u64) { + #[cfg_attr(not(feature = "capture"), allow(dead_code, reason = "only capture skips frames"))] + pub(crate) fn report_skipped(&self, frames: u64) { match &self.codecs { Codecs::H264 { import, .. } => import.report_skipped(frames), Codecs::H265 { import, .. } => import.report_skipped(frames), @@ -220,8 +178,7 @@ impl Publisher { } /// Report the encoder's target bitrate, in the catalog's stats. - #[cfg(feature = "capture")] - fn report_bitrate(&self, bitrate: moq_net::bandwidth::Rate) { + pub(crate) fn report_bitrate(&self, bitrate: moq_net::bandwidth::Rate) { match &self.codecs { Codecs::H264 { import, .. } => import.report_bitrate(bitrate), Codecs::H265 { import, .. } => import.report_bitrate(bitrate), @@ -260,808 +217,33 @@ impl Publisher { } } -/// Capture and encode settings for [`publish_capture`]. -/// -/// The encoded geometry (width / height / framerate) comes from the capture -/// source, not the caller. For the bring-your-own-frames -/// [`Encoder`](super::Encoder) path, where you specify geometry, use -/// [`Config`](super::Config) instead. -/// -/// Frames are stamped on the catalog's [`clock`](moq_mux::catalog::Producer::clock), the one -/// its consumers are told about, so a concurrent audio capture on the same catalog stays aligned. -/// -/// `#[non_exhaustive]`: construct via [`Capture::default`] and set fields, so -/// new settings can be added without breaking callers. -#[derive(Clone, Debug, Default)] -#[non_exhaustive] -#[cfg(feature = "capture")] -pub struct Capture { - /// The source to capture. - pub capture: capture::Config, - /// Target bitrate; `None` derives one from the resolution. - /// - /// This is a ceiling, not a fixed rate: with [`bandwidth`](Self::bandwidth) - /// set, the encoder backs off below it while the uplink is congested and - /// climbs back afterwards, but never exceeds it. - pub bitrate: Option, - /// Output codec. Defaults to [`Codec::H264`]. - pub codec: Codec, - /// Encoder implementation preference. - pub kind: encoder::Kind, - /// The connection's bandwidth, as an allocator over - /// [`Session::send_bandwidth`](moq_net::Session::send_bandwidth) (or - /// `moq_tokio::Connection::send_bandwidth`, which survives reconnects). - /// - /// Set it and the encoder reserves this track's ceiling, then tracks its share of - /// the estimate per the default [`moq_mux::rate::Policy`], so a closing - /// uplink gets a softer picture instead of a stalled one. Pass the same allocator - /// to every sender on the connection, including the audio side: that's what keeps - /// their bitrates summing to the uplink instead of each matching it. - /// - /// Defaults to [`Allocator::unlimited`](moq_net::bandwidth::Allocator::unlimited), - /// which holds [`bitrate`](Self::bitrate) regardless of congestion. That's what you - /// want when the estimate isn't meaningful (a local file, a test harness) or - /// unavailable (a publisher that only accepts inbound sessions). - pub bandwidth: moq_net::bandwidth::Allocator, - /// Open the source and encode only while a subscriber wants the track, closing it between - /// viewers. - /// - /// Off by default: capture runs for the whole publish, so a late subscriber or fetch finds - /// the recent past in the track's cache. On demand saves the capture and encode while nobody - /// watches, and leaves nothing to fetch from before the first viewer arrived. - pub on_demand: bool, - /// The VA-API render node the encoder opens; see [`Config::vaapi`](encoder::Config::vaapi). - #[cfg(all(target_os = "linux", feature = "vaapi"))] - pub vaapi: Option, - /// The V4L2 M2M node the encoder opens; see [`Config::v4l2`](encoder::Config::v4l2). - #[cfg(all(target_os = "linux", feature = "v4l2"))] - pub v4l2: Option, -} - -#[cfg(feature = "capture")] -impl Capture { - /// The encoder for a `width` x `height` source at `framerate`, with these options. - fn encoder(&self, width: u32, height: u32, framerate: FrameRate) -> encoder::Config { - let mut config = encoder::Config::new(width, height, framerate); - config.bitrate = self.bitrate; - config.codec = self.codec; - config.kind = self.kind.clone(); - #[cfg(all(target_os = "linux", feature = "vaapi"))] - { - config.vaapi = self.vaapi.clone(); - } - #[cfg(all(target_os = "linux", feature = "v4l2"))] - { - config.v4l2 = self.v4l2.clone(); - } - config - } -} - -/// The capture publication's current lifecycle state. -#[cfg(feature = "capture")] -#[derive(Clone, Debug)] -#[non_exhaustive] -pub enum Status { - /// On demand, the source is closed until a subscriber wants the track. - Waiting, - /// The source and its encoder are opening. - Starting, - /// Frames are being captured, encoded, and published. - Live, - /// The capture, encoder, or track failed with this error, which [`Driver::run`] returns. - Failed(Error), - /// The MoQ track ended, so the driver has stopped. - Ended, - /// The [`Driver`] or its [`run`](Driver::run) future was dropped before it - /// finished, cancelling the capture. - Cancelled, -} - -/// A snapshot of a capture publication's lifecycle. -#[cfg(feature = "capture")] -#[derive(Clone, Debug)] -pub struct State { - status: Status, -} - -#[cfg(feature = "capture")] -impl State { - /// The publication's current lifecycle state. - pub fn status(&self) -> &Status { - &self.status - } -} - -#[cfg(feature = "capture")] -#[derive(Clone, Debug)] -struct Published { - state: State, - revision: u64, -} - -/// A new publication's state, before its driver runs. -#[cfg(feature = "capture")] -fn starting() -> kio::Producer { - kio::Producer::new(Published { - state: State { - status: Status::Starting, - }, - revision: 0, - }) -} - -/// Publish a lifecycle transition, skipping one that changes nothing. -#[cfg(feature = "capture")] -fn publish_status(state: &kio::Producer, status: Status) { - let Ok(mut published) = state.write() else { return }; - // A repeated status changes nothing an observer can see; a failure always carries news. - if matches!(status, Status::Failed(_)) - || std::mem::discriminant(&published.state.status) != std::mem::discriminant(&status) - { - published.state.status = status; - published.revision = published.revision.wrapping_add(1); - } -} - -/// Capture a webcam and publish it as a video track. -/// -/// Registers the track before anything about its source is known, and returns a -/// handle that observes the publication plus the [`Driver`] that captures it. The -/// driver opens the source to probe the mode it negotiates, so the catalog -/// rendition is exact before a frame is published, then captures and encodes -/// whether or not anyone is watching. Set [`Capture::on_demand`] to capture only -/// while a subscriber is watching instead. -#[cfg(feature = "capture")] -pub fn publish_capture( - broadcast: moq_net::broadcast::Producer, - catalog: moq_mux::catalog::Producer, - options: Capture, -) -> Result<(Publication, Driver), Error> { - let track = capture_track(broadcast, &catalog, &options)?; - let state = starting(); - let publication = Publication { - state: state.consume(), - observed: 0, - }; - let driver = Driver { - owned: Owned::Track(track), - catalog, - options, - state, - }; - Ok((publication, driver)) -} - -/// A handle that observes a capture publication. -/// -/// Clones observe the same publication. Dropping it changes nothing: the -/// [`Driver`] owns the publication. -#[cfg(feature = "capture")] -pub struct Publication { - state: kio::Consumer, - observed: u64, -} - -#[cfg(feature = "capture")] -impl Clone for Publication { - fn clone(&self) -> Self { - Self { - state: self.state.clone(), - observed: self.state.read().revision, - } - } -} - -#[cfg(feature = "capture")] -impl std::fmt::Debug for Publication { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Publication") - .field("state", &self.state.read().state) - .finish_non_exhaustive() - } -} - -#[cfg(feature = "capture")] -impl Publication { - /// Return the latest state without waiting. - pub fn state(&self) -> State { - self.state.read().state.clone() - } - - /// Wait for a lifecycle change, or return `None` after the driver exits. - pub async fn changed(&mut self) -> Option { - let observed = self.observed; - let published = self - .state - .wait(move |published| { - if published.revision != observed { - std::task::Poll::Ready((**published).clone()) - } else { - std::task::Poll::Pending - } - }) - .await - .ok()?; - self.observed = published.revision; - Some(published.state) - } - - /// Whether the driver has exited. - pub fn is_finished(&self) -> bool { - self.state.is_closed() - } -} - -/// The task that captures, encodes, and publishes one video track. -/// -/// Dropping it, or its [`run`](Self::run) future, stops the capture, which also -/// abandons a camera open or permission prompt still in flight. -#[cfg(feature = "capture")] -pub struct Driver { - owned: Owned, - catalog: moq_mux::catalog::Producer, - options: Capture, - state: kio::Producer, -} - -/// What a [`Driver`] owns of its track, one phase at a time, held in the driver so a -/// dropped [`run`](Driver::run) future still cancels it. -#[cfg(feature = "capture")] -enum Owned { - /// Registered, until the probe knows its rendition. - Track(moq_net::track::Producer), - /// Probed, and wrapped in the importer that publishes it. - Publisher(Box), - /// Ended or cancelled, so nothing is left to end. - Done, -} - -#[cfg(feature = "capture")] -impl Owned { - /// End the track cleanly, best effort: it is usually already going away. - fn finish(self) { - let ended = match self { - // The caller is done with the track, so the finished handle goes with it. - Self::Track(track) => track.finish().map(drop).map_err(Error::from), - Self::Publisher(mut producer) => producer.finish(), - Self::Done => Ok(()), - }; - if let Err(err) = ended { - tracing::debug!(error = %err, "video track finish after capture ended"); - } - } - - /// End the track with `err`. A track that already ended has nobody left to tell. - fn abort(self, err: moq_net::Error) { - match self { - Self::Track(track) => { - let _ = track.abort(err); - } - Self::Publisher(producer) => producer.abort(err), - Self::Done => {} - } - } -} - -#[cfg(feature = "capture")] -impl std::fmt::Debug for Driver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Driver").finish_non_exhaustive() - } -} - -#[cfg(feature = "capture")] -impl Drop for Driver { - fn drop(&mut self) { - // Dropped before `run` finished: say so rather than `Dropped`. - std::mem::replace(&mut self.owned, Owned::Done).abort(moq_net::Error::Cancel); - // A no-op once `run` closed the state on its own terminal status. - publish_status(&self.state, Status::Cancelled); - } -} - -#[cfg(feature = "capture")] -impl Driver { - /// Capture until the capture fails or its track ends. Closing or dropping the broadcast - /// leaves the track running; drop the [`Driver`] or this future to stop it. - /// - /// Frames are stamped on the catalog's clock, so a concurrent audio publish into the - /// same catalog stays aligned. Off macOS and iOS the future is `Send` and can be - /// spawned; on macOS the capture session is `!Send`, so await it on a local task there. - pub async fn run(self) -> Result<(), Error> { - self.run_from(DeviceSource).await - } - - /// [`run`](Self::run) from `source`, so the fixtures can drive it from a synthetic or - /// stalled one. - async fn run_from(mut self, source: S) -> Result<(), Error> { - let result = self.capture(source).await; - // This runs only when the capture ends on its own; dropping the future leaves the - // track to `Drop` instead, since async `Drop` can't finalize it. - let owned = std::mem::replace(&mut self.owned, Owned::Done); - match &result { - Ok(()) => owned.finish(), - // Abort with the real cause so subscribers see it. - Err(err) => owned.abort(moq_net::Error::Transport(err.to_string())), - } - publish_status( - &self.state, - match &result { - Ok(()) => Status::Ended, - Err(err) => Status::Failed(err.clone()), - }, - ); - let _ = self.state.close(); - result - } - - /// Probe `source`, wrap the track in an importer for its rendition, then run the capture loop. - async fn capture(&mut self, mut source: S) -> Result<(), Error> { - let clock = self.catalog.clock(); - let options = &self.options; - - // Open the camera to find out what it actually negotiated, since a requested size is only a - // hint (macOS ignores it outright) and the encoder is built from the mode, not the request. - // This buys a rendition that says exactly what the stream will carry, rather than one every - // consumer has to treat as provisional, and lets a consumer size itself against it without - // waiting for an encoder that may not run yet. - let (rendition, join, camera) = async { - let camera = source.open(&options.capture).await?; - let mut probe_config = options.encoder( - camera.width(), - camera.height(), - options - .capture - .framerate - .or_else(|| camera.framerate()) - .unwrap_or(DEFAULT_FRAMERATE), - ); - probe_config.color = camera.color(); - let rendition = probe_config.probe().await?; - // The encoder is built from the same defaults, so the probe's join is the one it emits. - Ok::<_, Error>((rendition, probe_config.effective_join(), camera)) - } - .await?; - - // The importer takes a clone, so a setup failure leaves the registered track for `run` - // to abort, and success drops ours so it never outlives the importer's. - let Owned::Track(track) = &self.owned else { - unreachable!("a driver runs once") - }; - let producer = Publisher::new(track.clone(), self.catalog.reserve().with_interval(join), rendition)?; - self.owned = Owned::Publisher(Box::new(producer)); - let Owned::Publisher(producer) = &mut self.owned else { - unreachable!("just set") - }; - // Capturing from the start, the loop keeps the probe's camera rather than reopening it, - // which could mean a second permission prompt or picker. On demand it closes until a viewer. - let mut source = Opened { - stream: (!options.on_demand).then_some(camera), - source, - }; - capture_loop(producer, &mut source, options, &clock, &self.state).await - } -} - -/// Register the capture's video track, before anything about its source is known. +/// A video track whose [`Producer`](super::Producer) [finished](super::Producer::finish) it: every +/// frame is published, and it can still be watched or aborted, but not written to. /// -/// Takes `broadcast` only to mint the track, which outlives it. -#[cfg(feature = "capture")] -fn capture_track( - broadcast: moq_net::broadcast::Producer, - catalog: &moq_mux::catalog::Producer, - options: &Capture, -) -> Result { - let suffix = match options.codec { - Codec::H264 => ".avc3", - Codec::H265 => ".hev1", - }; - // Advertised only when every backend the encoder may open honors a cut, since the info - // is fixed before the capture learns which one it got. - let refresh = super::backend::always_cuts(options.codec, &options.kind).then_some(REFRESH); - let info = catalog.track_info(hang::catalog::PRIORITY.video).with_refresh(refresh); - Ok(broadcast.unique_track(suffix, info)?) -} - -/// Off macOS and iOS, [`Driver::run`]'s future must stay `Send` so a server can -/// `tokio::spawn` it: the encoder runs on its own thread and the capture guard -/// is `Send` there. This is never called; it exists only to fail compilation if -/// the future ever regains a `!Send` component. Apple platforms are exempt (the inline encoder and objc -/// capture session are `!Send`). -#[cfg(all(feature = "capture", not(apple)))] -#[allow(dead_code, reason = "a compile-time `Send` assertion, never called")] -fn assert_driver_send(driver: Driver) { - fn is_send(_: &T) {} - is_send(&driver.run()); -} - -/// Where the capture opens its camera. Kept apart from the device backends so the -/// fixtures can drive the real capture from a synthetic or stalled source. -#[cfg(feature = "capture")] -trait CaptureSource { - async fn open(&mut self, config: &capture::Config) -> Result; +/// Dropping it releases the track like dropping the producer. +#[must_use = "dropping it releases the track and its catalog rendition"] +pub struct Finished { + publisher: Publisher, } -#[cfg(feature = "capture")] -struct DeviceSource; - -/// Hands out a stream that is already open before opening `source` again. -#[cfg(feature = "capture")] -struct Opened { - stream: Option, - source: S, -} - -#[cfg(feature = "capture")] -impl CaptureSource for Opened { - async fn open(&mut self, config: &capture::Config) -> Result { - match self.stream.take() { - Some(stream) => Ok(stream), - None => self.source.open(config).await, - } - } -} - -#[cfg(feature = "capture")] -impl CaptureSource for DeviceSource { - async fn open(&mut self, config: &capture::Config) -> Result { - capture::open(config).await +impl Finished { + /// Finish `publisher`'s track, keeping it published. + pub(crate) fn new(mut publisher: Publisher) -> Result { + publisher.finish()?; + Ok(Self { publisher }) } -} -/// The live rate control state. `None` once it has *retired*, which is the only -/// thing absence means now that every encoder has a share to read: an allocator -/// with nothing to divide grants `None` rather than being absent. Retiring stops -/// the `select!` arm from spinning on a channel that is permanently ready. -#[cfg(feature = "capture")] -type RateControl = Option; - -/// Wait for the next target bitrate, or forever when rate control is off or finished. -/// -/// Cancel-safe: [`Grant::poll_next`](rate::Grant::poll_next) keeps its state in -/// the follower, so losing this race to a frame drops no target. -#[cfg(feature = "capture")] -async fn next_target(rate: &mut RateControl) -> Result { - match rate { - Some(follower) => follower.next().await, - // Retired: park this arm forever so `select!` ignores it. - None => std::future::pending().await, - } -} - -/// Retune the encoder to the next target, returning it once the encoder takes it. -/// -/// An error means the grant is gone (the session ended for good), so rate control -/// retires and the encoder holds its current bitrate. -#[cfg(feature = "capture")] -async fn apply_target( - encoder: &mut Sink, - rate: &mut RateControl, - target: Result, -) -> Option { - let follower = rate.as_mut()?; - - let Ok(target) = target else { - tracing::debug!("bandwidth estimate ended; holding the current encoder bitrate"); - *rate = None; - return None; - }; - let rate::Target::Enabled(bitrate) = target else { - unreachable!("a capture follows its grant with no lower rendition, so it never disables") - }; - - match encoder.set_bitrate(bitrate).await { - Ok(()) => { - tracing::debug!(bitrate = bitrate.as_bps(), "adjusted encoder bitrate"); - Some(bitrate) - } - // The encoder can't retune, so keep encoding at the rate it opened with - // and stop asking. Dropping the source also stops the estimate arm, which - // would otherwise wake this loop for nothing on every change. - Err(Error::BitrateUnsupported(name)) => { - tracing::warn!(encoder = name, "encoder cannot follow the bandwidth estimate"); - *rate = None; - None - } - // A transient failure: the follower takes the target back and retries it, - // so a quiet grant still recovers. - Err(err) => { - tracing::warn!(error = %err, bitrate = bitrate.as_bps(), "failed to adjust encoder bitrate"); - follower.failed(); - None - } - } -} - -/// An application dropping or closing its producer ends publishing normally; -/// a specific abort (such as a transport reset) stays visible in diagnostics. -#[cfg(feature = "capture")] -fn log_track_ended(err: moq_net::Error) { - if matches!(err, moq_net::Error::Dropped | moq_net::Error::Closed) { - tracing::debug!("video track no longer announced; stopping capture"); - } else { - tracing::warn!(error = %err, "video track aborted; stopping capture"); - } -} - -#[cfg(any(feature = "capture", all(test, feature = "openh264")))] -fn capture_stopped(producer: &mut Publisher) -> Result<(), Error> { - // The shared clock keeps advancing while capture is stopped. Mark the break before waiting - // for demand again so the next timestamp does not stretch the previous frame across the gap. - producer.discontinuity() -} - -/// Wait until capture should run (`run`) or release its source (`!run`). On demand it runs -/// while some viewer wants the track; otherwise it runs, and never releases, while the track -/// is open. Errors once the track closes. -#[cfg(feature = "capture")] -async fn until_capturing(demand: &moq_net::track::Demand, on_demand: bool, run: bool) -> Result<(), moq_net::Error> { - let mut demand = demand.clone(); - loop { - let wanted = demand.wanted_changed().await?.is_some(); - if (wanted || !on_demand) == run { - return Ok(()); - } - } -} - -// Race source setup or an encode against capture releasing its source, so an on-demand track -// its last viewer left, or a track that ended, stops waiting promptly. `None` when it released; -// the next `until_capturing` reports a track that ended. -#[cfg(feature = "capture")] -async fn wait_capture( - demand: &moq_net::track::Demand, - on_demand: bool, - work: impl std::future::Future>, -) -> Result, Error> { - tokio::select! { - biased; - _ = until_capturing(demand, on_demand, false) => Ok(None), - res = work => res.map(Some), - } -} - -/// Return an encoder nobody watches to its ceiling, and start its rate control over from there -/// for the next viewer. Its groups wait in the cache rather than being paced to a subscriber, -/// and a past viewer's grant is stale by the time the next one arrives. Returns the ceiling once -/// the encoder takes it. -#[cfg(feature = "capture")] -async fn rest( - encoder: &mut Sink, - rate: &mut RateControl, - claim: &moq_net::bandwidth::Reservation, - ceiling: moq_net::bandwidth::Rate, -) -> Option { - // Retired: the encoder cannot retune, or the grant is gone and it holds its rate. - rate.as_ref()?; - match encoder.set_bitrate(ceiling).await { - Ok(()) => { - *rate = Some(rate::Grant::new(claim.consume(), rate::Policy::new(ceiling))); - Some(ceiling) - } - Err(Error::BitrateUnsupported(name)) => { - tracing::warn!(encoder = name, "encoder cannot follow the bandwidth estimate"); - *rate = None; - None - } - // The follower still matches the rate the encoder kept, so the next viewer follows on from it. - Err(err) => { - tracing::warn!(error = %err, bitrate = ceiling.as_bps(), "failed to rest the encoder bitrate"); - None - } + /// A watch-only handle to the track's subscriber demand. + pub fn demand(&self) -> moq_net::track::Demand { + self.publisher.demand() } -} -/// Async capture/encode loop. Opens the camera and keeps it open, or -/// [`on_demand`](Capture::on_demand), opens it while at least one viewer is -/// watching and releases it when the last one leaves. It never returns on its -/// own; dropping its future stops it. -/// -/// Cancel safety: every wait here is a real `.await` (a frame read, a demand -/// transition, or an encode), so dropping this future (e.g. on Ctrl+C) drops -/// `camera` and `encoder`, which release the device (LED off) and join the -/// encode thread. Both the capture and encode threads sit idle between frames, -/// so their joins return promptly unless the underlying device or encoder is -/// itself wedged. -#[cfg(feature = "capture")] -async fn capture_loop( - producer: &mut Publisher, - source: &mut S, - options: &Capture, - clock: &moq_mux::Clock, - state: &kio::Producer, -) -> Result<(), Error> { - // This track's claim on the connection. Taken on the first open, because the - // negotiated mode is what finally says how much this encoder can ever send, and - // held across reopens so the claim doesn't lapse while the camera is closed. - let mut reservation: Option = None; - let capture = &options.capture; - let on_demand = options.on_demand; - // The published track's demand, followed onto the next track when a frame moves the rendition. - let mut demand = producer.demand(); - // The last timestamp sent to an encoder, which a reopened camera's frames must exceed. - let mut last: Option = None; - - loop { - // On demand, idle until a viewer subscribes. The catalog rendition was published when - // the track was created, so a subscriber can get here without a frame ever having been - // encoded. Either way the track ending is a clean exit. - if on_demand { - publish_status(state, Status::Waiting); - } - if let Err(err) = until_capturing(&demand, on_demand, true).await { - log_track_ended(err); - return Ok(()); - } - publish_status(state, Status::Starting); - - // Open the camera and an encoder sized to its negotiated mode. - let Some(mut camera) = wait_capture(&demand, on_demand, source.open(capture)).await? else { - continue; - }; - // Capture timestamps use a private monotonic timeline. Sample both clocks - // once at open so every queued frame maps to the shared broadcast epoch - // without mistaking dequeue time for acquisition time. - let mut capture_epoch = - u64::try_from(clock.now().as_micros()?.saturating_sub(camera.now().as_micros()?)).unwrap_or(u64::MAX); - // Prefer an explicit --fps, otherwise the camera's reported rate, falling - // back only if the backend doesn't expose one. - let framerate = capture - .framerate - .or_else(|| camera.framerate()) - .unwrap_or(DEFAULT_FRAMERATE); - let mut encoder_config = options.encoder(camera.width(), camera.height(), framerate); - encoder_config.color = camera.color(); - // Off macOS and iOS this opens the encoder on a dedicated thread; see `sink`. - // No cut on reopen: a fresh encoder opens with a keyframe on every backend, - // so the viewer whose subscription reopened the camera can decode from the - // first frame regardless, and a backend that cannot cut still captures. - let Some(mut encoder) = wait_capture(&demand, on_demand, Sink::open(&encoder_config)).await? else { - continue; - }; - // A reopen can land on another backend than the probe did, so ask this one. - let cuts = encoder.applied().cut; - tracing::info!(encoder = encoder.name(), device = camera.label(), "capturing"); - publish_status(state, Status::Live); - - // A reopen can negotiate a different mode (a display resized while nothing was - // subscribed), and the claim follows it: the old ceiling would otherwise cap a - // larger mode below what it can send, or keep claiming room a smaller one no - // longer needs. - let ceiling = encoder_config.resolved_bitrate(); - let claim = reservation.get_or_insert_with(|| options.bandwidth.reserve(&demand, ceiling)); - claim.update(ceiling); - - // Rate control is per encoder: this one opened at the configured bitrate, - // so the policy's ceiling is that rate and the target starts there. A - // reopened camera starts optimistic again rather than inheriting the - // backed-off rate from whatever the link was doing last time. - let mut rate = Some(rate::Grant::new(claim.consume(), rate::Policy::new(ceiling))); - // The encoder's current target, reported again on any track the capture moves to. - let mut bitrate = ceiling; - producer.report_bitrate(bitrate); - // Per encoder, since a fresh one opens with a keyframe that serves every earlier ask. - let mut forced = Cuts::default(); - // Rate control follows the grant only while someone watches. The first poll of `watch` - // reports the current demand. - let mut watch = demand.clone(); - let mut watched = false; - // The camera's skips already accounted for, from its first read on: frames it replaced - // while this encoder was still opening are startup, not frames the encoder fell behind on. - let mut reported = None; - - loop { - // Race the next frame against demand, so an on-demand capture releases the - // camera promptly when the last viewer leaves. `biased` checks demand first - // so an unwatched on-demand track stops before reading another frame. - let frame = tokio::select! { - biased; - wanted = watch.wanted_changed() => { - let wanted = match wanted { - Ok(wanted) => wanted.is_some(), - Err(err) => { - log_track_ended(err); - return Ok(()); - } - }; - if on_demand && !wanted { - break; // no viewers: release the camera, then wait for one - } - if watched - && !wanted && let Some(rested) = rest(&mut encoder, &mut rate, claim, ceiling).await - { - bitrate = rested; - producer.report_bitrate(bitrate); - } - watched = wanted; - continue; - } - // Retune between frames rather than mid-encode, and only when - // the policy says the target actually moved. - target = next_target(&mut rate), if watched => { - if let Some(applied) = apply_target(&mut encoder, &mut rate, target).await { - bitrate = applied; - producer.report_bitrate(bitrate); - } - continue; - } - // A read error is terminal for this selection (the source is gone - // or was refused); `None` just ends the stream, so reopen below. - frame = camera.read() => frame?, - }; - - let Some(mut frame) = frame else { break }; - // Frames the camera replaced while this loop was encoding or retuning. - let skipped = camera.skipped(); - if let Some(reported) = reported { - producer.report_skipped(skipped - reported); - } - reported = Some(skipped); - let mut mapped = map_capture_timestamp(capture_epoch, frame.timestamp)?; - // A camera never rewinds its own stream, but a reopened one maps from the shared clock, - // which the previous stream can lead: a device re-anchor floors just past its last - // timestamp, ahead of arrival. Shift the new stream past it, keeping its spacing. - if let Some(last) = last - && mapped <= last - { - let behind = u64::try_from(last.as_micros()? - mapped.as_micros()?).unwrap_or(u64::MAX); - capture_epoch = capture_epoch.saturating_add(behind).saturating_add(1); - mapped = map_capture_timestamp(capture_epoch, frame.timestamp)?; - } - last = Some(mapped); - frame.timestamp = mapped; - // A backend that cannot cut leaves every group to its cadence. - let wanted = cuts.then(|| producer.wanted()).flatten(); - if forced.due(wanted) { - encoder.cut().await?; - } - if wait_capture(&demand, on_demand, encoder.encode(frame)).await?.is_none() { - break; - } - while let Some(encoded) = encoder.receive()? { - producer.publish(&encoded)?; - } - // A frame whose decoder identity differs from the probed one (a reopen onto another - // mode or backend) publishes under a new name and ends this track. Capture follows it - // there with a claim against the new track, which nobody watches yet: its viewers find - // it in the catalog, so on demand the camera closes until one subscribes there. - if producer.track().name() != demand.name() { - demand = producer.demand(); - tracing::info!(track = %demand.name(), "capture moved to a new track"); - *claim = options.bandwidth.reserve(&demand, ceiling); - if on_demand { - break; - } - watch = demand.clone(); - watched = false; - // The new track's stats start empty, so they learn the target whether or not it rests. - if let Some(rested) = rest(&mut encoder, &mut rate, claim, ceiling).await { - bitrate = rested; - } - producer.report_bitrate(bitrate); - } - } - - // Frames the camera replaced after the last read, before demand or the stream ended - // the loop: no read follows to report them. - if let Some(reported) = reported { - producer.report_skipped(camera.skipped() - reported); - } - // Drop the camera (LED off) and encoder before opening another. - drop(camera); - drop(encoder); - capture_stopped(producer)?; - tracing::info!("capture stopped; released source"); + /// Abort the track with `err`, so subscribers that have not drained it see the real cause. + pub fn abort(self, err: moq_net::Error) { + self.publisher.abort(err); } } -#[cfg(feature = "capture")] -fn map_capture_timestamp(epoch_micros: u64, timestamp: Timestamp) -> Result { - let capture_micros = u64::try_from(timestamp.as_micros()?).unwrap_or(u64::MAX); - Ok(Timestamp::from_micros(epoch_micros.saturating_add(capture_micros))?) -} - #[cfg(test)] mod tests { #![cfg_attr( @@ -1070,19 +252,12 @@ mod tests { )] use moq_mux::catalog::Stream as _; + use moq_net::Timestamp; use super::*; use crate::Frame; use crate::encode::{Codec, Config, Encoder}; - #[cfg(feature = "capture")] - #[test] - fn capture_clock_mapping_is_monotonic() { - let first = map_capture_timestamp(10_000, Timestamp::from_micros(2_000).unwrap()).unwrap(); - let second = map_capture_timestamp(10_000, Timestamp::from_micros(2_001).unwrap()).unwrap(); - assert!(second > first); - } - /// Encode a handful of synthetic frames for `codec` and publish them through a real /// [`Publisher`], returning the catalog rendition's track name and config. /// @@ -1190,7 +365,7 @@ mod tests { for timestamp in [0, 10_000_000] { if timestamp > 0 { - capture_stopped(&mut producer).unwrap(); + producer.discontinuity().unwrap(); } encoder.cut().unwrap(); let surface = crate::Surface::rgba(&rgba, crate::Size::new(320, 240)).unwrap(); @@ -1234,7 +409,7 @@ mod tests { for encoded in encoder.encode_all(&frame).unwrap() { producer.publish(&encoded).unwrap(); } - capture_stopped(&mut producer).unwrap(); + producer.discontinuity().unwrap(); } // 640x360 takes a higher level than the probed 320x240, so it is a new track. @@ -1377,1018 +552,6 @@ mod tests { assert_eq!(config.coded_width, Some(320)); assert_eq!(config.coded_height, Some(240)); } - - /// Clock fixtures: the real capture loop, fed by a synthetic camera against a pinned - /// broadcast clock, graded on the timestamps a subscriber reads back. The fixture installs a - /// manual clock, since paused tokio time would skip ahead while the encoder thread works: - /// time passes only where a test advances it, so a slow runner moves nothing. - /// - /// Each expectation is the acquisition instant measured on the broadcast clock. The loop - /// samples the broadcast clock and then the camera's timeline when it opens a camera, so a - /// published timestamp may land up to `SAMPLING` early, never late. - #[cfg(all(feature = "capture", feature = "openh264"))] - mod clock { - use kio::clock::Wall; - use std::time::Duration; - - use kio::clock::Instant; - - use super::*; - use crate::capture::Synthetic; - - /// How early a mapped timestamp may land: the gap between the loop's two clock samples. - const SAMPLING: Duration = Duration::from_millis(250); - /// Rounding slack on the late side: each clock reading truncates to a microsecond. - const ROUNDING: u64 = 2; - /// Retain every fixture group, so a slow runner never evicts one before it is read. - const RETAIN: Duration = Duration::from_secs(600); - - /// Hands the loop one fixture-supplied stream per camera open. - struct Opens(tokio::sync::mpsc::UnboundedReceiver); - - impl CaptureSource for Opens { - async fn open(&mut self, _config: &capture::Config) -> Result { - self.0 - .recv() - .await - .ok_or_else(|| Error::SourceUnavailable("the fixture stopped opening cameras".to_string())) - } - } - - struct Fixture { - time: kio::clock::Installed, - epoch: Instant, - clock: moq_mux::Clock, - catalog: moq_mux::catalog::Producer, - consumer: moq_net::broadcast::Consumer, - _broadcast: moq_net::broadcast::Producer, - opens: tokio::sync::mpsc::UnboundedSender, - stop: Option>, - task: tokio::task::JoinHandle>, - } - - impl Fixture { - /// Start the capture loop on a broadcast whose clock began `behind` ago, at `wall`. - async fn start(behind: Duration, wall: Wall) -> Self { - let time = kio::clock::Manual::new().install(&tokio::runtime::Handle::current()); - time.advance(behind).unwrap(); - let epoch = kio::clock::now() - .checked_sub(behind) - .expect("a monotonic clock that far back"); - let clock = moq_mux::Clock::at(epoch, wall).unwrap(); - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let config = moq_mux::catalog::Config::default() - .with_clock(clock.clone()) - .with_max_age(RETAIN); - let catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); - let track = broadcast - .create_track( - "video", - catalog.track_info(hang::catalog::PRIORITY.video).with_max_age(RETAIN), - ) - .unwrap(); - - let mut probe = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); - probe.kind = encoder::Kind::Software; - let mut producer = Publisher::new( - track, - catalog.reserve().with_interval(probe.effective_join()), - probe.probe().await.unwrap(), - ) - .unwrap(); - - let (opens, rx) = tokio::sync::mpsc::unbounded_channel(); - let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); - // Local: a capture stream is `!Send` on macOS, so each test runs on a `LocalSet`. - let capturing = clock.clone(); - let task = tokio::task::spawn_local(async move { - let mut source = Opens(rx); - // Each subscription opens the camera, so a fixture can idle and restart it. - let options = Capture { - kind: encoder::Kind::Software, - on_demand: true, - ..Capture::default() - }; - let state = starting(); - tokio::select! { - res = capture_loop(&mut producer, &mut source, &options, &capturing, &state) => res?, - _ = stopped => {} - } - producer.finish() - }); - - Self { - time, - epoch, - clock, - catalog, - consumer, - _broadcast: broadcast, - opens, - stop: Some(stop), - task, - } - } - - /// Subscribe to the video track, which is what opens the camera. - async fn subscribe(&self) -> moq_mux::container::Consumer { - let snapshot = self.catalog.snapshot(); - let (name, rendition) = snapshot.video.renditions.iter().next().expect("the probed rendition"); - let container = moq_mux::catalog::hang::Container::try_from(rendition).unwrap(); - let track = self - .consumer - .track(name) - .unwrap() - .subscribe( - moq_net::track::Subscription::default() - .with_delay(moq_net::track::Delay::default().with_max(RETAIN)), - ) - .await - .unwrap(); - moq_mux::container::Consumer::new(track, container) - } - - /// Plug in the camera the loop opens next. - fn camera(&self) -> Synthetic { - self.camera_sized(320, 240) - } - - /// Plug in a camera of another mode for the loop to open next. - fn camera_sized(&self, width: u32, height: u32) -> Synthetic { - let (camera, stream) = - Synthetic::open(crate::Size::new(width, height), crate::FrameRate::new(30, 1).unwrap()); - self.opens.send(stream).unwrap(); - camera - } - - /// `instant` on the broadcast clock, in microseconds. - fn at(&self, instant: Instant) -> u64 { - u64::try_from(instant.duration_since(self.epoch).as_micros()).unwrap() - } - - /// Stop the loop and finalize the track, as a clean end of capture does. - async fn finish(mut self) -> (moq_mux::catalog::Producer, moq_net::broadcast::Consumer) { - let _ = self.stop.take().expect("finished once").send(()); - self.task.await.unwrap().unwrap(); - (self.catalog, self.consumer) - } - - /// A frame acquired at `captured` publishes at that instant on the broadcast clock. - fn assert_acquired(&self, published: u64, captured: Instant) { - let exact = self.at(captured); - let early = u64::try_from(SAMPLING.as_micros()).unwrap(); - assert!( - published + early >= exact && published <= exact + ROUNDING, - "published {published}us, acquired at {exact}us on the broadcast clock" - ); - } - } - - pub(super) fn surface() -> crate::frame::Surface { - surface_sized(320, 240) - } - - fn surface_sized(width: u32, height: u32) -> crate::frame::Surface { - crate::frame::Surface::I420(crate::frame::I420 { - width, - height, - data: vec![0x80; usize::try_from(width * height * 3 / 2).unwrap()], - color: None, - }) - } - - pub(super) fn us(micros: u64) -> Timestamp { - Timestamp::from_micros(micros).unwrap() - } - - async fn read(track: &mut moq_mux::container::Consumer) -> u64 { - loop { - let read = track.read().await.unwrap().expect("a published frame"); - if let moq_mux::container::Read::Frame { frame, .. } = read { - return u64::try_from(frame.timestamp.as_micros().unwrap()).unwrap(); - } - } - } - - /// Read the next frame not already in `seen`: a resubscription replays retained groups first. - async fn read_new( - track: &mut moq_mux::container::Consumer, - seen: &[u64], - ) -> u64 { - loop { - let timestamp = read(track).await; - if !seen.contains(×tamp) { - return timestamp; - } - } - } - - /// A camera whose first frame arrives long after the broadcast began stamps it at its - /// acquisition: not zero, and not the later instant the loop dequeued it. - #[tokio::test] - async fn a_late_first_frame_publishes_its_acquisition() { - tokio::task::LocalSet::new() - .run_until(async { - let fixture = Fixture::start(Duration::from_secs(5), kio::clock::wall()).await; - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - - let captured = Instant::now(); - // Delivered well after acquisition: dequeue time must not leak into the timestamp. - fixture.time.advance(Duration::from_millis(50)).unwrap(); - camera.push_at(surface(), captured); - let published = read(&mut track).await; - - assert!(published >= 4_000_000, "{published}us restarted the broadcast at zero"); - fixture.assert_acquired(published, captured); - fixture.finish().await; - }) - .await - } - - /// The catalog's stats carry the encoder's target and every frame the camera replaced - /// before the loop read it, once the loop has read its first. - #[tokio::test] - async fn stats_report_the_target_and_skipped_frames() { - tokio::task::LocalSet::new() - .run_until(async { - let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - - // Replaced before the first read, while the encoder was still opening: not a skip. - camera.push_at(surface(), Instant::now()); - camera.push_at(surface(), Instant::now()); - read(&mut track).await; - // Back to back once encoding, so the loop only ever sees the second. - camera.push_at(surface(), Instant::now()); - camera.push_at(surface(), Instant::now()); - read(&mut track).await; - - let stats = fixture.catalog.stats().remove("video").expect("the video rendition"); - assert_eq!(stats.skipped, 1); - let ceiling = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()).resolved_bitrate(); - assert_eq!(stats.bitrate, Some(ceiling.as_bps())); - fixture.finish().await; - }) - .await - } - - /// Frames the camera replaced after the last read still count when the last viewer - /// leaves before another read. - #[tokio::test] - async fn stats_report_skips_left_when_the_viewer_leaves() { - tokio::task::LocalSet::new() - .run_until(async { - let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - camera.push_at(surface(), Instant::now()); - read(&mut track).await; - - // The viewer leaves, then the camera replaces a frame before the loop runs again. - drop(track); - camera.push_at(surface(), Instant::now()); - camera.push_at(surface(), Instant::now()); - while !camera.released() { - tokio::task::yield_now().await; - } - - let stats = fixture.catalog.stats().remove("video").expect("the video rendition"); - assert_eq!(stats.skipped, 1); - fixture.finish().await; - }) - .await - } - - /// A device clock that restarts at zero, mid-stream or across a reopen, continues the - /// broadcast forward with the device's spacing instead of rewinding it. - #[tokio::test] - async fn a_device_clock_restart_continues_forward() { - tokio::task::LocalSet::new() - .run_until(async { - let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - - // The device numbers from zero, and the clock keeps pace with it. - camera.push_native(surface(), us(0)); - let first = read(&mut track).await; - fixture.time.advance(Duration::from_millis(40)).unwrap(); - camera.push_native(surface(), us(40_000)); - let second = read(&mut track).await; - assert_eq!(second - first, 40_000, "the device's spacing survives"); - - // The device restarts its clock without the stream ending, re-anchoring just past - // the last timestamp and so ahead of the clock. - camera.push_native(surface(), us(0)); - let restarted = read(&mut track).await; - assert!(restarted >= second, "{restarted}us rewound behind {second}us"); - fixture.time.advance(Duration::from_millis(40)).unwrap(); - camera.push_native(surface(), us(40_000)); - let resumed = read(&mut track).await; - assert_eq!(resumed - restarted, 40_000, "the device's spacing resumes"); - - // The device goes away and comes back numbering from zero again. - camera.close(); - let camera = fixture.camera(); - let pushed = Instant::now(); - camera.push_native(surface(), us(0)); - let reopened = read(&mut track).await; - let arrived = fixture.at(Instant::now()); - assert!(reopened >= resumed, "{reopened}us rewound across the reopen"); - let early = u64::try_from(SAMPLING.as_micros()).unwrap(); - assert!(reopened + early >= fixture.at(pushed) && reopened <= arrived + ROUNDING); - fixture.finish().await; - }) - .await - } - - /// Releasing the camera while nobody watches keeps the broadcast clock running: the - /// frame after a resume lands after the idle gap, at its own acquisition. - #[tokio::test] - async fn a_restart_after_idle_keeps_the_gap() { - tokio::task::LocalSet::new() - .run_until(async { - let idle = Duration::from_millis(300); - let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; - - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - let captured = Instant::now(); - camera.push_at(surface(), captured); - let before = read(&mut track).await; - fixture.assert_acquired(before, captured); - drop(track); - // The loop releases the camera before the idle gap starts. - while !camera.released() { - tokio::task::yield_now().await; - } - drop(camera); - - fixture.time.advance(idle).unwrap(); - - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - let captured = Instant::now(); - camera.push_at(surface(), captured); - let after = read_new(&mut track, &[before]).await; - fixture.assert_acquired(after, captured); - assert!( - after - before >= u64::try_from(idle.as_micros()).unwrap(), - "the {idle:?} idle gap collapsed to {}us", - after - before - ); - fixture.finish().await; - }) - .await - } - - /// A reopen onto a mode the probed level cannot carry is a new codec string, so it moves - /// the rendition to a new track: the old one ends, and a viewer who follows the catalog - /// there reopens the camera on it. - #[tokio::test] - async fn a_reopen_onto_a_new_level_follows_the_new_track() { - tokio::task::LocalSet::new() - .run_until(async { - let fixture = Fixture::start(Duration::from_secs(1), kio::clock::wall()).await; - let probed = fixture - .catalog - .snapshot() - .video - .renditions - .keys() - .next() - .unwrap() - .clone(); - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - camera.push_native(surface(), us(0)); - read(&mut track).await; - - // The device goes away and comes back in a mode four times the size. - camera.close(); - // Reopening takes the device a while, so its first frame is newer than the last. - fixture.time.advance(Duration::from_millis(40)).unwrap(); - let camera = fixture.camera_sized(640, 480); - camera.push_native(surface_sized(640, 480), us(40_000)); - // The probed track ends without the reopened camera's frames. - while let Some(read) = track.read().await.unwrap() { - assert!( - !matches!(read, moq_mux::container::Read::Frame { .. }), - "the reopened camera publishes on the probed track" - ); - } - - let mut moved = fixture.subscribe().await; - let name = fixture - .catalog - .snapshot() - .video - .renditions - .keys() - .next() - .unwrap() - .clone(); - assert_eq!(name, format!("{probed}/1")); - // Another reopen, another while. - fixture.time.advance(Duration::from_millis(40)).unwrap(); - let camera = fixture.camera_sized(640, 480); - camera.push_native(surface_sized(640, 480), us(80_000)); - let first = read(&mut moved).await; - let second = read_new(&mut moved, &[first]).await; - assert!(second > first, "the new track carries the reopened camera"); - fixture.finish().await; - }) - .await - } - - /// The wall mapping is pinned when the broadcast clock is built. A system clock stepped - /// an hour since then retimes neither the published timestamps nor the advertised mapping. - #[tokio::test] - async fn a_system_wall_adjustment_retimes_nothing() { - tokio::task::LocalSet::new() - .run_until(async { - // Whole seconds, so the advertised mapping holds it exactly. - let now = kio::clock::wall().checked_duration_since(Wall::UNIX_EPOCH).unwrap(); - let wall = Wall::UNIX_EPOCH + Duration::from_secs(now.as_secs() - 3600); - let fixture = Fixture::start(Duration::from_secs(1), wall).await; - let advertised = fixture.catalog.snapshot().media.clock; - assert_eq!(advertised, fixture.clock.wall()); - - let mut track = fixture.subscribe().await; - let camera = fixture.camera(); - let captured = Instant::now(); - camera.push_at(surface(), captured); - let published = read(&mut track).await; - - // Timestamps follow the monotonic epoch and map to walls under the pinned mapping. - fixture.assert_acquired(published, captured); - let mapped = advertised.unwrap().wall_clock(us(published)).unwrap(); - assert_eq!(mapped, wall + Duration::from_micros(published)); - assert_eq!(fixture.catalog.snapshot().media.clock, advertised); - fixture.finish().await; - }) - .await - } - } - - /// Publish fixtures: the real capture publish against sources that answer, or never do, on cue. - #[cfg(all(feature = "capture", feature = "openh264"))] - mod publish { - use std::collections::VecDeque; - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::Duration; - - use super::clock::{surface, us}; - use super::*; - use crate::capture::Synthetic; - use crate::encode::backend::probe; - - /// Hands out one queued stream per open, then stalls like a permission prompt nobody answers. - struct Streams { - queued: VecDeque, - opens: Arc, - } - - impl Streams { - fn new(queued: impl IntoIterator) -> Self { - Self { - queued: queued.into_iter().collect(), - opens: Arc::default(), - } - } - } - - impl CaptureSource for Streams { - async fn open(&mut self, _config: &capture::Config) -> Result { - self.opens.fetch_add(1, Ordering::SeqCst); - match self.queued.pop_front() { - Some(stream) => Ok(stream), - None => std::future::pending().await, - } - } - } - - fn camera() -> (Synthetic, capture::Stream) { - Synthetic::open(crate::Size::new(320, 240), crate::FrameRate::new(30, 1).unwrap()) - } - - async fn until(done: impl Fn() -> bool) { - while !done() { - tokio::task::yield_now().await; - } - } - - /// [`publish_capture`] from `source` rather than a device. - async fn publish_from( - broadcast: moq_net::broadcast::Producer, - catalog: moq_mux::catalog::Producer, - options: Capture, - source: impl CaptureSource, - ) -> Result<(), Error> { - let (_, driver) = publish_capture(broadcast, catalog, options)?; - driver.run_from(source).await - } - - async fn status(publication: &mut Publication, reached: impl Fn(&Status) -> bool) { - while !reached(publication.state().status()) { - publication.changed().await.expect("driver still alive"); - } - } - - /// What the capture is fed next, one at a time. - enum Step { - /// A frame 10ms after the previous one, waited on until the first subscriber has it. - Frame(crate::frame::Surface), - /// Another subscriber joining at the live edge, held until the end. - Join, - } - - /// A frame of seeded noise, which no encoder can predict from the one before it. - fn noise(seed: u32) -> crate::frame::Surface { - let mut state = seed.wrapping_mul(2_654_435_761) | 1; - let data = (0..320 * 240 * 3 / 2) - .map(|_| { - state ^= state << 13; - state ^= state >> 17; - state ^= state << 5; - 120 + (state as u8 >> 4) - }) - .collect(); - crate::frame::Surface::I420(crate::frame::I420 { - width: 320, - height: 240, - data, - color: None, - }) - } - - /// Drive the real capture publish through `steps`, returning the info the track advertised and the - /// payload size of every frame the first subscriber read, grouped as it read them. - async fn capture_groups(steps: Vec) -> (moq_net::track::Info, Vec>) { - use moq_mux::container::Container as _; - - let started = kio::clock::Instant::now(); - let result = local(async { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let catalog = - moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let options = Capture { - kind: encoder::Kind::Software, - // Room for noise, which rate control would otherwise skip frames of. - bitrate: Some(moq_net::bandwidth::Rate::from_mbps(4)), - ..Capture::default() - }; - // Ten nominal fps makes the default two-second GOP twenty frames, which every - // script below stays inside. The capture keeps the probe's camera. - let rate = crate::FrameRate::new(10, 1).unwrap(); - let (camera, stream) = Synthetic::open(crate::Size::new(320, 240), rate); - let run = tokio::task::spawn_local(publish_from( - broadcast.clone(), - catalog, - options, - Streams::new([stream]), - )); - let mut catalog = - moq_mux::catalog::Consumer::<()>::new(&consumer, moq_mux::catalog::CatalogFormat::Hang) - .await - .unwrap(); - let (name, mut subscriber, format) = loop { - let snapshot = catalog.next().await.unwrap().expect("a catalog"); - if let Some((name, rendition)) = snapshot.video.renditions.iter().next() { - break ( - name.clone(), - consumer.track(name).unwrap().subscribe(None).await.unwrap(), - moq_mux::catalog::hang::Container::try_from(rendition).unwrap(), - ); - } - }; - let info = subscriber.info().clone(); - - // Each picture the subscriber reads, with the group it arrived in. - let (published, mut received) = tokio::sync::mpsc::unbounded_channel(); - let collect = tokio::task::spawn_local(async move { - while let Some(mut group) = subscriber.recv_group().await.unwrap() { - while let Some(wire) = group.read_frame().await.unwrap() { - for frame in format.decode(wire).unwrap() { - // A duration marker closes the group but is not an encoded picture. - if format.end(&frame).is_none() { - published.send((group.start, frame.payload.len())).unwrap(); - } - } - } - } - }); - - let mut groups: Vec> = Vec::new(); - let mut sequence = None; - let mut joined = Vec::new(); - let mut millis = 0; - for step in steps { - match step { - Step::Frame(surface) => { - camera.push_native(surface, us(millis * 1000)); - millis += 10; - // Wait for delivery before the next step, which also keeps the - // capture's latest slot from being replaced. - let (group, len) = received.recv().await.expect("the frame reached the subscriber"); - if sequence != Some(group) { - sequence = Some(group); - groups.push(Vec::new()); - } - groups.last_mut().unwrap().push(len); - } - Step::Join => joined.push(consumer.track(&name).unwrap().subscribe(None).await.unwrap()), - } - } - assert!(!run.is_finished(), "the capture ended early: {:?}", run.await); - run.abort(); - collect.abort(); - (info, groups) - }) - .await; - assert_eq!(started.elapsed(), Duration::ZERO); - result - } - - /// A capture whose encoder always cuts advertises the refresh ratio, and a join opens a - /// group at the next frame only once the deltas outweigh it, well within 500ms of the - /// keyframe before. - #[tokio::test(start_paused = true)] - async fn a_join_over_the_refresh_ratio_forces_a_keyframe() { - let mut steps = vec![ - Step::Frame(noise(0)), - Step::Frame(surface()), - Step::Frame(surface()), - // Two flat deltas cost next to nothing against a noise keyframe. - Step::Join, - Step::Frame(surface()), - ]; - steps.extend((1..=10).map(|seed| Step::Frame(noise(seed)))); - steps.extend([Step::Join, Step::Frame(surface()), Step::Frame(surface())]); - let (info, groups) = capture_groups(steps).await; - assert_eq!(info.refresh, Some(REFRESH)); - - let lengths: Vec<_> = groups.iter().map(Vec::len).collect(); - assert_eq!(lengths, [14, 2], "only the second join cut, at the frame after it"); - let ratio = usize::try_from(REFRESH).unwrap(); - let first = &groups[0]; - assert!( - first[1..3].iter().sum::() <= ratio * first[0], - "the first join was under the ratio: {first:?}" - ); - assert!( - first[1..].iter().sum::() > ratio * first[0], - "the second join was over the ratio: {first:?}" - ); - } - - /// Refresh is advertised only when every backend the encoder may open honors a cut, since - /// the track info is fixed before the capture learns which one it got. - #[test] - fn a_backend_that_may_refuse_cuts_advertises_no_refresh() { - for (kind, refresh) in [ - (encoder::Kind::Software, Some(REFRESH)), - (encoder::Kind::Named(probe::NO_CUT.into()), None), - ] { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = - moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let options = Capture { - kind, - ..Capture::default() - }; - let track = capture_track(broadcast.clone(), &catalog, &options).unwrap(); - assert_eq!(track.info().refresh, refresh); - } - } - - /// Run `test` on a `LocalSet`, since capture streams are `!Send` on macOS. - /// - /// Off Apple, encoding waits on an OS thread, so a blocking task keeps Tokio from - /// auto-advancing the paused clock to a timer deadline while that thread answers. - async fn local(test: impl std::future::Future) -> T { - let (hold, released) = tokio::sync::oneshot::channel::<()>(); - let clock = tokio::task::spawn_blocking(move || { - let _ = released.blocking_recv(); - }); - let result = tokio::task::LocalSet::new().run_until(test).await; - drop(hold); - clock.await.unwrap(); - result - } - - /// The name of the first video rendition `consumer`'s catalog publishes. - async fn rendition_name(consumer: &moq_net::broadcast::Consumer) -> String { - let mut catalog = moq_mux::catalog::Consumer::<()>::new(consumer, moq_mux::catalog::CatalogFormat::Hang) - .await - .unwrap(); - loop { - let snapshot = catalog.next().await.unwrap().expect("a catalog"); - if let Some(name) = snapshot.video.renditions.keys().next() { - return name.clone(); - } - } - } - - /// Nobody watching, the capture still encodes, from the probe's camera rather than a - /// second open, and a subscriber arriving later reads what it wrote meanwhile. - #[tokio::test(start_paused = true)] - async fn an_unwatched_capture_keeps_its_past() { - local(async { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let catalog = - moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let mut options = Capture { - kind: encoder::Kind::Software, - ..Capture::default() - }; - // Half a frame per second rounds the default two-second GOP down to one frame, so - // every frame opens a group. - options.capture.framerate = Some(crate::FrameRate::new(1, 2).unwrap()); - let (camera, stream) = camera(); - let source = Streams::new([stream]); - let opens = source.opens.clone(); - let run = tokio::task::spawn_local(publish_from(broadcast.clone(), catalog, options, source)); - - let name = rendition_name(&consumer).await; - let mut track = consumer.track(&name).unwrap(); - // Each frame opens a group of its own, starting at the frame's pts. - camera.push_native(surface(), us(0)); - let first = track.largest().await.expect("the track is live").group; - camera.push_native(surface(), us(2_000_000)); - while track.largest().await.expect("the track is live").group < first + 2_000_000 {} - assert_eq!( - opens.load(Ordering::SeqCst), - 1, - "the capture reopened the probe's camera" - ); - assert!(!camera.released(), "the capture released its camera"); - - let mut subscriber = consumer - .track(&name) - .unwrap() - .subscribe( - moq_net::track::Subscription::default() - .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(5))), - ) - .await - .unwrap(); - // The groups start their frames' pts apart, wherever the capture's timeline begins. - let first = subscriber.recv_group().await.unwrap().expect("a cached group").start; - let second = subscriber.recv_group().await.unwrap().expect("a cached group").start; - assert_eq!(second - first, 2_000_000); - - assert!(!run.is_finished(), "the capture ended early: {:?}", run.await); - run.abort(); - }) - .await - } - - /// On demand, the probe's camera closes at once, and only a viewer opens it again. - #[tokio::test(start_paused = true)] - async fn on_demand_opens_the_camera_for_a_viewer() { - local(async { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let catalog = - moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let options = Capture { - kind: encoder::Kind::Software, - on_demand: true, - ..Capture::default() - }; - let (probed, probe) = camera(); - let (camera, stream) = camera(); - let source = Streams::new([probe, stream]); - let opens = source.opens.clone(); - let (mut publication, driver) = publish_capture(broadcast.clone(), catalog, options).unwrap(); - assert!(matches!(publication.state().status(), Status::Starting)); - let run = tokio::task::spawn_local(driver.run_from(source)); - - let name = rendition_name(&consumer).await; - until(|| probed.released()).await; - assert_eq!(opens.load(Ordering::SeqCst), 1, "only the probe opened the camera"); - status(&mut publication, |status| matches!(status, Status::Waiting)).await; - - let mut subscriber = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - camera.push_native(surface(), us(0)); - subscriber.recv_group().await.unwrap().expect("a group"); - assert_eq!(opens.load(Ordering::SeqCst), 2, "the viewer opened the camera"); - assert!(matches!(publication.state().status(), Status::Live)); - - drop(subscriber); - until(|| camera.released()).await; - status(&mut publication, |status| matches!(status, Status::Waiting)).await; - assert!(!run.is_finished(), "the capture ended early: {:?}", run.await); - let mut subscriber = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - run.abort(); - assert!(run.await.unwrap_err().is_cancelled()); - // Dropping the driver's future cancels the publication and its track. - let error = loop { - match subscriber.recv_group().await { - Ok(Some(_)) => {} - Ok(None) => panic!("the cancelled track ended cleanly"), - Err(error) => break error, - } - }; - assert!(format!("{error:?}").contains("Cancel"), "{error:?}"); - let mut last = None; - while let Some(state) = publication.changed().await { - last = Some(state); - } - assert!(matches!(last.as_ref().map(State::status), Some(Status::Cancelled))); - assert!(publication.is_finished()); - }) - .await - } - - /// Drive `run` until the bitrates the probe encoder accepted satisfy `done`, returning them. - async fn settle( - mut run: std::pin::Pin<&mut impl std::future::Future>>, - done: impl Fn(&[u64]) -> bool, - ) -> Vec { - let mut accepted = Vec::new(); - loop { - accepted.extend(probe::bitrates()); - if done(&accepted) { - return accepted; - } - tokio::select! { - res = &mut run => panic!("the capture ended: {res:?}"), - () = tokio::task::yield_now() => {} - } - } - } - - /// A watched capture follows its grant. Once nobody watches, it returns to its ceiling - /// and ignores the grant, and the next viewer's grant applies afresh. - #[tokio::test(start_paused = true)] - #[expect( - clippy::await_holding_lock, - reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" - )] - async fn an_unwatched_capture_rests_at_its_ceiling() { - use moq_net::bandwidth::Rate; - - let _probe = probe::exclusive(); - let ceiling = Rate::from_mbps(4); - let estimate = moq_net::bandwidth::Producer::new(); - estimate.set(Some(Rate::from_mbps(1))).unwrap(); - - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let track = broadcast - .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let watched = track.clone(); - let mut rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); - rendition.kind = encoder::Kind::Software; - let mut producer = Publisher::new( - track, - catalog.reserve().with_interval(rendition.effective_join()), - rendition.probe().await.unwrap(), - ) - .unwrap(); - - let (_camera, stream) = camera(); - let mut source = Streams::new([stream]); - let options = Capture { - kind: encoder::Kind::Named(probe::NAME.into()), - bitrate: Some(ceiling), - bandwidth: moq_net::bandwidth::Allocator::new(estimate.consume()), - ..Capture::default() - }; - let clock = catalog.clock(); - let state = starting(); - let mut run = std::pin::pin!(capture_loop(&mut producer, &mut source, &options, &clock, &state)); - - let viewer = watched.subscribe(None); - let followed = settle(run.as_mut(), |set| !set.is_empty()).await; - assert!(followed.iter().all(|&bps| bps < ceiling.as_bps()), "{followed:?}"); - - drop(viewer); - let rested = settle(run.as_mut(), |set| set.contains(&ceiling.as_bps())).await; - assert_eq!(rested, [ceiling.as_bps()]); - - // Unwatched, a grant change retunes nothing, however long it holds. - estimate.set(Some(Rate::from_kbps(500))).unwrap(); - tokio::select! { - res = &mut run => panic!("the capture ended: {res:?}"), - () = kio::clock::sleep(Duration::from_secs(60)) => {} - } - assert_eq!(probe::bitrates(), Vec::::new()); - - let _viewer = watched.subscribe(None); - let resumed = settle(run.as_mut(), |set| !set.is_empty()).await; - assert!(resumed.iter().all(|&bps| bps <= 500_000), "{resumed:?}"); - } - - /// A backend that cannot cut still captures, leaving every group to its cadence. - #[tokio::test] - #[expect( - clippy::await_holding_lock, - reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" - )] - async fn a_backend_that_cannot_cut_still_captures() { - let _probe = probe::exclusive(); - - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let track = broadcast - .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let _subscriber = track.subscribe(None); - let mut rendition = Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); - rendition.kind = encoder::Kind::Software; - let mut producer = Publisher::new( - track, - catalog.reserve().with_interval(rendition.effective_join()), - rendition.probe().await.unwrap(), - ) - .unwrap(); - - let (camera, stream) = camera(); - let mut source = Streams::new([stream]); - let options = Capture { - kind: encoder::Kind::Named(probe::NO_CUT.into()), - ..Capture::default() - }; - let clock = catalog.clock(); - let state = starting(); - let mut run = std::pin::pin!(capture_loop(&mut producer, &mut source, &options, &clock, &state)); - - // The probe holds its first frame back, so this one reaches the codec but not the track. - camera.push_native(surface(), us(0)); - tokio::select! { - res = &mut run => panic!("the capture ended: {res:?}"), - () = until(|| probe::take().iter().any(|(event, _)| *event == "encode")) => {} - } - } - } - - /// Rate control against the probe backend, which logs every bitrate it accepts. - #[cfg(feature = "capture")] - mod rate_control { - use std::time::Duration; - - use moq_net::bandwidth::Rate; - - use super::*; - use crate::encode::backend::probe; - - /// After congestion clears, the encoder climbs back to its ceiling although the grant - /// never changes again, and a failed step is retried rather than waiting for one. - #[tokio::test(start_paused = true)] - #[expect( - clippy::await_holding_lock, - reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" - )] - async fn a_quiet_grant_still_recovers_the_ceiling() { - let _probe = probe::exclusive(); - let ceiling = Rate::from_mbps(4); - let mut config = encoder::Config::new(320, 240, crate::FrameRate::new(30, 1).unwrap()); - config.kind = encoder::Kind::Named(probe::NAME.into()); - config.bitrate = Some(ceiling); - let mut sink = Sink::open(&config).await.unwrap(); - - let estimate = moq_net::bandwidth::Producer::new(); - let mut rate = Some(rate::Grant::new(estimate.consume(), rate::Policy::new(ceiling))); - - estimate.set(Some(Rate::from_mbps(1))).unwrap(); - let next = next_target(&mut rate).await; - apply_target(&mut sink, &mut rate, next).await; - - // The last grant change: the link recovers a second later. - tokio::time::advance(Duration::from_secs(1)).await; - estimate.set(Some(ceiling)).unwrap(); - probe::refuse_once(ceiling.as_bps()); - - // A minute without a wake is far past anything the ramp still needs. - while let Ok(next) = kio::clock::timeout(Duration::from_secs(60), next_target(&mut rate)).await { - apply_target(&mut sink, &mut rate, next).await; - } - let bitrates = probe::bitrates(); - assert_eq!(bitrates.first(), Some(&1_000_000), "{bitrates:?}"); - assert_eq!( - bitrates.last(), - Some(&ceiling.as_bps()), - "the ramp stalled: {bitrates:?}" - ); - assert!( - probe::take() - .iter() - .filter(|(event, _)| *event == "set_bitrate") - .count() > bitrates.len() - ); - assert!(rate.is_some(), "a transient failure keeps rate control running"); - } - } } /// A finished [`Publisher`] keeps its track and rendition published, but not its splitter. diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 87442150ccc..3e4922c7e79 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -31,9 +31,11 @@ //! startup so they can be exact), and the camera stays open and encodes //! whether or not anyone is watching, unless `encode::Capture::on_demand` //! limits it to while a subscriber is. Requires the `capture` feature. -//! - [`encode::Encoder`] encodes [`Frame`]s you supply (from capture, a -//! decoder, or your own pixels via [`Surface::rgba`]) and -//! [`encode::Publisher`] publishes the results. +//! - [`encode::Producer`] encodes [`Frame`]s you supply (from capture, a +//! decoder, or your own pixels via [`Surface::rgba`]) and publishes them, +//! following the connection's bandwidth grant between frames. +//! - [`encode::Encoder`] encodes without publishing, and +//! [`encode::Publisher`] publishes frames you encoded yourself. //! - [`decode`] subscribes to an H.264, H.265, or AV1 track and decodes it to //! raw frames with a native backend (VideoToolbox on macOS and iOS, Media Foundation / //! DXVA on Windows, NVDEC, VAAPI, or an ARM SoC's V4L2 M2M decoder on Linux, From 7c2963e32517d7204c4ad4539690d2f6527a732c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 09:47:59 -0800 Subject: [PATCH 04/14] refactor: moq-ffi, moq-c, and moq-boy publish video through encode::Producer Their encode loops and grant followers go: the bindings' producers wrap moq_video::encode::Producer, which follows the grant between frames, and the binding's lower rendition becomes its Options::lower. The tests that drove the deleted followers move into moq-video against the lifted type. Co-Authored-By: Claude Opus 5.5 --- dart/moq_ffi/lib/src/moq.dart | 6 +- quest/m2/README.md | 3 +- quest/m2/finish-sweep.md | 7 +- quest/m2/native-media-config.md | 1 - quest/m2/unused-api/media.md | 1 - quest/m2/video-producer.md | 74 -- rs/moq-boy/src/video.rs | 91 +- rs/moq-c/src/test.rs | 66 -- rs/moq-c/src/video.rs | 924 ++--------------- rs/moq-ffi/src/bandwidth.rs | 7 +- rs/moq-ffi/src/test.rs | 47 +- rs/moq-ffi/src/video.rs | 1166 ++-------------------- rs/moq-video/src/encode/backend/probe.rs | 12 + rs/moq-video/src/encode/producer.rs | 144 +++ 14 files changed, 359 insertions(+), 2190 deletions(-) delete mode 100644 quest/m2/video-producer.md diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 74bbb704978..b0bdc8fc0f4 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -25824,10 +25824,10 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqvideoproducer_reservation() != 1052) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqvideoproducer_set_bitrate() != 19081) { + if (uniffi_moq_ffi_checksum_method_moqvideoproducer_set_bitrate() != 5725) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqvideoproducer_write() != 17427) { + if (uniffi_moq_ffi_checksum_method_moqvideoproducer_write() != 39148) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqaudiocodec_aac() != 62161) { @@ -25947,7 +25947,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_constructor_moqvideoencoder_new() != 22531) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_constructor_moqvideoproducer_new() != 51291) { + if (uniffi_moq_ffi_checksum_constructor_moqvideoproducer_new() != 47759) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } } diff --git a/quest/m2/README.md b/quest/m2/README.md index 42df4d4cfa6..aa7d0baf5b7 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -61,8 +61,7 @@ removal, a rename, or a wire, catalog, or format change that is free until [declared ends](/quest/m2/time-addressing/consumers.md), back from m3; [finish](/quest/m2/finish-sweep.md), [TS outcomes](/quest/m2/ts-typed-outcomes.md), and [moq-json writes](/quest/m2/json-write-at.md). -- **Native media and gateways.** [The video producer](/quest/m2/video-producer.md), - [the hole wait](/quest/m2/delay-hole-wait.md), +- **Native media and gateways.** [The hole wait](/quest/m2/delay-hole-wait.md), [device and cursor](/quest/m2/native-media-config.md), [sink names](/quest/m2/audio-sink-names.md), [moq-vaapi's exports](/quest/m2/vaapi-exports.md), diff --git a/quest/m2/finish-sweep.md b/quest/m2/finish-sweep.md index e5befe53efd..61f0e3340b1 100644 --- a/quest/m2/finish-sweep.md +++ b/quest/m2/finish-sweep.md @@ -54,9 +54,9 @@ For the PR: job. - `moq_video::encode::Producer::finish(self) -> Result`, with `Finished::abort`, matching the audio producer line for line. This - type is being renamed by - [the frame-taking video producer](/quest/m2/video-producer.md); whichever - lands second merges main and keeps both changes. + type is now `encode::Publisher`, and the frame-taking `encode::Producer` + wraps it with a provisional `Finished` of the same shape (`demand`, + `abort`); this quest's `Finished` replaces that one. - Callers hold these behind locks in moq-ffi and moq-c (`rs/moq-ffi/src/{video,media,audio}.rs`, `rs/moq-c/src/{video,publish}.rs`): move the producer out of its slot on finish and store the `Finished`, as @@ -73,5 +73,4 @@ importers, `moq_mux::import::{Track, TrackStream}`, and ## Related -- [The frame-taking video producer](/quest/m2/video-producer.md) - renames the video type this changes - [TS import reports typed outcomes](/quest/m2/ts-typed-outcomes.md) - changes the TS importers' `finish` error type diff --git a/quest/m2/native-media-config.md b/quest/m2/native-media-config.md index d94378de62d..1314a2cbf2b 100644 --- a/quest/m2/native-media-config.md +++ b/quest/m2/native-media-config.md @@ -79,6 +79,5 @@ configs, `capture::Config::cursor`). CLI: none. Wire: none. ## Related -- [moq-video has a producer that takes frames](/quest/m2/video-producer.md) - changes `encode::Capture` and its driver in the same crate - [GPU health](/quest/m3/gpu-health.md) - the render-node identity the selector reuses - [The cursor apart from the frame](/quest/m4/capture-cursor.md) - adds the third cursor variant diff --git a/quest/m2/unused-api/media.md b/quest/m2/unused-api/media.md index d91a25cad9f..8ae561a811c 100644 --- a/quest/m2/unused-api/media.md +++ b/quest/m2/unused-api/media.md @@ -98,6 +98,5 @@ Public API: moq-mux, moq-video, moq-transcode, hang, `@moq/hang`, and ## Related -- [moq-video has a producer that takes frames](/quest/m2/video-producer.md) - reworks the encode module these items sit in - [Client health in use](/quest/m3/stats/health-consumers.md) - publishes `Health` once something reads it - [The OBS adapter](/quest/m3/obs-moq-video/adapter.md) - the planned consumer of `Preset` and `Applied` diff --git a/quest/m2/video-producer.md b/quest/m2/video-producer.md deleted file mode 100644 index 35a7d2ee21c..00000000000 --- a/quest/m2/video-producer.md +++ /dev/null @@ -1,74 +0,0 @@ -# [L] moq-video has a producer that takes frames - -## Goal - -`moq_video::encode::Producer` takes raw frames, owns its encoder, and follows -the uplink grant, as `moq_audio::encode::Producer` does for PCM. The type -that publishes already encoded access units keeps working under a name that -says what it takes (recommended: `encode::Publisher`). moq-boy, moq-ffi, and -moq-c publish video through the new producer, so the probe, catalog -reservation, encode loop, and grant following exist once, in moq-video. - -## Plan - -Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): - -- ✅ Rename today's type and build the frame-taking producer before the cut - (rejected: rename only and add the producer after the cut under the freed - name; keep both names and add the frame-taking type later under a third). - The maintainer: "Rename and build it now." A name that means two layers in - sibling crates freezes at the cut, and the freed name is only worth - reserving if the type that earns it ships with it. - -Facts found by the audit: - -- `moq_audio::encode::Producer` takes PCM (`write`, - `rs/moq-audio/src/encode/producer.rs:407`), owns its encoder, and follows - the grant through `Options::bandwidth` (`:43`). - `moq_video::encode::Producer` takes encoded access units (`publish`, - `rs/moq-video/src/encode/producer.rs:163`) and knows no encoder or grant. - Same name, same module path, different layer. -- Every caller with its own frames assembles the video path by hand: probe - the config, reserve a catalog slot with the effective join, build the - producer, run the encode, receive, publish loop. `rs/moq-boy/src/video.rs:59-65` - is the shortest witness; `rs/moq-ffi/src/video.rs` (2,129 lines) and - `rs/moq-c/src/video.rs` (1,820 lines) carry the rest. -- Grant following with rung disable, the part that is easy to get wrong, - exists three times: private in the capture driver - (`rs/moq-video/src/encode/producer.rs:718`, `:840`, `:940`), in moq-ffi - (`rs/moq-ffi/src/video.rs:370`, `:524`), and in moq-c - (`rs/moq-c/src/video.rs:286`, `:319`, `:363`). -- No quest owns it: [the grant getter](/quest/m3/video-publish-grant.md) - only adds a getter to the capture handle, and - [the mux wrap](/quest/m5/net-wrap/mux.md) only makes the producer generic. - -For the PR: - -- Rename first, as its own commit: mechanical, about 19 references outside - moq-audio (`grep -rn 'encode::Producer' rs`). The rename also reaches the - docs (`doc/lib/rs/moq-video.md`) and `rs/moq-video/README.md`. -- Lift the frame-taking producer out of the capture driver - (`publish_capture`'s `Driver`), which then runs on it, so the capture path - and a caller's own frames share one encode loop and one grant follower. - Shape it after audio: an `Options` with `bandwidth`, `write(&Frame)`, and - `finish(self) -> Result` from the start (the - [finish sweep](/quest/m2/finish-sweep.md) converts the renamed type). -- Move moq-boy, moq-ffi, and moq-c onto it and delete their copies of the - grant follower. A binding option the lifted type cannot express (moq-ffi's - `lower` rendition for the simulcast ladder, `rs/moq-ffi/src/video.rs:370`) - becomes an option of the producer, not a reason to keep a copy. -- If lifting the driver would change a backend's behaviour in a way the cut - cannot absorb, stop and say so on the PR instead of shipping the rename - alone: the maintainer rejected that end state. -- Tests: the capture driver's grant tests run against the lifted type; a - frame-taking producer follows a falling and recovering grant and disables - and re-enables its rendition; `just test interop --all` for moq-ffi. - -Public API: moq-video breaks (`encode::Producer` changes meaning; the encoded -publisher is renamed). moq-ffi and moq-c keep their names. Wire: none. - -## Related - -- [Terminal operations consume](/quest/m2/finish-sweep.md) - gives the renamed type `finish(self)`; whichever lands second rebases -- [One device selector](/quest/m2/native-media-config.md) - changes `encode::Config` in the same crate -- [A published capture exposes its grant](/quest/m3/video-publish-grant.md) - adds a getter to the capture handle this runs on diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index a6020632321..3fbf367772e 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -2,8 +2,8 @@ //! //! Runs on a dedicated thread so the emulator's frame loop never blocks on the //! encoder. Frames arrive on a bounded channel; if the encoder falls behind, -//! frames are dropped to keep latency low. moq-video does the RGBA -> H.264 -//! encode and the avc3 publish; this module keeps moq-boy's threading, +//! frames are dropped to keep latency low. moq-video's encode producer does the +//! RGBA -> H.264 encode and the avc3 publish; this module keeps moq-boy's threading, //! frame-dropping, force-keyframe and timing-stats behavior. use kio::clock::Instant; @@ -43,8 +43,8 @@ enum EncoderMsg { } impl VideoEncoder { - /// Publish the catalog rendition, then hand the encoder to a worker thread that only runs while - /// someone is watching. The returned handle feeds it frames; the emulator never blocks on it. + /// Publish the catalog rendition, then hand the encoder to a worker thread. The returned + /// handle feeds it frames; the emulator never blocks on it. pub async fn spawn(broadcast: moq_net::broadcast::Producer, catalog: moq_mux::catalog::Producer) -> Self { let (tx, rx) = tokio::sync::mpsc::channel(4); @@ -54,24 +54,24 @@ impl VideoEncoder { moq_video::encode::Config::new(WIDTH, HEIGHT, moq_video::FrameRate::new(FRAMERATE, 1).unwrap()); config.kind = moq_video::encode::Kind::Software; - // Probed from a throwaway encoder, before the emulator has run a frame, so a - // viewer can discover (and unpause) a moq-boy that has published nothing. - let rendition = config.probe().await.expect("failed to probe the H.264 encoder"); - let reserved = catalog.reserve().with_interval(config.effective_join()); - let track = broadcast - .unique_track(".avc3", reserved.track_info(hang::catalog::PRIORITY.video)) - .expect("failed to create the avc3 track"); - let producer = - moq_video::encode::Publisher::new(track, reserved, rendition).expect("failed to create avc3 producer"); + // The rendition is probed before the emulator has run a frame, so a viewer can discover + // (and unpause) a moq-boy that has published nothing. + let options = moq_video::encode::Options::new(config); + let producer = moq_video::encode::Producer::new(&broadcast, catalog, options) + .await + .expect("failed to open the H.264 encoder"); let demand = producer.demand(); let force_keyframe = Arc::new(AtomicBool::new(false)); let encode_duration = Arc::new(AtomicU64::new(0)); let fk = force_keyframe.clone(); let ed = encode_duration.clone(); + // The thread drives the producer's futures through this runtime, whose timers its rate + // control uses. + let runtime = tokio::runtime::Handle::current(); let thread = std::thread::Builder::new() .name("video-encoder".into()) - .spawn(move || encoder_thread(rx, producer, config, fk, ed)) + .spawn(move || runtime.block_on(encoder_thread(rx, producer, fk, ed))) .expect("failed to spawn video encoder thread"); Self { @@ -93,7 +93,8 @@ impl VideoEncoder { /// Publish a marker group marking a pause, so the PTS jump on resume reads as a break /// rather than one very long frame, and a viewer arriving mid-pause isn't served the - /// pre-pause group as if it were live. + /// pre-pause group as if it were live. The encoder closes until the next frame, so a + /// paused emulator holds none. /// /// Blocks rather than dropping on a full channel: a dropped frame costs one frame, a /// dropped marker silently reinstates the bug it exists to prevent. @@ -115,39 +116,26 @@ impl VideoEncoder { } } -fn encoder_thread( +/// Encode each frame on its own thread, so a slow encode never stalls the emulator. +/// +/// Driven by `Handle::block_on` on a thread of its own, so the producer's awaits block only it. +async fn encoder_thread( mut rx: tokio::sync::mpsc::Receiver, - mut producer: moq_video::encode::Publisher, - config: moq_video::encode::Config, + mut producer: moq_video::encode::Producer, force_keyframe: Arc, encode_duration: Arc, ) { - let mut encoder: Option = None; - - while let Some(msg) = rx.blocking_recv() { - let msg = match msg { + while let Some(msg) = rx.recv().await { + let (rgba, ts) = match msg { EncoderMsg::Frame { rgba, ts } => (rgba, ts), EncoderMsg::Discontinuity => { - if let Err(e) = producer.discontinuity() { - tracing::warn!(error = %e, "failed to mark the video discontinuity"); + // Marked even when the drain fails, which only loses the last frames. + if let Err(e) = producer.discontinuity().await { + tracing::warn!(error = %e, "failed to drain the video encoder at the discontinuity"); } continue; } }; - let (rgba, ts) = msg; - let enc = match encoder.as_mut() { - Some(enc) => enc, - None => { - // Opened on the first frame, so a paused emulator holds no encoder. - match moq_video::encode::Encoder::new(&config) { - Ok(enc) => encoder.insert(enc), - Err(e) => { - tracing::error!(error = %e, "H.264 encoder init failed"); - return; - } - } - } - }; // Asked for before the conversion below, which can fail: the encoder holds the // request until a frame actually arrives, so dropping this frame delays the @@ -155,7 +143,7 @@ fn encoder_thread( // otherwise the encoder's own GOP keys the stream. A refusal is terminal: // only a backend that cannot cut at all gives one, and this never selects one. if force_keyframe.swap(false, Ordering::AcqRel) - && let Err(e) = enc.cut() + && let Err(e) = producer.cut().await { tracing::error!(error = %e, "H.264 encoder cannot cut a group; stopping encoder"); return; @@ -169,26 +157,11 @@ fn encoder_thread( continue; } }; - // A single bad frame is tolerable; keep going. - if let Err(e) = enc.encode(&moq_video::Frame::new(surface, ts)) { - tracing::error!(error = %e, "H.264 encode error"); - } - loop { - match enc.receive() { - Ok(Some(encoded)) => { - if let Err(e) = producer.publish(&encoded) { - // Publish only fails once the track/broadcast is gone, which - // is terminal -- stop rather than flooding logs every frame. - tracing::error!(error = %e, "video publish failed; stopping encoder"); - return; - } - } - Ok(None) => break, - Err(e) => { - tracing::error!(error = %e, "H.264 encode error"); - break; - } - } + // A frame of the configured size only fails to encode or publish once the codec or the + // track is gone, which is terminal: stop rather than flooding logs every frame. + if let Err(e) = producer.write(moq_video::Frame::new(surface, ts)).await { + tracing::error!(error = %e, "video encode failed; stopping encoder"); + return; } encode_duration.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); } diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index b790ff6215f..2f5f19bf420 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -8023,72 +8023,6 @@ fn encode_video_rung_needs_a_grant_and_a_band() { assert_eq!(moq_origin_close(origin), 0); } -/// A rung watches its lower rendition's own ceiling, which wakes it when that moves or -/// ends, so it notices even when the lower rendition reserves nothing it shares. -#[test] -fn encode_video_ceiling_wakes_a_rung_above() { - struct Flag(std::sync::atomic::AtomicBool); - impl std::task::Wake for Flag { - fn wake(self: std::sync::Arc) { - self.0.store(true, std::sync::atomic::Ordering::SeqCst); - } - } - - let origin = id(moq_origin_create()); - let broadcast = publish_broadcast(origin, b"rung"); - let input = moq_video_encoder_input { - format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA as u32, - width: 320, - height: 240, - framerate: moq_video_frame_rate { - numerator: 30, - denominator: 1, - }, - }; - let output = moq_video_encoder_output { - codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, - bitrate: 500_000, - join_us: 0, - kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, - encoder: std::ptr::null(), - encoder_len: 0, - }; - // No allocator, so nothing it does moves a reservation a rung would share. - let low = id(unsafe { moq_encode_video(broadcast, &input, &output, std::ptr::null()) }); - let ceiling = State::lock() - .video - .producer(Id::try_from(low).unwrap()) - .unwrap() - .lock() - .as_ref() - .unwrap() - .ceiling - .consume(); - - let flag = std::sync::Arc::new(Flag(std::sync::atomic::AtomicBool::new(false))); - let waiter = kio::Waiter::new(std::task::Waker::from(flag.clone())); - let arm = || ceiling.poll(&waiter, |_| std::task::Poll::<()>::Pending).is_pending(); - assert!(arm()); - - assert_eq!(moq_encode_video_bitrate(low, 400_000), 0); - assert!( - flag.0.swap(false, std::sync::atomic::Ordering::SeqCst), - "a moved ceiling wakes the rung" - ); - assert_eq!(*ceiling.read(), moq_net::bandwidth::Rate::from_bps(400_000)); - assert!(arm()); - - assert_eq!(moq_encode_video_finish(low), 0); - assert!( - flag.0.load(std::sync::atomic::Ordering::SeqCst), - "an ended rendition wakes the rung" - ); - assert!(!arm(), "the ceiling closes with the rendition"); - - assert_eq!(moq_publish_close(broadcast), 0); - assert_eq!(moq_origin_close(origin), 0); -} - /// Listen on an ephemeral loopback port with a generated certificate. fn listen(cb: &Callback) -> (u32, String, String) { let bind = "127.0.0.1:0"; diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index 96a0444cde0..dd97bddfb19 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -267,184 +267,28 @@ pub(crate) fn block_on(future: impl std::future::Future) -> T { pollster::block_on(future) } -fn video_ceiling( - config: &moq_video::encode::Config, - rendition: &hang::catalog::VideoConfig, -) -> moq_net::bandwidth::Rate { - config - .bitrate - .or_else(|| rendition.bitrate.map(moq_net::bandwidth::Rate::from_bps)) - .unwrap_or_else(|| { - moq_net::bandwidth::Rate::from_bps( - (config.size().pixels() as f64 * config.framerate.as_f64() * 0.07) as u64, - ) - }) -} - -/// The policy a rendition follows: its manual `ceiling`, above the `lower` rendition's -/// current ceiling, if any. -fn video_policy(ceiling: moq_net::bandwidth::Rate, lower: Option) -> moq_mux::rate::Policy { - let mut policy = moq_mux::rate::Policy::new(ceiling); - policy.lower = lower; - policy -} - -/// The lower rendition's current ceiling, or `None` once it finished, which leaves -/// this one nothing to fall back to, so it never disables. Arms `waiter` for either -/// change. -fn lower_ceiling( - lower: &Option>, - waiter: &kio::Waiter, -) -> Option { - let lower = lower.as_ref()?; - match lower.poll(waiter, |_| std::task::Poll::<()>::Pending) { - std::task::Poll::Pending => Some(*lower.read()), - // The closure never completes, so this is the channel closing. - std::task::Poll::Ready(_) => None, - } -} - -/// Feed each grant through the same policy moq-video uses, and retune the live -/// encoder, disabling and resuming a rung. Retires on -/// [`BitrateUnsupported`](moq_video::Error::BitrateUnsupported). -async fn follow_reservation( - inner: Shared, - lower: Option>, - consumer: moq_net::bandwidth::Consumer, - reservation: std::sync::Weak, - ceiling: moq_net::bandwidth::Rate, - target: moq_mux::rate::Target, -) { - let noop = kio::Waiter::noop(); - let mut follower = moq_mux::rate::Grant::new(consumer, video_policy(ceiling, lower_ceiling(&lower, &noop))); - follower.assume(target); - loop { - // Both ceilings are read on every wake, before the follower filters a grant against - // them. This one's `moq_encode_video_bitrate` moves its reservation after moving its - // ceiling, which wakes this poll; the lower one's ceiling wakes it directly, wherever - // that rendition reserves. The lower one is read first, so no two encoder locks are - // ever held at once. - let next = kio::wait(|waiter| { - let below = lower_ceiling(&lower, waiter); - let guard = inner.lock(); - let Some(producer) = guard.as_ref().filter(|producer| producer.follows(&reservation)) else { - return std::task::Poll::Ready(None); - }; - follower.set_policy(video_policy(*producer.ceiling.read(), below)); - follower.poll_next(waiter).map(Some) - }) - .await; - let Some(Ok(target)) = next else { - return; - }; - let below = lower_ceiling(&lower, &noop); - let mut guard = inner.lock(); - // Checked again under the lock it applies under: a move or finish may have replaced this - // claim while the target was on its way. - let Some(producer) = guard.as_mut().filter(|producer| producer.follows(&reservation)) else { - return; - }; - let policy = video_policy(*producer.ceiling.read(), below); - let before = producer.producer.demand(); - let retuned = retune(&mut follower, policy, target, |target| producer.apply(target)); - // A disable drains the encoder into the track, and output with a new decoder identity - // moves the rendition, so the claim moves with it and this task ends. - if producer.follow_track(before.name(), &inner) || retuned.is_err() { - return; - } - } -} - -/// Apply the follower's next `target` through `apply`, unless the `policy` moved since it -/// was asked for: then follow under the new policy, whose manual ceiling -/// `moq_encode_video_bitrate` already applied, instead. Returns the target applied, or -/// `Err` with the name of an encoder that cannot follow at all. -fn retune( - follower: &mut moq_mux::rate::Grant, - policy: moq_mux::rate::Policy, - target: moq_mux::rate::Target, - apply: impl FnOnce(moq_mux::rate::Target) -> Result<(), moq_video::Error>, -) -> Result, &'static str> { - if *follower.policy() != policy { - // The target was never applied, so take it back first: a policy that keeps the - // follower's state (a moved lower rendition) would otherwise count it applied. - follower.failed(); - follower.set_policy(policy); - return Ok(None); - } - match apply(target) { - Ok(()) => { - tracing::debug!(?target, "adjusted encoder bitrate"); - Ok(Some(target)) - } - Err(moq_video::Error::BitrateUnsupported(name)) => { - tracing::warn!(encoder = name, "encoder cannot follow the bandwidth estimate"); - Err(name) - } - // The follower takes the target back and retries it, so a quiet grant still - // recovers. - Err(err) => { - tracing::warn!(error = %err, ?target, "failed to adjust encoder bitrate"); - follower.failed(); - Ok(None) - } - } -} - -/// An encoder paired with the track publishing its output, plus the pixel format -/// its caller feeds it (fixed at publish time, so a frame carries only pixels and -/// a timestamp). +/// A [`Producer`](moq_video::encode::Producer) plus the pixel format its caller feeds it (fixed at +/// publish time, so a frame carries only pixels and a timestamp). /// -/// The encoder is a [`Sink`](moq_video::encode::Sink) rather than a bare -/// `Encoder` because a C caller drives a handle from whichever thread it likes, -/// so a bare `Encoder` would be built on one thread and dropped on another, -/// unbalancing the per-thread COM apartment the Windows backend opens. The sink -/// owns the thread instead, so every caller is welcome. +/// The producer's encoder is a [`Sink`](moq_video::encode::Sink), which owns the thread it runs +/// on: a C caller drives a handle from whichever thread it likes, so a bare `Encoder` would be +/// built on one thread and dropped on another, unbalancing the per-thread COM apartment the +/// Windows backend opens. pub(crate) struct VideoEncoder { - /// `None` while the rendition is disabled for lack of bandwidth. - encoder: Option, - producer: moq_video::encode::Publisher, - /// The encoder settings, to reopen at [`ceiling`](Self::ceiling) when a disabled - /// rendition resumes. - config: moq_video::encode::Config, - /// The track priority this rendition was published at; a higher rung takes one below. - priority: i8, - /// The next lower rendition's ceiling, set when this one is a simulcast rung. A - /// consumer, so a rung never keeps the rendition below it publishing. - lower: Option>, + producer: moq_video::encode::Producer, format: moq_video_pixel_format, /// The encoded resolution, from the publish config. Frames carry only pixels, /// so this is what says how to read them. size: moq_video::Size, - /// The bandwidth claim on the current track and its follower, when published against an - /// allocator. Beside the encoder, so a move to a new track, a finish, and an abort change - /// all three together. - follow: Option, - /// What the encoder currently produces, which a follower started on a moved track resumes - /// from rather than assuming the ceiling. - target: moq_mux::rate::Target, - /// The manual ceiling [`moq_encode_video_bitrate`] last applied, which caps every - /// grant. Behind the encoder's lock, so the follower never reads an old, higher - /// one and retunes over a newer cap. A channel, so a rung above this rendition - /// wakes when it moves and when this rendition ends. - pub(crate) ceiling: kio::Producer, + /// Whether it was published against an allocator, so [`moq_encode_video_reservation`] + /// hands out its claim. + reserved: bool, children: crate::publish::Children, } -/// A rendition's bandwidth claim on its current track, and the task retuning the encoder to -/// the claim's grant. -struct Following { - /// Reserves against the track a decoder identity change moves the rendition to. - allocator: moq_net::bandwidth::Allocator, - reservation: Arc, - /// Dropped with this claim, which stops the follower. - _stop: oneshot::Sender<()>, - follower: crate::ffi::Background, -} - impl crate::publish::Abort for Shared { fn abort(self, err: moq_net::Error) { - // Dropping the claim stops the rate follower; the encoder closes with it. + // The encoder and the bandwidth claim close with the producer. if let Some(encoder) = self.take() { encoder.producer.abort(err); } @@ -532,101 +376,9 @@ fn surface(_frame: &moq_video::Frame) -> *const c_void { std::ptr::null() } -/// End a video track, given the result of draining its encoder into it. -/// -/// A clean finish is a promise that the track holds everything the publisher -/// produced, so a lost tail has to end the track as an abort instead. Finishing -/// anyway would leave a truncated stream indistinguishable from a complete one, -/// and only the local caller would ever learn otherwise. -fn finalize( - mut producer: moq_video::encode::Publisher, - drained: Result<(), moq_video::Error>, - children: &crate::publish::Children, - child: crate::publish::Child, -) -> Result<(), Error> { - match drained { - Ok(()) => { - producer.finish()?; - // The broadcast keeps the finished track, so its finish still holds it. - children.keep(child, producer); - Ok(()) - } - Err(err) => { - children.forget(child); - producer.abort(moq_net::Error::Transport(err.to_string())); - Err(err.into()) - } - } -} - impl VideoEncoder { - /// Reserve the ceiling against the current track and follow the grant, replacing any earlier - /// claim, whose dropped stop ends its follower. `shared` is the handle holding this encoder, - /// whose lock the caller holds, so the claim and its follower change together. - fn follow_grant(&mut self, shared: &Shared, allocator: moq_net::bandwidth::Allocator) { - let ceiling = *self.ceiling.read(); - let reservation = Arc::new(allocator.reserve(&self.producer.demand(), ceiling)); - let (stop, stopped) = oneshot::channel(); - let follower = crate::ffi::Background::new({ - let shared = shared.clone(); - let lower = self.lower.clone(); - let consumer = reservation.consume(); - let identity = Arc::downgrade(&reservation); - let target = self.target; - async move { - tokio::select! { - biased; - _ = stopped => {} - _ = follow_reservation(shared, lower, consumer, identity, ceiling, target) => {} - } - Ok(()) - } - }); - self.follow = Some(Following { - allocator, - reservation, - _stop: stop, - follower, - }); - } - - /// Move the claim to the track the rendition is on now, if publishing moved it off the one - /// named `before`, such as on a decoder identity change. Returns whether it moved. - fn follow_track(&mut self, before: &str, shared: &Shared) -> bool { - if self.producer.demand().name() == before { - return false; - } - let Some(follow) = &self.follow else { - return false; - }; - self.follow_grant(shared, follow.allocator.clone()); - true - } - - /// Whether this rendition still follows `reservation`, rather than having moved off the - /// track it was reserved on or ended. The [`Weak`](std::sync::Weak) holds the allocation, so - /// a newer reservation never takes its address. - fn follows(&self, reservation: &std::sync::Weak) -> bool { - self.follow - .as_ref() - .is_some_and(|follow| std::ptr::eq(Arc::as_ptr(&follow.reservation), reservation.as_ptr())) - } - - /// A rate follower's panic, which ends adaptation but not the track. - fn check(&self) -> Result<(), Error> { - self.follow.as_ref().map_or(Ok(()), |follow| follow.follower.check()) - } - - /// Encode and publish one picture. A picture that moves the rendition to a new track moves - /// its claim there too; `shared` is the handle holding this encoder, whose lock the caller - /// holds. - fn publish_frame( - &mut self, - shared: &Shared, - timestamp: moq_net::Timestamp, - data: &[u8], - ) -> Result<(), Error> { - self.check()?; + /// Encode and publish one picture, following the bandwidth grant first. + fn publish_frame(&mut self, timestamp: moq_net::Timestamp, data: &[u8]) -> Result<(), Error> { // A buffer that isn't one picture at the configured size is rejected here, // by the surface constructors, rather than reinterpreted. let size = self.size; @@ -636,239 +388,75 @@ impl VideoEncoder { } moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA => moq_video::Surface::rgba(data, size)?, }; - - // Disabled: nothing is encoded until the grant resumes the rendition. - let Some(encoder) = self.encoder.as_mut() else { - return Ok(()); - }; let frame = moq_video::Frame::new(surface, timestamp); - // A backend that pipelines readies an earlier frame's output, so `publish` - // drains zero or more access units rather than one per call. - block_on(encoder.encode(frame))?; - let before = self.producer.demand(); - let published = publish(encoder, &mut self.producer); - // A frame whose decoder identity changed, such as a resume at a ceiling past the declared - // level, moved the rendition to a newly named track, whose demand the claim has to watch. - // The claim moves even when publishing on the new track failed, so it never stays on the - // ended one. - self.follow_track(before.name(), shared); - Ok(published?) + Ok(block_on(self.producer.write(frame))?) } fn publish_cut(&mut self) -> Result<(), Error> { - self.check()?; - // A keyframe is what a cut is on the wire: the importer closes the open - // group and starts a new one at it. A disabled rung resumes on a keyframe anyway. - match self.encoder.as_mut() { - Some(encoder) => Ok(block_on(encoder.cut())?), - None => Ok(()), - } + Ok(block_on(self.producer.cut())?) } fn publish_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { - self.check()?; let bitrate = moq_net::bandwidth::Rate::from_bps(bitrate); - // A disabled rung has no encoder to retune; it reopens at the new ceiling. - if let Some(encoder) = self.encoder.as_mut() { - block_on(encoder.set_bitrate(bitrate))?; - self.target = moq_mux::rate::Target::Enabled(bitrate); - } - let Ok(mut ceiling) = self.ceiling.write() else { - unreachable!("only dropping the encoder closes its ceiling") - }; - *ceiling = bitrate; - drop(ceiling); - if let Some(follow) = &self.follow { - follow.reservation.update(bitrate); - } - Ok(()) + Ok(block_on(self.producer.set_bitrate(bitrate))?) } - /// Apply a rate-control target, leaving the rendition as it was on an error. + /// Drain and finish the encoder behind handle `id`. /// - /// A disable drains the encoder into the track and closes it; a resume reopens one - /// at the ceiling, whose first frame is a keyframe on every backend, and retunes it - /// to the target. An encoder that cannot retune still resumes, at its ceiling, and - /// reports [`BitrateUnsupported`](moq_video::Error::BitrateUnsupported) so rate - /// control retires. - fn apply(&mut self, target: moq_mux::rate::Target) -> Result<(), moq_video::Error> { - let bitrate = match target { - moq_mux::rate::Target::Enabled(bitrate) => bitrate, - moq_mux::rate::Target::Disabled => { - // The catalog first, the only step that leaves the rendition as it was when it - // fails. After it the encoder goes whatever the drain reports, so the catalog never - // advertises a rendition whose frames are dropped. - self.producer.set_enabled(false)?; - if let Some(encoder) = self.encoder.take() { - self.target = moq_mux::rate::Target::Disabled; - // The encoder is gone either way, so a lost tail only costs those frames. - let drained = drain(encoder, &mut self.producer); - if let Err(err) = drained { - tracing::warn!(error = %err, "failed to drain a disabled rendition's encoder"); - } - self.producer.discontinuity()?; - } - return Ok(()); + /// A drain that fails ends the track with an error instead, since a clean finish promises + /// the subscriber everything the publisher produced. + fn publish_finish(self, id: Id) -> Result<(), Error> { + let child = crate::publish::Child::Video(id); + match block_on(self.producer.finish()) { + Ok(finished) => { + // The broadcast keeps the finished track, so its finish still holds it. + self.children.keep(child, finished); + Ok(()) + } + Err(err) => { + self.children.forget(child); + Err(err.into()) } - }; - if let Some(encoder) = self.encoder.as_mut() { - block_on(encoder.set_bitrate(bitrate))?; - self.target = target; - return Ok(()); - } - - let ceiling = *self.ceiling.read(); - let mut config = self.config.clone(); - config.bitrate = Some(ceiling); - let mut encoder = block_on(moq_video::encode::Sink::open(&config))?; - let retuned = match bitrate < ceiling { - true => block_on(encoder.set_bitrate(bitrate)), - false => Ok(()), - }; - if let Err(err) = &retuned - && !matches!(err, moq_video::Error::BitrateUnsupported(_)) - { - return retuned; } - self.producer.set_enabled(true)?; - self.encoder = Some(encoder); - self.target = moq_mux::rate::Target::Enabled(match retuned { - Ok(()) => bitrate.min(ceiling), - Err(_) => ceiling, - }); - retuned - } - - /// Drain and finish the encoder behind handle `id`. - fn publish_finish(self, id: Id) -> Result<(), Error> { - let VideoEncoder { - encoder, - mut producer, - follow, - children, - ceiling, - .. - } = self; - // The claim goes before the drain, so siblings can take the room while the last frames go - // out, and the rendition ends before it, so a rung above stops counting on it now. - let follower = follow.map(|follow| follow.follower); - drop(ceiling); - // Drain the codec into the track before ending it, so the last frames land - // in it rather than being dropped with the encoder. - let drained = match encoder { - Some(encoder) => drain(encoder, &mut producer), - None => Ok(()), - }; - finalize(producer, drained, &children, crate::publish::Child::Video(id))?; - // A follower panic only stops rate adaptation, so the track still ends - // cleanly before the panic is reported. - follower.map_or(Ok(()), |follower| follower.check()) } } -/// Publish every access unit `encoder` has ready. -fn publish( - encoder: &mut moq_video::encode::Sink, - producer: &mut moq_video::encode::Publisher, -) -> Result<(), moq_video::Error> { - while let Some(encoded) = encoder.receive()? { - producer.publish(&encoded)?; - } - Ok(()) -} - -/// Flush `encoder` and publish its tail, closing it. -fn drain( - mut encoder: moq_video::encode::Sink, - producer: &mut moq_video::encode::Publisher, -) -> Result<(), moq_video::Error> { - block_on(encoder.flush())?; - publish(&mut encoder, producer) -} - impl Video { - /// Advertise a track for an already-opened encoder. - /// - /// The encoder and the rendition it will emit are both resolved by the caller, - /// and before this, so a config this machine can't encode fails without leaving - /// a track advertised that will never carry frames. - /// - /// A `lower` encoder makes this one a simulcast rung above it: it takes the track - /// priority one below, and disables when its bandwidth grant can no longer fund it. - pub fn publish( + /// Hand out a handle for a published encoder, so the broadcast's close ends it. + pub(crate) fn insert( &mut self, - target: crate::publish::Target<'_>, + children: &crate::publish::Children, + producer: moq_video::encode::Producer, format: moq_video_pixel_format, - config: &moq_video::encode::Config, - rendition: hang::catalog::VideoConfig, - encoder: moq_video::encode::Sink, - lower: Option, + size: moq_video::Size, + reserved: bool, ) -> Result { - let reserved = target.catalog.reserve().with_interval(config.effective_join()); - let ceiling = video_ceiling(config, &rendition); - let (lower, priority) = match lower { - Some(lower) => { - let lower = self.producer(lower)?; - let guard = lower.lock(); - let below = guard.as_ref().ok_or(Error::MediaNotFound)?; - let below_ceiling = *below.ceiling.read(); - if below_ceiling >= ceiling { - return Err(Error::InvalidConfig(format!( - "the lower rendition's {} bps is not below this one's {} bps", - below_ceiling.as_bps(), - ceiling.as_bps() - ))); - } - (Some(below.ceiling.consume()), below.priority.saturating_sub(1)) - } - None => (None, hang::catalog::PRIORITY.video), - }; - // The track's suffix by codec, the one `moq_video::encode::publish_capture` uses too. - let suffix = match &rendition.codec { - hang::catalog::VideoCodec::H264(_) => ".avc3", - hang::catalog::VideoCodec::H265(_) => ".hev1", - other => { - return Err(Error::InvalidConfig(format!( - "{other} is not a codec this producer can publish" - ))); - } - }; - let track = target.producer.unique_track(suffix, reserved.track_info(priority))?; - let producer = moq_video::encode::Publisher::new(track, reserved, rendition)?; let encoder = Shared::new(VideoEncoder { - encoder: Some(encoder), producer, - config: config.clone(), - priority, - lower, format, - size: config.size(), - follow: None, - target: moq_mux::rate::Target::Enabled(ceiling), - ceiling: kio::Producer::new(ceiling), - children: target.children.clone(), + size, + reserved, + children: children.clone(), }); - target - .children - .insert(&mut self.producers, encoder, crate::publish::Child::Video) + children.insert(&mut self.producers, encoder, crate::publish::Child::Video) } - /// Reserve the encoder's ceiling against `allocator` and follow the grant. - pub(crate) fn follow(&self, id: Id, allocator: &moq_net::bandwidth::Allocator) -> Result<(), Error> { - let shared = self.producer(id)?; - let mut guard = shared.lock(); - guard - .as_mut() + /// The encoder's place in a simulcast ladder, for the rendition above it. + pub(crate) fn rung(&self, id: Id) -> Result { + Ok(self + .producer(id)? + .lock() + .as_ref() .ok_or(Error::MediaNotFound)? - .follow_grant(&shared, allocator.clone()); - Ok(()) + .producer + .rung()) } pub(crate) fn reservation(&self, id: Id) -> Result>, Error> { let shared = self.producer(id)?; let guard = shared.lock(); let encoder = guard.as_ref().ok_or(Error::MediaNotFound)?; - Ok(encoder.follow.as_ref().map(|follow| follow.reservation.clone())) + Ok(encoder.reserved.then(|| encoder.producer.reservation().clone())) } /// A watch-only handle to the encoded track's subscriber demand. @@ -1106,28 +694,26 @@ pub unsafe extern "C" fn moq_encode_video( } let (format, config) = unsafe { encoder_config(raw_input, raw_output)? }; - - // Both before the global lock is taken: bringing up a hardware encoder is slow - // enough that every other call would wait behind it. The probe runs first and - // closes its encoder before this one opens, so only one codec session is live. - let rendition = block_on(config.probe())?; - let encoder = block_on(moq_video::encode::Sink::open(&config))?; - - let mut state = State::lock(); - let allocator = bandwidth.map(|id| state.bandwidth.allocator(id)).transpose()?; - let State { publish, video, .. } = &mut *state; - let id = video.publish( - publish.target(broadcast)?, - format, - &config, - rendition.clone(), - encoder, - lower, - )?; - if let Some(allocator) = allocator.as_ref() { - video.follow(id, allocator)?; - } - Ok(id) + let size = config.size(); + let mut options = moq_video::encode::Options::new(config); + + // Everything the producer needs from the global state, taken out of it: bringing up a + // hardware encoder is slow enough that every other call would wait behind it. + let (producer, catalog, children) = { + let mut state = State::lock(); + if let Some(bandwidth) = bandwidth { + options.bandwidth = state.bandwidth.allocator(bandwidth)?; + } + if let Some(lower) = lower { + options.lower = Some(state.video.rung(lower)?); + } + let target = state.publish.target(broadcast)?; + (target.producer.clone(), target.catalog.clone(), target.children.clone()) + }; + let encoder = block_on(moq_video::encode::Producer::new(&producer, catalog, options))?; + State::lock() + .video + .insert(&children, encoder, format, size, bandwidth.is_some()) }) } @@ -1183,6 +769,9 @@ pub unsafe extern "C" fn moq_encode_video_demand( /// `lower` rendition is disabled for lack of bandwidth, the frame is dropped /// without encoding. /// +/// Published against a bandwidth allocator, the encoder follows its grant here, +/// before encoding, so it retunes between frames rather than mid-encode. +/// /// # Safety /// - `frame` must point to a valid [`moq_video_encoder_frame`]. /// - `frame->data` must point to `frame->data_size` bytes. @@ -1198,7 +787,7 @@ pub unsafe extern "C" fn moq_encode_video_frame(producer: u32, frame: *const moq guard .as_mut() .ok_or(Error::MediaNotFound)? - .publish_frame(&producer, frame.timestamp.try_into()?, data) + .publish_frame(frame.timestamp.try_into()?, data) }) } @@ -1235,7 +824,7 @@ pub extern "C" fn moq_encode_video_cut(producer: u32) -> i32 { /// for and adapt downwards from there. /// /// When this encoder was published against a bandwidth allocator, the reservation -/// and follower ceiling move with it, so a later grant cannot retune above this +/// and the grant's ceiling move with it, so a later grant cannot retune above this /// value. /// /// Returns a negative code if this backend cannot retune while running. That is @@ -1453,368 +1042,3 @@ pub extern "C" fn moq_decode_video_read_free(id: u32) -> i32 { State::lock().video.read_free(id) }) } - -#[cfg(test)] -mod tests { - use super::*; - - /// A video track wired up without an encoder, plus a subscriber on it: enough - /// to pin what [`finalize`] shows the far end. - async fn track_under_test() -> (moq_video::encode::Publisher, moq_net::track::Subscriber) { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let config = moq_mux::catalog::Config::default() - .with_catalog(moq_mux::catalog::hang::Catalog::::default()); - let catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); - let consumer = broadcast.consume(); - // Probed rather than hand-built, so the test track carries what a real one would. Software, - // like the encoders the tests open: a hardware default (VideoToolbox on macOS) declares - // another profile, so the first software frame would already move to a new track. - let mut encode = moq_video::encode::Config::new(320, 240, moq_video::FrameRate::new(30, 1).unwrap()); - encode.kind = moq_video::encode::Kind::Software; - let rendition = encode.probe().await.unwrap(); - let reserved = catalog.reserve().with_interval(encode.effective_join()); - let track = broadcast - .unique_track(".avc3", reserved.track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let producer = moq_video::encode::Publisher::new(track, reserved, rendition).unwrap(); - - let name = producer.demand().name().to_string(); - let track = consumer.track(&name).unwrap().subscribe(None).await.unwrap(); - (producer, track) - } - - /// A resume that reopens the encoder onto a larger picture publishes a new track (a higher - /// level is a new codec string), and the bandwidth claim moves there with it rather than - /// watching the track that ended. - #[tokio::test] - async fn a_resume_onto_a_new_level_reserves_on_the_new_track() { - let (producer, _track) = track_under_test().await; - let first = producer.demand().name().to_string(); - let probed = moq_video::encode::Config::new(320, 240, moq_video::FrameRate::new(30, 1).unwrap()); - let mut config = moq_video::encode::Config::new(640, 480, moq_video::FrameRate::new(30, 1).unwrap()); - config.kind = moq_video::encode::Kind::Software; - let mut opened = probed.clone(); - opened.kind = moq_video::encode::Kind::Software; - let estimate = moq_net::bandwidth::Producer::new(); - let shared = Shared::new(VideoEncoder { - encoder: Some(moq_video::encode::Sink::open(&opened).await.unwrap()), - producer, - config: config.clone(), - priority: hang::catalog::PRIORITY.video, - lower: None, - format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA, - size: probed.size(), - follow: None, - target: moq_mux::rate::Target::Enabled(moq_net::bandwidth::Rate::from_mbps(1)), - ceiling: kio::Producer::new(moq_net::bandwidth::Rate::from_mbps(1)), - children: Default::default(), - }); - let allocator = moq_net::bandwidth::Allocator::new(estimate.consume()); - let moved = tokio::task::spawn_blocking(move || { - let reservation = |shared: &Shared| { - let guard = shared.lock(); - guard - .as_ref() - .unwrap() - .follow - .as_ref() - .map(|follow| follow.reservation.clone()) - }; - let publish = |timestamp: moq_net::Timestamp, data: &[u8]| { - let mut guard = shared.lock(); - guard.as_mut().unwrap().publish_frame(&shared, timestamp, data).unwrap(); - }; - shared.lock().as_mut().unwrap().follow_grant(&shared, allocator); - let before = reservation(&shared).unwrap(); - publish(moq_net::Timestamp::ZERO, &vec![0x80; 320 * 240 * 4]); - assert!(Arc::ptr_eq(&before, &reservation(&shared).unwrap())); - - // Disabled, then resumed onto the larger picture. - { - let mut guard = shared.lock(); - let video = guard.as_mut().unwrap(); - video.apply(moq_mux::rate::Target::Disabled).unwrap(); - video.size = config.size(); - video - .apply(moq_mux::rate::Target::Enabled(moq_net::bandwidth::Rate::from_mbps(1))) - .unwrap(); - } - publish( - moq_net::Timestamp::from_micros(66_666).unwrap(), - &vec![0x80; 640 * 480 * 4], - ); - let after = reservation(&shared).expect("the claim moved with the track"); - assert!(!Arc::ptr_eq(&before, &after), "the claim still watches the ended track"); - // Finishing takes the claim with the encoder, so nothing watches the ended rendition. - let finished = shared.take().unwrap(); - let name = finished.producer.demand().name().to_string(); - finished.publish_finish(Id::try_from(1u32).unwrap()).unwrap(); - assert_eq!(Arc::strong_count(&after), 1, "only this test still holds the claim"); - name - }) - .await - .unwrap(); - assert_eq!(moved, format!("{first}/1")); - } - - /// A disable that fails partway leaves the catalog and the encoder agreeing, so viewers - /// never wait on a rendition the catalog calls enabled while its frames are dropped. - #[tokio::test] - async fn a_failed_disable_leaves_the_catalog_and_encoder_agreeing() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let config = moq_mux::catalog::Config::default() - .with_catalog(moq_mux::catalog::hang::Catalog::::default()); - let catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); - let mut encode = moq_video::encode::Config::new(320, 240, moq_video::FrameRate::new(30, 1).unwrap()); - encode.kind = moq_video::encode::Kind::Software; - let rendition = encode.probe().await.unwrap(); - let track = broadcast - .create_track("camera", catalog.track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let producer = moq_video::encode::Publisher::new(track.clone(), catalog.reserve(), rendition).unwrap(); - let shared = Shared::new(VideoEncoder { - encoder: Some(moq_video::encode::Sink::open(&encode).await.unwrap()), - producer, - config: encode.clone(), - priority: hang::catalog::PRIORITY.video, - lower: None, - format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420, - size: encode.size(), - follow: None, - target: moq_mux::rate::Target::Enabled(moq_net::bandwidth::Rate::from_mbps(1)), - ceiling: kio::Producer::new(moq_net::bandwidth::Rate::from_mbps(1)), - children: Default::default(), - }); - let (shared, disabled) = tokio::task::spawn_blocking(move || { - let disabled = { - let mut guard = shared.lock(); - let video = guard.as_mut().unwrap(); - video - .publish_frame(&shared, moq_net::Timestamp::ZERO, &vec![0x80; 320 * 240 * 3 / 2]) - .unwrap(); - // The track breaks, so the break marker the disable writes fails. - track.abort(moq_net::Error::Cancel).unwrap(); - video.apply(moq_mux::rate::Target::Disabled) - }; - (shared, disabled) - }) - .await - .unwrap(); - assert!(disabled.is_err(), "the break marker should fail on a broken track"); - let encoding = shared.lock().as_ref().unwrap().encoder.is_some(); - let enabled = catalog.snapshot().video.renditions["camera"].enabled; - assert_eq!(encoding, enabled, "the catalog and the encoder disagree"); - } - - /// A clean finish reaches the subscriber as the end of the track, which is what - /// makes the abort case below meaningful rather than vacuous. - #[tokio::test] - async fn a_successful_drain_ends_the_track_cleanly() { - let (producer, mut track) = track_under_test().await; - finalize( - producer, - Ok(()), - &Default::default(), - crate::publish::Child::Video(Id::try_from(1u32).unwrap()), - ) - .unwrap(); - assert!(matches!(track.recv_group().await, Ok(None)), "expected a clean end"); - } - - /// Regression: a lost tail must reach the subscriber as an abort. Finishing the - /// track anyway would report a truncated stream as a complete one, and only the - /// publisher would ever know otherwise. - #[tokio::test] - async fn a_failed_drain_aborts_the_track() { - let (producer, mut track) = track_under_test().await; - let err = moq_video::Error::Codec(anyhow::anyhow!("the codec lost the tail").into()); - finalize( - producer, - Err(err), - &Default::default(), - crate::publish::Child::Video(Id::try_from(1u32).unwrap()), - ) - .unwrap_err(); - - let Err(err) = track.recv_group().await else { - panic!("expected an abort, not a clean end"); - }; - assert!( - err.to_string().contains("the codec lost the tail"), - "the abort should carry the drain failure: {err}" - ); - } - - /// A rate follower panic stops adaptation, not the track: finishing still ends - /// it cleanly, then reports the panic. - #[tokio::test] - async fn finish_ends_the_track_after_a_rate_follower_panic() { - let (producer, mut track) = track_under_test().await; - let mut config = moq_video::encode::Config::new(320, 240, moq_video::FrameRate::new(30, 1).unwrap()); - config.kind = moq_video::encode::Kind::Software; - let encoder = moq_video::encode::Sink::open(&config).await.unwrap(); - let follower = crate::ffi::Background::new(async { panic!("injected rate follower panic") }); - assert!(matches!(follower.finished().await, Err(Error::Panic))); - let estimate = moq_net::bandwidth::Producer::new(); - let allocator = moq_net::bandwidth::Allocator::new(estimate.consume()); - let reservation = Arc::new(allocator.reserve(&producer.demand(), moq_net::bandwidth::Rate::from_mbps(1))); - let video = VideoEncoder { - encoder: Some(encoder), - producer, - config: config.clone(), - priority: hang::catalog::PRIORITY.video, - lower: None, - format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_I420, - size: config.size(), - follow: Some(Following { - allocator, - reservation, - _stop: oneshot::channel().0, - follower, - }), - target: moq_mux::rate::Target::Enabled(moq_net::bandwidth::Rate::from_mbps(1)), - ceiling: kio::Producer::new(moq_net::bandwidth::Rate::from_mbps(1)), - children: Default::default(), - }; - - let id = Id::try_from(1u32).unwrap(); - let result = tokio::task::spawn_blocking(move || video.publish_finish(id)) - .await - .unwrap(); - assert!(matches!(result, Err(Error::Panic)), "{result:?}"); - assert!(matches!(track.recv_group().await, Ok(None)), "expected a clean end"); - } -} - -#[cfg(test)] -mod follow_tests { - use std::time::Duration; - - use moq_mux::rate::{Grant, Policy, Target}; - use moq_net::bandwidth::Rate; - - use super::*; - - /// Retune through `set`, which only ever sees an enabled target: these policies set no - /// lower rendition, so nothing disables. - fn rate( - follower: &mut Grant, - ceiling: Rate, - target: Target, - set: impl FnOnce(Rate) -> Result<(), moq_video::Error>, - ) -> Result, &'static str> { - retune(follower, Policy::new(ceiling), target, |target| match target { - Target::Enabled(bitrate) => set(bitrate), - Target::Disabled => unreachable!("no lower rendition"), - }) - } - - /// A lower rendition moving between a target and its apply drops that target, and the - /// follower asks for it again under the new policy rather than counting it applied. - #[tokio::test(start_paused = true)] - async fn a_moved_lower_retries_the_dropped_target() { - let policy = |lower| { - let mut policy = Policy::new(Rate::from_mbps(4)); - policy.lower = Some(lower); - policy - }; - let grant = moq_net::bandwidth::Producer::new(); - let mut follower = Grant::new(grant.consume(), policy(Rate::from_mbps(1))); - grant.set(Some(Rate::from_mbps(2))).unwrap(); - let dropped = follower.next().await.unwrap(); - let moved = retune(&mut follower, policy(Rate::from_kbps(900)), dropped, |_| { - unreachable!("a target under a stale policy is never applied") - }); - assert_eq!(moved, Ok(None)); - - let retried = kio::clock::timeout(Duration::from_secs(60), follower.next()) - .await - .expect("the dropped target is asked for again"); - assert_eq!(retried.unwrap(), Target::Enabled(Rate::from_mbps(2))); - } - - fn bitrate(target: Target) -> Rate { - let Target::Enabled(bitrate) = target else { - unreachable!("no lower rendition") - }; - bitrate - } - - /// Retune at every target until the follower goes a minute without one. - async fn quiet(follower: &mut Grant, ceiling: Rate, mut set: impl FnMut(Rate) -> Result<(), moq_video::Error>) { - while let Ok(bitrate) = kio::clock::timeout(Duration::from_secs(60), follower.next()).await { - rate(follower, ceiling, bitrate.unwrap(), &mut set).unwrap(); - } - } - - /// The grant recovers once and never changes again, and the final raise fails once: - /// the follower still carries the encoder back to its ceiling. - #[tokio::test(start_paused = true)] - async fn a_quiet_grant_retries_a_failed_raise() { - let ceiling = Rate::from_mbps(4); - let grant = moq_net::bandwidth::Producer::new(); - let mut follower = Grant::new(grant.consume(), Policy::new(ceiling)); - let mut applied = Vec::new(); - let mut refuse = Some(ceiling); - let mut set = |bitrate| { - if refuse.take_if(|refused| *refused == bitrate).is_some() { - return Err(moq_video::Error::Unsupported("a transient refusal".into())); - } - applied.push(bitrate); - Ok(()) - }; - - grant.set(Some(Rate::from_mbps(1))).unwrap(); - let bitrate = follower.next().await.unwrap(); - rate(&mut follower, ceiling, bitrate, &mut set).unwrap(); - tokio::time::advance(Duration::from_secs(1)).await; - grant.set(Some(ceiling)).unwrap(); - quiet(&mut follower, ceiling, &mut set).await; - - assert_eq!(refuse, None, "the final raise was never refused"); - assert_eq!(applied.last(), Some(&ceiling), "{applied:?}"); - } - - /// `moq_encode_video_bitrate` lowers the ceiling while a raise toward the old one is - /// due. The ceiling is read under the encoder lock, so the raise is dropped rather - /// than retuning above the new cap, and the encoder recovers to the new cap. - #[tokio::test(start_paused = true)] - async fn a_lowered_ceiling_beats_a_due_raise() { - let old = Rate::from_mbps(4); - let lowered = Rate::from_mbps(2); - let grant = moq_net::bandwidth::Producer::new(); - let mut follower = Grant::new(grant.consume(), Policy::new(old)); - let applied = std::cell::RefCell::new(Vec::new()); - let set = |bitrate| { - applied.borrow_mut().push(bitrate); - Ok(()) - }; - - grant.set(Some(Rate::from_mbps(1))).unwrap(); - assert_eq!(follower.next().await.unwrap(), Target::Enabled(Rate::from_mbps(1))); - tokio::time::advance(Duration::from_secs(1)).await; - grant.set(Some(old)).unwrap(); - let first = follower.next().await.unwrap(); - rate(&mut follower, old, first, set).unwrap(); - - // `moq_encode_video_bitrate` already retuned the encoder to the new cap. - let due = follower.next().await.unwrap(); - assert!(bitrate(due) > bitrate(first), "a raise toward the old ceiling"); - assert_eq!(rate(&mut follower, lowered, due, set), Ok(None)); - assert_eq!(follower.policy().max, lowered); - quiet(&mut follower, lowered, set).await; - assert_eq!(*applied.borrow(), [bitrate(first)], "nothing retuned past the new cap"); - - // A later attack and recovery climbs back to the new cap, not the old one. - grant.set(Some(Rate::from_mbps(1))).unwrap(); - let bitrate = follower.next().await.unwrap(); - rate(&mut follower, lowered, bitrate, set).unwrap(); - tokio::time::advance(Duration::from_secs(1)).await; - grant.set(Some(old)).unwrap(); - quiet(&mut follower, lowered, set).await; - - let applied = applied.into_inner(); - assert_eq!(applied.last(), Some(&lowered), "{applied:?}"); - assert!(applied.iter().all(|bitrate| *bitrate <= lowered), "{applied:?}"); - } -} diff --git a/rs/moq-ffi/src/bandwidth.rs b/rs/moq-ffi/src/bandwidth.rs index 6f576e36770..b5c8f9e3899 100644 --- a/rs/moq-ffi/src/bandwidth.rs +++ b/rs/moq-ffi/src/bandwidth.rs @@ -54,12 +54,13 @@ impl MoqAllocator { /// means no estimate, so hold the current rate, and `Some(0)` is a real zero rate. #[derive(uniffi::Object)] pub struct MoqReservation { - inner: moq_net::bandwidth::Reservation, + /// Shared with the producer that holds the claim, such as a video producer's. + inner: Arc, } impl MoqReservation { - pub(crate) fn new(inner: moq_net::bandwidth::Reservation) -> Self { - Self { inner } + pub(crate) fn new(inner: impl Into>) -> Self { + Self { inner: inner.into() } } pub(crate) fn consume(&self) -> moq_net::bandwidth::Consumer { diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 7b55eaeedf9..e5190467161 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7789,13 +7789,25 @@ async fn video_encoder_follows_a_shrinking_grant() { .unwrap(); assert_eq!(reservation.rate(), Some(1_000_000)); - // A drop applies at once, so this waits on the follow thread alone, never a timer. - video.wait_applied(1_000_000); + // A drop applies at once, on the next frame written. + video.write(gray_frame(0)).unwrap(); + assert_eq!(video.applied_bitrate(), 1_000_000); video.finish().unwrap(); broadcast.close().unwrap(); } +/// One mid-gray 320x240 RGBA picture at `micros`. +fn gray_frame(micros: u64) -> crate::video::MoqVideoFrame { + crate::video::MoqVideoFrame { + timestamp: crate::media::MoqTimestamp { + value: micros, + scale: Some(1_000_000), + }, + data: vec![0x80; 320 * 240 * 4], + } +} + fn software_camera(bitrate: u64) -> (crate::video::MoqVideoEncoderInput, crate::video::MoqVideoEncoderOutput) { use crate::video::*; ( @@ -7817,36 +7829,6 @@ fn software_camera(bitrate: u64) -> (crate::video::MoqVideoEncoderInput, crate:: ) } -/// Dropping a producer parked in `changed()` must stop the follow thread, not -/// leak it until the session allocator dies. -#[tokio::test] -async fn dropping_a_video_producer_stops_the_rate_follower() { - use crate::bandwidth::MoqAllocator; - - let estimate = moq_net::bandwidth::Producer::new(); - let bandwidth = std::sync::Arc::new(MoqAllocator::new(moq_net::bandwidth::Allocator::new( - estimate.consume(), - ))); - let broadcast = MoqBroadcastProducer::new().unwrap(); - let (input, output) = software_camera(4_000_000); - let video = crate::video::MoqVideoProducer::new( - &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), - crate::video::MoqVideoProducerOptions { - input, - output, - track: Some("camera".into()), - bandwidth: Some(bandwidth), - lower: None, - }, - ) - .unwrap(); - - kio::clock::timeout(TIMEOUT, tokio::task::spawn_blocking(move || drop(video))) - .await - .expect("rate follower did not stop") - .expect("drop panicked"); -} - /// `set_bitrate` is the manual ceiling: a later grant cannot retune above it. #[tokio::test] async fn set_bitrate_caps_a_later_bandwidth_grant() { @@ -7886,6 +7868,7 @@ async fn set_bitrate_caps_a_later_bandwidth_grant() { .set(Some(moq_net::bandwidth::Rate::from_bps(3_000_000))) .unwrap(); assert_eq!(reservation.rate(), Some(1_000_000)); + video.write(gray_frame(0)).unwrap(); assert_eq!(video.applied_bitrate(), 1_000_000); video.finish().unwrap(); diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index cf92787a6a5..b6971c9ac4e 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -11,9 +11,6 @@ //! timestamp. use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - -use tokio::sync::oneshot; use crate::bandwidth::{MoqAllocator, MoqReservation}; use crate::consumer::MoqBroadcastConsumer; @@ -282,50 +279,6 @@ impl MoqVideoEncoder { } } -/// Publish every access unit `encoder` has ready. -fn publish( - encoder: &mut moq_video::encode::Sink, - producer: &mut moq_video::encode::Publisher, -) -> Result<(), moq_video::Error> { - while let Some(encoded) = encoder.receive()? { - producer.publish(&encoded)?; - } - Ok(()) -} - -/// Flush `encoder` and publish its tail, closing it. -fn drain( - mut encoder: moq_video::encode::Sink, - producer: &mut moq_video::encode::Publisher, -) -> Result<(), moq_video::Error> { - block_on(encoder.flush())?; - publish(&mut encoder, producer) -} - -/// End a video track, given the result of draining its encoder into it. -/// -/// A clean finish is a promise that the track holds everything the publisher -/// produced, so a lost tail has to end the track as an abort instead. Finishing -/// anyway would leave a truncated stream indistinguishable from a complete one, -/// and only the local caller would ever learn otherwise. -fn finalize( - mut producer: moq_video::encode::Publisher, - drained: Result<(), moq_video::Error>, - children: &crate::producer::Children, -) -> Result<(), MoqError> { - match drained { - Ok(()) => { - producer.finish()?; - children.keep(producer); - Ok(()) - } - Err(err) => { - producer.abort(moq_net::Error::Transport(err.to_string())); - Err(err.into()) - } - } -} - /// Wait out an encode-thread round trip from a synchronous export. /// /// The producer methods are sync, matching every other write path here @@ -340,259 +293,10 @@ fn block_on(future: impl std::future::Future) -> T { pollster::block_on(future) } -/// An encoder paired with the track publishing its output. -/// -/// The encoder is a [`Sink`](moq_video::encode::Sink) rather than a bare -/// `Encoder` because this object is shared across threads: uniffi hands it to -/// whichever thread the caller writes from, and an `Encoder` built on one thread -/// and dropped on another unbalances the per-thread COM apartment the Windows -/// backend opens. The sink owns the thread instead, so every caller is welcome. -struct VideoProducer { - /// `None` while the rendition is disabled for lack of bandwidth. - encoder: Option, - producer: moq_video::encode::Publisher, - /// The encoder settings, to reopen at [`ceiling`](Self::ceiling) when a disabled - /// rendition resumes. - config: moq_video::encode::Config, - format: MoqVideoPixelFormat, - /// The encoded resolution, from the publish config. Frames carry only pixels, - /// so this is what says how to read them. - size: moq_video::Size, - /// The manual ceiling [`MoqVideoProducer::set_bitrate`] last applied, which caps - /// every grant. Behind the encoder's lock, so the follower never reads an old, - /// higher one and retunes over a newer cap. A channel, so a rung above this - /// rendition wakes when it moves and when this rendition ends. - ceiling: kio::Producer, - /// The track priority this rendition was published at; a higher rung takes one below. - priority: i8, - /// What the encoder currently produces, which a follower started on a moved track resumes - /// from rather than assuming the ceiling. - target: moq_mux::rate::Target, - /// The bandwidth claim on the current track and its follower, when published against - /// an allocator. Beside the encoder, so a move to a new track, a finish, and an abort - /// change all three together. - follow: Option, - /// Notified under this rendition's lock each time the follower retunes, so a test - /// awaits the follower instead of polling `target`. - #[cfg(test)] - retuned: Arc, -} - -/// A rendition's bandwidth claim on its current track, and the thread retuning the encoder -/// to the claim's grant. -struct Following { - /// Reserves against the track a decoder identity change moves the rendition to. Held - /// so the reservation's registry outlives extra bandwidth handles. - allocator: Arc, - /// The next lower rendition's ceiling, which sets this one's band. - lower: Option>, - reservation: Arc, - follower: Follow, -} - -impl Following { - /// Reserve `producer`'s ceiling against its current track and follow the grant from its - /// current target, retuning the producer in `inner`. - fn start( - allocator: Arc, - lower: Option>, - producer: &VideoProducer, - inner: &Shared, - ) -> Self { - let ceiling = *producer.ceiling.read(); - let reservation = allocator.reserve_demand(&producer.producer.demand(), ceiling.as_bps()); - let follower = spawn_follow(Followed { - inner: inner.clone(), - lower: lower.clone(), - consumer: reservation.consume(), - reservation: Arc::downgrade(&reservation), - ceiling, - target: producer.target, - }); - Self { - allocator, - lower, - reservation, - follower, - } - } -} - -/// The thread following a grant, stopped when its [`Following`] is replaced or dropped. -/// -/// `close` participates in the follower's `select!`, so dropping it wakes a thread parked in -/// `consumer.changed()` instead of leaking it until the allocator dies. -struct Follow { - close: oneshot::Sender<()>, - thread: std::thread::JoinHandle<()>, - failed: Arc, -} - -impl Follow { - fn check(&self) -> Result<(), MoqError> { - if self.failed.load(Ordering::Acquire) { - return Err(panic_error()); - } - Ok(()) - } - - /// Stop the thread and wait for it, reporting its panic. - /// - /// The thread takes the producer's lock, so call this without holding it. - fn stop(self) -> Result<(), MoqError> { - drop(self.close); - self.thread.join().map_err(|_| panic_error())?; - if self.failed.load(Ordering::Acquire) { - return Err(panic_error()); - } - Ok(()) - } -} - fn panic_error() -> MoqError { crate::session::map_closed_error(moq_tokio::Error::Panic) } -impl VideoProducer { - /// Whether this rendition still follows `reservation`, rather than having moved off the - /// track it was reserved on or ended. The [`Weak`](std::sync::Weak) holds the allocation, so - /// a newer reservation never takes its address. - fn follows(&self, reservation: &std::sync::Weak) -> bool { - self.follow - .as_ref() - .is_some_and(|follow| std::ptr::eq(Arc::as_ptr(&follow.reservation), reservation.as_ptr())) - } - - /// Move the claim to the track the rendition is on now, if publishing moved it off the one - /// named `before`, such as on a decoder identity change. Returns the replaced follower, which - /// takes the producer's lock on its way out. - fn follow_track(&mut self, before: &str, inner: &Shared) -> Option { - if self.producer.demand().name() == before { - return None; - } - let follow = self.follow.as_ref()?; - let moved = Following::start(follow.allocator.clone(), follow.lower.clone(), self, inner); - Some(std::mem::replace(self.follow.as_mut()?, moved).follower) - } - - fn write(&mut self, frame: MoqVideoFrame) -> Result<(), MoqError> { - // A buffer that isn't one picture at the configured size is rejected here, - // by the surface constructors, rather than reinterpreted. - let surface = match self.format { - MoqVideoPixelFormat::I420 => moq_video::Surface::I420(moq_video::I420::new(self.size, frame.data)?), - MoqVideoPixelFormat::Rgba => moq_video::Surface::rgba(&frame.data, self.size)?, - }; - - // Disabled: nothing is encoded until the grant resumes the rendition. - let Some(encoder) = self.encoder.as_mut() else { - return Ok(()); - }; - let frame = moq_video::Frame::new(surface, frame.timestamp.try_into()?); - // A backend that pipelines readies an earlier frame's output, so `publish` - // drains zero or more access units rather than one per call. - block_on(encoder.encode(frame))?; - publish(encoder, &mut self.producer)?; - Ok(()) - } - - /// Retune the live encoder and cap later grants at `bitrate`. - fn set_ceiling(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), MoqError> { - // A disabled rung has no encoder to retune; it reopens at the new ceiling. - if let Some(encoder) = self.encoder.as_mut() { - block_on(encoder.set_bitrate(bitrate))?; - self.target = moq_mux::rate::Target::Enabled(bitrate); - } - // Under the same lock as the retune, so the follower sees both or neither. - let Ok(mut ceiling) = self.ceiling.write() else { - unreachable!("only dropping the producer closes its ceiling") - }; - *ceiling = bitrate; - drop(ceiling); - if let Some(follow) = &self.follow { - follow.reservation.update(bitrate.as_bps()); - } - Ok(()) - } - - /// Apply a rate-control target, leaving the rendition as it was on an error. - /// - /// A disable drains the encoder into the track and closes it; a resume reopens one - /// at the ceiling, whose first frame is a keyframe on every backend, and retunes it - /// to the target. An encoder that cannot retune still resumes, at its ceiling, and - /// reports [`BitrateUnsupported`](moq_video::Error::BitrateUnsupported) so rate - /// control retires. - fn apply(&mut self, target: moq_mux::rate::Target) -> Result<(), moq_video::Error> { - let bitrate = match target { - moq_mux::rate::Target::Enabled(bitrate) => bitrate, - moq_mux::rate::Target::Disabled => { - // The catalog first, the only step that leaves the rendition as it was when it - // fails. After it the encoder goes whatever the drain reports, so the catalog never - // advertises a rendition whose frames are dropped. - self.producer.set_enabled(false)?; - if let Some(encoder) = self.encoder.take() { - self.target = moq_mux::rate::Target::Disabled; - // The encoder is gone either way, so a lost tail only costs those frames. - let drained = drain(encoder, &mut self.producer); - if let Err(err) = drained { - tracing::warn!(error = %err, "failed to drain a disabled rendition's encoder"); - } - self.producer.discontinuity()?; - } - return Ok(()); - } - }; - if let Some(encoder) = self.encoder.as_mut() { - block_on(encoder.set_bitrate(bitrate))?; - self.target = target; - return Ok(()); - } - - let ceiling = *self.ceiling.read(); - let mut config = self.config.clone(); - config.bitrate = Some(ceiling); - let mut encoder = block_on(moq_video::encode::Sink::open(&config))?; - let retuned = match bitrate < ceiling { - true => block_on(encoder.set_bitrate(bitrate)), - false => Ok(()), - }; - if let Err(err) = &retuned - && !matches!(err, moq_video::Error::BitrateUnsupported(_)) - { - return retuned; - } - self.producer.set_enabled(true)?; - self.encoder = Some(encoder); - self.target = moq_mux::rate::Target::Enabled(match retuned { - Ok(()) => bitrate.min(ceiling), - Err(_) => ceiling, - }); - retuned - } - - /// Drain the encoder into the track and end it, returning the follower to stop once the - /// producer's lock is released. - fn finish(self, children: &crate::producer::Children) -> (Result<(), MoqError>, Option) { - let Self { - encoder, - mut producer, - ceiling, - follow, - .. - } = self; - // The claim goes before the drain, so siblings can take the room while the last frames go - // out, and the rendition ends before it, so a rung above stops counting on it now. - let follower = follow.map(|follow| follow.follower); - drop(ceiling); - // Drain the codec into the track before ending it, so the last frames land - // in it rather than being dropped with the encoder. - let drained = match encoder { - Some(encoder) => drain(encoder, &mut producer), - None => Ok(()), - }; - (finalize(producer, drained, children), follower) - } -} - /// Producer for a raw-video track. /// /// Constructed from a broadcast's media catalog. Each @@ -605,48 +309,35 @@ impl VideoProducer { #[derive(uniffi::Object)] pub struct MoqVideoProducer { /// `None` once finished or aborted. - inner: Shared, + /// + /// The encoder lives in a [`Sink`](moq_video::encode::Sink) inside the producer, which owns + /// the thread it runs on: uniffi hands this object to whichever thread the caller writes + /// from, and an encoder built on one thread and dropped on another unbalances the + /// per-thread COM apartment the Windows backend opens. + inner: std::sync::Mutex>, + format: MoqVideoPixelFormat, + /// The encoded resolution, from the publish config. Frames carry only pixels, so this is + /// what says how to read them. + size: moq_video::Size, + /// The allocator the producer reserves against, `None` when published without one. Held + /// so the reservation's registry outlives extra bandwidth handles. + bandwidth: Option>, children: crate::producer::Children, } impl MoqVideoProducer { - /// Run `f` on the open rendition, failing with its follower's panic or `Closed`. - fn with(&self, f: impl FnOnce(&mut VideoProducer) -> Result) -> Result { + /// Run `f` on the open producer, failing with `Closed` once it finished. + fn with(&self, f: impl FnOnce(&mut moq_video::encode::Producer) -> Result) -> Result { let mut guard = self.inner.lock().map_err(|_| panic_error())?; let producer = guard.as_mut().ok_or(MoqError::Closed)?; - if let Some(follow) = &producer.follow { - follow.follower.check()?; - } f(producer) } - fn track_demand(&self) -> Result { - self.with(|producer| Ok(producer.producer.demand())) - } - /// The bitrate the encoder produces at, or 0 while disabled. #[cfg(test)] pub(crate) fn applied_bitrate(&self) -> u64 { - applied(self.inner.lock().unwrap().as_ref()) - } - - /// Block until the follower has retuned the encoder to `bitrate`. - #[cfg(test)] - pub(crate) fn wait_applied(&self, bitrate: u64) { - let mut guard = self.inner.lock().unwrap(); - while applied(guard.as_ref()) != bitrate { - let retuned = guard.as_ref().expect("the rendition is open").retuned.clone(); - guard = retuned.wait(guard).unwrap(); - } - } -} - -/// The bitrate `producer` encodes at, or 0 while disabled or gone. -#[cfg(test)] -fn applied(producer: Option<&VideoProducer>) -> u64 { - match producer.map(|producer| producer.target) { - Some(moq_mux::rate::Target::Enabled(bitrate)) => bitrate.as_bps(), - _ => 0, + self.with(|producer| Ok(producer.bitrate().map_or(0, moq_net::bandwidth::Rate::as_bps))) + .unwrap() } } @@ -659,13 +350,9 @@ impl crate::producer::Child for MoqVideoProducer { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); - let Some(VideoProducer { producer, follow, .. }) = producer else { - return; - }; - if let Some(follow) = follow { - let _ = follow.follower.stop(); + if let Some(producer) = producer { + producer.abort(err); } - producer.abort(err); } } @@ -679,7 +366,7 @@ impl Drop for MoqVideoProducer { impl MoqVideoProducer { /// A watch-only handle to whether this video track has subscribers. pub fn demand(&self) -> Result, MoqError> { - Ok(MoqTrackDemand::new(self.track_demand()?)) + Ok(MoqTrackDemand::new(self.with(|producer| Ok(producer.demand()))?)) } /// Encode and publish one raw frame. @@ -688,20 +375,19 @@ impl MoqVideoProducer { /// call that emits nothing on the wire is normal rather than an error. While a /// [`lower`](MoqVideoProducerOptions::lower) rung is disabled for lack of /// bandwidth, the frame is dropped without encoding. + /// + /// Published against a [`MoqAllocator`], the encoder follows the bandwidth grant here, + /// before encoding, so it retunes between frames rather than mid-encode. pub fn write(&self, frame: MoqVideoFrame) -> Result<(), MoqError> { let _guard = crate::ffi::runtime().enter(); - let (written, moved) = self.with(|producer| { - let before = producer.producer.demand(); - let written = producer.write(frame); - // A frame whose decoder identity changed, such as a resume at a ceiling past the - // declared level, moved the rendition to a newly named track, and the claim moves - // with it under the same lock. It moves even when publishing on the new track - // failed, so the claim never stays on the ended one. - Ok((written, producer.follow_track(before.name(), &self.inner))) - })?; - // Outside the lock, which the old follower takes on its way out. - let stopped = moved.map_or(Ok(()), Follow::stop); - written.and(stopped) + // A buffer that isn't one picture at the configured size is rejected here, + // by the surface constructors, rather than reinterpreted. + let surface = match self.format { + MoqVideoPixelFormat::I420 => moq_video::Surface::I420(moq_video::I420::new(self.size, frame.data)?), + MoqVideoPixelFormat::Rgba => moq_video::Surface::rgba(&frame.data, self.size)?, + }; + let frame = moq_video::Frame::new(surface, frame.timestamp.try_into()?); + self.with(|producer| Ok(block_on(producer.write(frame))?)) } /// Cut a new group at the next written frame. @@ -721,14 +407,7 @@ impl MoqVideoProducer { /// without the control): nothing is queued, and groups keep falling at the /// configured interval. pub fn cut(&self) -> Result<(), MoqError> { - self.with(|producer| { - // A keyframe is what a cut is on the wire: the importer closes the open - // group and starts a new one at it. A disabled rung resumes on a keyframe anyway. - if let Some(encoder) = producer.encoder.as_mut() { - block_on(encoder.cut())?; - } - Ok(()) - }) + self.with(|producer| Ok(block_on(producer.cut())?)) } /// Retune the live encoder to `bitrate` bits per second, taking effect from @@ -741,7 +420,7 @@ impl MoqVideoProducer { /// and adapt downwards from there. /// /// When this producer was published against a [`MoqAllocator`], the reservation - /// and follower ceiling move with it, so a later grant cannot retune above this + /// and the grant's ceiling move with it, so a later grant cannot retune above this /// value. /// /// Errors if this backend cannot retune while running. That is not fatal: the @@ -749,7 +428,11 @@ impl MoqVideoProducer { /// stop publishing. pub fn set_bitrate(&self, bitrate: u64) -> Result<(), MoqError> { let _guard = crate::ffi::runtime().enter(); - self.with(|producer| producer.set_ceiling(moq_net::bandwidth::Rate::from_bps(bitrate))) + self.with(|producer| { + Ok(block_on( + producer.set_bitrate(moq_net::bandwidth::Rate::from_bps(bitrate)), + )?) + }) } /// This encoder's bandwidth reservation, if it was published against a @@ -758,8 +441,9 @@ impl MoqVideoProducer { /// decoder identity publishes under a new track name and reserves against that /// track instead, so read this again after [`write`](Self::write). pub fn reservation(&self) -> Option> { + self.bandwidth.as_ref()?; let guard = self.inner.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - Some(guard.as_ref()?.follow.as_ref()?.reservation.clone()) + Some(Arc::new(MoqReservation::new(guard.as_ref()?.reservation().clone()))) } /// Flush any frames the codec is still holding and finalize the track. @@ -771,13 +455,10 @@ impl MoqVideoProducer { // missing a track that still publishes. let mut inner = self.inner.lock().map_err(|_| panic_error())?; let producer = inner.take().ok_or(MoqError::Closed)?; - let (finished, follower) = producer.finish(&self.children); - drop(inner); - // A follower panic only stops rate adaptation, so the encoded frames still - // end the track cleanly before the panic is reported. - let followed = follower.map_or(Ok(()), Follow::stop); - finished?; - followed + // A failed drain already ended the track with an error, so readers see the failure + // rather than a truncated stream that looks complete. + self.children.keep(block_on(producer.finish())?); + Ok(()) } } @@ -793,8 +474,8 @@ impl MoqVideoProducer { /// before a frame is written to it. /// /// Set `options.bandwidth` to reserve this track's configured bitrate against the - /// session's allocator and follow the grant with the same policy the Rust - /// capture encoder uses. [`set_bitrate`](MoqVideoProducer::set_bitrate) is + /// session's allocator and follow the grant between frames with the same policy the + /// Rust capture encoder uses. [`set_bitrate`](MoqVideoProducer::set_bitrate) is /// the manual ceiling: it retunes the encoder and moves the reservation. Set /// `options.lower` too for each rendition above the lowest of one source. #[uniffi::constructor] @@ -813,272 +494,33 @@ impl MoqVideoProducer { lower, } = options; let config = video_encoder_config(&input, &output)?; - // The lower rendition's handle, its ceiling, and the priority this one takes below it. - let lower = match lower { - Some(lower) => { - if bandwidth.is_none() { - return Err(MoqError::Video { - reason: "a lower rendition needs a bandwidth allocator to follow".into(), - }); - } - let guard = lower.inner.lock().map_err(|_| panic_error())?; - let below = guard.as_ref().ok_or(MoqError::Closed)?; - Some(( - below.ceiling.consume(), - *below.ceiling.read(), - below.priority.saturating_sub(1), - )) - } - None => None, - }; - let priority = lower - .as_ref() - .map_or(hang::catalog::PRIORITY.video, |(_, _, priority)| *priority); - - // Both before the track exists: a config this machine can't encode should fail - // without leaving a track advertised that will never carry frames. The probe - // closes its encoder before this one opens, so only one codec session is live. - let rendition = block_on(config.probe())?; - let encoder = block_on(moq_video::encode::Sink::open(&config))?; - let join = config.effective_join(); - let ceiling = video_ceiling(&config, &rendition); - if let Some((_, below, _)) = &lower - && *below >= ceiling - { - return Err(MoqError::Video { - reason: format!( - "the lower rendition's {} bps is not below this one's {} bps", - below.as_bps(), - ceiling.as_bps() - ), - }); + let size = config.size(); + let mut options = moq_video::encode::Options::new(config); + options.track = track; + if let Some(bandwidth) = &bandwidth { + options.bandwidth = bandwidth.inner().clone(); } - // The track's suffix by codec, the one `moq_video::encode::publish_capture` uses too. - let suffix = match &rendition.codec { - hang::catalog::VideoCodec::H264(_) => ".avc3", - hang::catalog::VideoCodec::H265(_) => ".hev1", - other => { + if let Some(lower) = lower { + if bandwidth.is_none() { return Err(MoqError::Video { - reason: format!("{other} is not a codec this producer can publish"), + reason: "a lower rendition needs a bandwidth allocator to follow".into(), }); } - }; - let children = broadcast.with_state(|state| Ok(state.children.clone()))?; - let producer = broadcast.with_state(|state| { - let info = state.catalog.track_info(priority); - let track = match track { - Some(name) => state.broadcast.create_track(name, info)?, - None => state.broadcast.unique_track(suffix, info)?, - }; - Ok(moq_video::encode::Publisher::new( - track, - state.catalog.reserve().with_interval(join), - rendition, - )?) - })?; - - let video = MoqVideoProducer { - inner: Arc::new(std::sync::Mutex::new(Some(VideoProducer { - encoder: Some(encoder), - producer, - format: input.format, - size: config.size(), - config, - ceiling: kio::Producer::new(ceiling), - priority, - target: moq_mux::rate::Target::Enabled(ceiling), - follow: None, - #[cfg(test)] - retuned: Arc::default(), - }))), - children: children.clone(), - }; - if let Some(allocator) = bandwidth { - // Its follower takes this lock, so it waits until the claim is in place. - let mut guard = video.inner.lock().map_err(|_| panic_error())?; - let producer = guard.as_mut().expect("just constructed"); - let lower = lower.map(|(lower, _, _)| lower); - producer.follow = Some(Following::start(allocator, lower, producer, &video.inner)); + options.lower = Some(lower.with(|lower| Ok(lower.rung()))?); } - children.adopt(Arc::new(video)) - } -} - -/// The bitrate this encoder reserved, which is the configured ceiling or the -/// one [`Config::probe`](moq_video::encode::Config::probe) filled in. -fn video_ceiling( - config: &moq_video::encode::Config, - rendition: &hang::catalog::VideoConfig, -) -> moq_net::bandwidth::Rate { - config - .bitrate - .or_else(|| rendition.bitrate.map(moq_net::bandwidth::Rate::from_bps)) - .unwrap_or_else(|| { - // Same 0.07 bits/pixel/s default moq-video uses when neither is set. - moq_net::bandwidth::Rate::from_bps( - (config.size().pixels() as f64 * config.framerate.as_f64() * 0.07) as u64, - ) - }) -} -/// The rendition behind a [`MoqVideoProducer`], `None` once finished or aborted. -type Shared = Arc>>; - -/// What the follow thread reads and retunes. -struct Followed { - inner: Shared, - /// The next lower rendition's ceiling, which sets this one's band. A consumer, so a rung - /// never keeps the rendition below it publishing. - lower: Option>, - consumer: moq_net::bandwidth::Consumer, - /// The claim this thread follows, which retires it once the rendition moves off it or ends. - reservation: std::sync::Weak, - ceiling: moq_net::bandwidth::Rate, - /// What the encoder produces when the follower starts. - target: moq_mux::rate::Target, -} - -fn spawn_follow(followed: Followed) -> Follow { - let (close, closed) = oneshot::channel(); - let failed = Arc::new(AtomicBool::new(false)); - let failure = failed.clone(); - // A dedicated thread rather than the FFI current-thread runtime: this loop - // `pollster::block_on`s `set_bitrate`, which would stall that runtime. - let thread = std::thread::Builder::new() - .name("moq-ffi-rate".into()) - .spawn(move || { - let completed = pollster::block_on(moq_tokio::panic::isolate(async move { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("rate-control runtime") - .block_on(async move { - tokio::select! { - biased; - _ = closed => {} - _ = follow_reservation(followed) => {} - } - }) - })); - if completed.is_none() { - failure.store(true, Ordering::Release); - } - }) - .expect("failed to spawn rate-control thread"); - Follow { close, thread, failed } -} - -/// The policy a rendition follows: its manual `ceiling`, above the `lower` rendition's -/// current ceiling, if any. -fn video_policy(ceiling: moq_net::bandwidth::Rate, lower: Option) -> moq_mux::rate::Policy { - let mut policy = moq_mux::rate::Policy::new(ceiling); - policy.lower = lower; - policy -} - -/// The lower rendition's current ceiling, or `None` once it finished or was dropped, -/// which leaves this one nothing to fall back to, so it never disables. Arms `waiter` -/// for either change. -fn lower_ceiling( - lower: &Option>, - waiter: &kio::Waiter, -) -> Option { - let lower = lower.as_ref()?; - match lower.poll(waiter, |_| std::task::Poll::<()>::Pending) { - std::task::Poll::Pending => Some(*lower.read()), - // The closure never completes, so this is the channel closing. - std::task::Poll::Ready(_) => None, - } -} - -/// Feed each grant through the same policy moq-video uses, and retune the live -/// encoder, disabling and resuming a rung. Retires on -/// [`BitrateUnsupported`](moq_video::Error::BitrateUnsupported). -async fn follow_reservation(followed: Followed) { - let Followed { - inner, - lower, - consumer, - reservation, - ceiling, - target, - } = followed; - let noop = kio::Waiter::noop(); - let mut follower = moq_mux::rate::Grant::new(consumer, video_policy(ceiling, lower_ceiling(&lower, &noop))); - follower.assume(target); - loop { - // Both ceilings are read on every wake, before the follower filters a grant against - // them. This one's `set_bitrate` moves its reservation after moving its ceiling, - // which wakes this poll; the lower one's ceiling wakes it directly, wherever that - // rendition reserves. The lower one is read first, so no two encoder locks are - // ever held at once. - let next = kio::wait(|waiter| { - let below = lower_ceiling(&lower, waiter); - let guard = inner.lock().unwrap(); - let Some(producer) = guard.as_ref().filter(|producer| producer.follows(&reservation)) else { - return std::task::Poll::Ready(None); - }; - follower.set_policy(video_policy(*producer.ceiling.read(), below)); - follower.poll_next(waiter).map(Some) - }) - .await; - let Some(Ok(target)) = next else { - return; - }; - let below = lower_ceiling(&lower, &noop); - let mut guard = inner.lock().unwrap(); - // Checked again under the lock it applies under: a move or finish may have replaced this - // claim while the target was on its way. - let Some(producer) = guard.as_mut().filter(|producer| producer.follows(&reservation)) else { - return; - }; - let policy = video_policy(*producer.ceiling.read(), below); - let before = producer.producer.demand(); - let retuned = retune(&mut follower, policy, target, |target| producer.apply(target)); - #[cfg(test)] - producer.retuned.notify_all(); - // A disable drains the encoder into the track, and output with a new decoder identity - // moves the rendition, so the claim moves with it. The replaced follower is this - // thread, which ends here, so it is detached rather than joined. - if producer.follow_track(before.name(), &inner).is_some() || retuned.is_err() { - return; - } - } -} - -/// Apply the follower's next `target` through `apply`, unless the `policy` moved since it -/// was asked for: then follow under the new policy, whose manual ceiling -/// `MoqVideoProducer::set_bitrate` already applied, instead. Returns the target applied, -/// or `Err` with the name of an encoder that cannot follow at all. -fn retune( - follower: &mut moq_mux::rate::Grant, - policy: moq_mux::rate::Policy, - target: moq_mux::rate::Target, - apply: impl FnOnce(moq_mux::rate::Target) -> Result<(), moq_video::Error>, -) -> Result, &'static str> { - if *follower.policy() != policy { - // The target was never applied, so take it back first: a policy that keeps the - // follower's state (a moved lower rendition) would otherwise count it applied. - follower.failed(); - follower.set_policy(policy); - return Ok(None); - } - match apply(target) { - Ok(()) => { - tracing::debug!(?target, "adjusted encoder bitrate"); - Ok(Some(target)) - } - Err(moq_video::Error::BitrateUnsupported(name)) => { - tracing::warn!(encoder = name, "encoder cannot follow the bandwidth estimate"); - Err(name) - } - // The follower takes the target back and retries it, so a quiet grant still - // recovers. - Err(err) => { - tracing::warn!(error = %err, ?target, "failed to adjust encoder bitrate"); - follower.failed(); - Ok(None) - } + // Out of the broadcast's lock, which the probe and the encoder open would hold for as + // long as the codec takes to start. + let (target, catalog, children) = broadcast + .with_state(|state| Ok((state.broadcast.clone(), state.catalog.clone(), state.children.clone())))?; + let producer = block_on(moq_video::encode::Producer::new(&target, catalog, options))?; + children.adopt(Arc::new(MoqVideoProducer { + inner: std::sync::Mutex::new(Some(producer)), + format: input.format, + size, + bandwidth, + children: children.clone(), + })) } } @@ -1550,326 +992,10 @@ mod finish_tests { assert!(packets.iter().all(|p| !p.data.is_empty()), "an empty access unit"); assert!(encoder.receive().unwrap().is_none(), "flush readied everything"); } - - #[tokio::test] - async fn finish_ends_the_track_after_a_rate_follower_panic() { - let estimate = moq_net::bandwidth::Producer::new(); - let bandwidth = Arc::new(MoqAllocator::new(moq_net::bandwidth::Allocator::new( - estimate.consume(), - ))); - let broadcast = crate::producer::MoqBroadcastProducer::new().unwrap(); - let input = MoqVideoEncoderInput { - format: MoqVideoPixelFormat::Rgba, - width: 320, - height: 240, - framerate: crate::video::MoqVideoFrameRate { - numerator: 30, - denominator: 1, - }, - }; - let output = MoqVideoEncoderOutput { - codec: MoqVideoCodec::H264, - bitrate: Some(4_000_000), - join: None, - kind: MoqVideoEncoderKind::Software, - }; - let video = crate::video::MoqVideoProducer::new( - &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), - crate::video::MoqVideoProducerOptions { - input, - output, - track: Some("camera".into()), - bandwidth: Some(bandwidth), - lower: None, - }, - ) - .unwrap(); - let track = broadcast - .consume() - .unwrap() - .subscribe_track("camera".into(), None) - .await - .unwrap(); - // Stand in for a follower thread that panicked. - video - .inner - .lock() - .unwrap() - .as_ref() - .unwrap() - .follow - .as_ref() - .expect("published against an allocator") - .follower - .failed - .store(true, Ordering::Release); - - let result = video.finish(); - assert!(matches!(result, Err(MoqError::Task { .. })), "{result:?}"); - drop(video); - let result = track.recv_group().await.map(|group| group.is_some()); - assert!(matches!(result, Ok(false)), "the track should end cleanly: {result:?}"); - } - - /// A frame that moves the rendition to a new track moves its claim there in the same step, - /// and a finish releases that claim: no claim outlives the track it watches. - #[tokio::test] - async fn a_moved_claim_follows_its_track_and_ends_with_it() { - let estimate = moq_net::bandwidth::Producer::new(); - let bandwidth = Arc::new(MoqAllocator::new(moq_net::bandwidth::Allocator::new( - estimate.consume(), - ))); - let broadcast = crate::producer::MoqBroadcastProducer::new().unwrap(); - let input = MoqVideoEncoderInput { - format: MoqVideoPixelFormat::Rgba, - width: 320, - height: 240, - framerate: crate::video::MoqVideoFrameRate { - numerator: 30, - denominator: 1, - }, - }; - let output = MoqVideoEncoderOutput { - codec: MoqVideoCodec::H264, - bitrate: Some(1_000_000), - join: None, - kind: MoqVideoEncoderKind::Software, - }; - let video = crate::video::MoqVideoProducer::new( - &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), - crate::video::MoqVideoProducerOptions { - input, - output, - track: Some("camera".into()), - bandwidth: Some(bandwidth), - lower: None, - }, - ) - .unwrap(); - let mut larger = moq_video::encode::Config::new(640, 480, moq_video::FrameRate::new(30, 1).unwrap()); - larger.kind = moq_video::encode::Kind::Software; - - tokio::task::spawn_blocking(move || { - let picture = |width: usize, height: usize, micros: u64| MoqVideoFrame { - timestamp: MoqTimestamp { - value: micros, - scale: Some(1_000_000), - }, - data: vec![0x80; width * height * 4], - }; - video.write(picture(320, 240, 0)).unwrap(); - let before = video.reservation().expect("published against an allocator"); - // Disabled, then resumed onto a larger picture, a higher level and so a new codec string. - video - .with(|producer| { - producer.apply(moq_mux::rate::Target::Disabled)?; - producer.config = larger.clone(); - producer.size = larger.size(); - producer.apply(moq_mux::rate::Target::Enabled(moq_net::bandwidth::Rate::from_mbps(1)))?; - Ok(()) - }) - .unwrap(); - video.write(picture(640, 480, 66_666)).unwrap(); - - assert_eq!(video.track_demand().unwrap().name(), "camera/1"); - let after = video.reservation().expect("the claim moved with the track"); - assert!(!Arc::ptr_eq(&before, &after), "the claim still watches the ended track"); - video.finish().unwrap(); - assert!(video.reservation().is_none(), "a finished track still holds a claim"); - }) - .await - .unwrap(); - } - - /// A disable that fails partway leaves the catalog and the encoder agreeing, so viewers - /// never wait on a rendition the catalog calls enabled while its frames are dropped. - #[tokio::test] - async fn a_failed_disable_leaves_the_catalog_and_encoder_agreeing() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let config = moq_mux::catalog::Config::default() - .with_catalog(moq_mux::catalog::hang::Catalog::::default()); - let catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); - let mut encode = moq_video::encode::Config::new(320, 240, moq_video::FrameRate::new(30, 1).unwrap()); - encode.kind = moq_video::encode::Kind::Software; - let rendition = encode.probe().await.unwrap(); - let track = broadcast - .create_track("camera", catalog.track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let producer = moq_video::encode::Publisher::new(track.clone(), catalog.reserve(), rendition).unwrap(); - let encoder = moq_video::encode::Sink::open(&encode).await.unwrap(); - let mut video = VideoProducer { - encoder: Some(encoder), - producer, - config: encode.clone(), - format: MoqVideoPixelFormat::I420, - size: encode.size(), - target: moq_mux::rate::Target::Enabled(moq_net::bandwidth::Rate::from_mbps(1)), - ceiling: kio::Producer::new(moq_net::bandwidth::Rate::from_mbps(1)), - priority: hang::catalog::PRIORITY.video, - follow: None, - retuned: Arc::default(), - }; - let (video, disabled) = tokio::task::spawn_blocking(move || { - video - .write(MoqVideoFrame { - timestamp: MoqTimestamp { - value: 0, - scale: Some(1_000_000), - }, - data: vec![0x80; 320 * 240 * 3 / 2], - }) - .unwrap(); - // The track breaks, so the break marker the disable writes fails. - track.abort(moq_net::Error::Cancel).unwrap(); - let disabled = video.apply(moq_mux::rate::Target::Disabled); - (video, disabled) - }) - .await - .unwrap(); - assert!(disabled.is_err(), "the break marker should fail on a broken track"); - let enabled = catalog.snapshot().video.renditions["camera"].enabled; - assert_eq!(video.encoder.is_some(), enabled, "the catalog and the encoder disagree"); - } -} - -#[cfg(test)] -mod follow_tests { - use std::time::Duration; - - use moq_mux::rate::{Grant, Policy, Target}; - use moq_net::bandwidth::Rate; - - use super::*; - - /// Retune through `set`, which only ever sees an enabled target: these policies set no - /// lower rendition, so nothing disables. - fn rate( - follower: &mut Grant, - ceiling: Rate, - target: Target, - set: impl FnOnce(Rate) -> Result<(), moq_video::Error>, - ) -> Result, &'static str> { - retune(follower, Policy::new(ceiling), target, |target| match target { - Target::Enabled(bitrate) => set(bitrate), - Target::Disabled => unreachable!("no lower rendition"), - }) - } - - /// A lower rendition moving between a target and its apply drops that target, and the - /// follower asks for it again under the new policy rather than counting it applied. - #[tokio::test(start_paused = true)] - async fn a_moved_lower_retries_the_dropped_target() { - let policy = |lower| { - let mut policy = Policy::new(Rate::from_mbps(4)); - policy.lower = Some(lower); - policy - }; - let grant = moq_net::bandwidth::Producer::new(); - let mut follower = Grant::new(grant.consume(), policy(Rate::from_mbps(1))); - grant.set(Some(Rate::from_mbps(2))).unwrap(); - let dropped = follower.next().await.unwrap(); - let moved = retune(&mut follower, policy(Rate::from_kbps(900)), dropped, |_| { - unreachable!("a target under a stale policy is never applied") - }); - assert_eq!(moved, Ok(None)); - - let retried = kio::clock::timeout(Duration::from_secs(60), follower.next()) - .await - .expect("the dropped target is asked for again"); - assert_eq!(retried.unwrap(), Target::Enabled(Rate::from_mbps(2))); - } - - fn bitrate(target: Target) -> Rate { - let Target::Enabled(bitrate) = target else { - unreachable!("no lower rendition") - }; - bitrate - } - - /// Retune at every target until the follower goes a minute without one. - async fn quiet(follower: &mut Grant, ceiling: Rate, mut set: impl FnMut(Rate) -> Result<(), moq_video::Error>) { - while let Ok(bitrate) = kio::clock::timeout(Duration::from_secs(60), follower.next()).await { - rate(follower, ceiling, bitrate.unwrap(), &mut set).unwrap(); - } - } - - /// The grant recovers once and never changes again, and the final raise fails once: - /// the follower still carries the encoder back to its ceiling. - #[tokio::test(start_paused = true)] - async fn a_quiet_grant_retries_a_failed_raise() { - let ceiling = Rate::from_mbps(4); - let grant = moq_net::bandwidth::Producer::new(); - let mut follower = Grant::new(grant.consume(), Policy::new(ceiling)); - let mut applied = Vec::new(); - let mut refuse = Some(ceiling); - let mut set = |bitrate| { - if refuse.take_if(|refused| *refused == bitrate).is_some() { - return Err(moq_video::Error::Unsupported("a transient refusal".into())); - } - applied.push(bitrate); - Ok(()) - }; - - grant.set(Some(Rate::from_mbps(1))).unwrap(); - let bitrate = follower.next().await.unwrap(); - rate(&mut follower, ceiling, bitrate, &mut set).unwrap(); - tokio::time::advance(Duration::from_secs(1)).await; - grant.set(Some(ceiling)).unwrap(); - quiet(&mut follower, ceiling, &mut set).await; - - assert_eq!(refuse, None, "the final raise was never refused"); - assert_eq!(applied.last(), Some(&ceiling), "{applied:?}"); - } - - /// `set_bitrate` lowers the ceiling while a raise toward the old one is - /// due. The ceiling is read under the encoder lock, so the raise is dropped rather - /// than retuning above the new cap, and the encoder recovers to the new cap. - #[tokio::test(start_paused = true)] - async fn a_lowered_ceiling_beats_a_due_raise() { - let old = Rate::from_mbps(4); - let lowered = Rate::from_mbps(2); - let grant = moq_net::bandwidth::Producer::new(); - let mut follower = Grant::new(grant.consume(), Policy::new(old)); - let applied = std::cell::RefCell::new(Vec::new()); - let set = |bitrate| { - applied.borrow_mut().push(bitrate); - Ok(()) - }; - - grant.set(Some(Rate::from_mbps(1))).unwrap(); - assert_eq!(follower.next().await.unwrap(), Target::Enabled(Rate::from_mbps(1))); - tokio::time::advance(Duration::from_secs(1)).await; - grant.set(Some(old)).unwrap(); - let first = follower.next().await.unwrap(); - rate(&mut follower, old, first, set).unwrap(); - - // `set_bitrate` already retuned the encoder to the new cap. - let due = follower.next().await.unwrap(); - assert!(bitrate(due) > bitrate(first), "a raise toward the old ceiling"); - assert_eq!(rate(&mut follower, lowered, due, set), Ok(None)); - assert_eq!(follower.policy().max, lowered); - quiet(&mut follower, lowered, set).await; - assert_eq!(*applied.borrow(), [bitrate(first)], "nothing retuned past the new cap"); - - // A later attack and recovery climbs back to the new cap, not the old one. - grant.set(Some(Rate::from_mbps(1))).unwrap(); - let bitrate = follower.next().await.unwrap(); - rate(&mut follower, lowered, bitrate, set).unwrap(); - tokio::time::advance(Duration::from_secs(1)).await; - grant.set(Some(old)).unwrap(); - quiet(&mut follower, lowered, set).await; - - let applied = applied.into_inner(); - assert_eq!(applied.last(), Some(&lowered), "{applied:?}"); - assert!(applied.iter().all(|bitrate| *bitrate <= lowered), "{applied:?}"); - } } #[cfg(test)] mod rung_tests { - use moq_mux::rate::Target; - use moq_net::bandwidth::Rate; - use super::*; fn input() -> MoqVideoEncoderInput { @@ -1893,16 +1019,6 @@ mod rung_tests { } } - fn frame(i: u64) -> MoqVideoFrame { - MoqVideoFrame { - timestamp: MoqTimestamp { - value: i * 33_333, - scale: Some(1_000_000), - }, - data: vec![(i * 40) as u8; 320 * 240 * 3 / 2], - } - } - struct Ladder { broadcast: Arc, low: Arc, @@ -1933,12 +1049,6 @@ mod rung_tests { Ladder { broadcast, low, high } } - fn enabled(broadcast: &crate::producer::MoqBroadcastProducer, track: &str) -> bool { - broadcast - .with_state(|state| Ok(state.catalog.snapshot().video.renditions[track].enabled)) - .unwrap() - } - async fn priority(broadcast: &crate::producer::MoqBroadcastProducer, track: &str) -> i8 { let track = broadcast .consume() @@ -1975,7 +1085,7 @@ mod rung_tests { lower: Some(ladder.low.clone()), }, ); - assert!(matches!(below, Err(MoqError::Video { .. }))); + assert!(below.is_err()); let ungranted = MoqVideoProducer::new( &catalog, MoqVideoProducerOptions { @@ -1986,145 +1096,11 @@ mod rung_tests { lower: Some(ladder.high.clone()), }, ); - assert!(matches!(ungranted, Err(MoqError::Video { .. }))); - } - - /// A rung's follower wakes on its lower rendition's own ceiling, whether that moves or - /// ends, so it notices even when the lower rendition reserves nothing it shares. - #[tokio::test] - async fn a_rung_wakes_when_its_unreserved_lower_moves_or_ends() { - struct Flag(AtomicBool); - impl std::task::Wake for Flag { - fn wake(self: Arc) { - self.0.store(true, Ordering::SeqCst); - } - } - - let broadcast = crate::producer::MoqBroadcastProducer::new().unwrap(); - let catalog = crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(); - let low = MoqVideoProducer::new( - &catalog, - MoqVideoProducerOptions { - input: input(), - output: output(500_000), - track: Some("low".into()), - bandwidth: None, - lower: None, - }, - ) - .unwrap(); - let lower = Some(low.inner.lock().unwrap().as_ref().unwrap().ceiling.consume()); - - let flag = Arc::new(Flag(AtomicBool::new(false))); - let waiter = kio::Waiter::new(std::task::Waker::from(flag.clone())); - assert_eq!(lower_ceiling(&lower, &waiter), Some(Rate::from_kbps(500))); - - low.set_bitrate(400_000).unwrap(); - assert!(flag.0.swap(false, Ordering::SeqCst), "a moved ceiling wakes the rung"); - assert_eq!(lower_ceiling(&lower, &waiter), Some(Rate::from_kbps(400))); - - low.finish().unwrap(); - assert!(flag.0.load(Ordering::SeqCst), "an ended lower rendition wakes the rung"); - assert_eq!(lower_ceiling(&lower, &waiter), None); - } - - /// A rung resumed at a ceiling past its declared level publishes a new track, and its - /// bandwidth claim moves there with it rather than watching the track that ended. - #[tokio::test] - async fn a_rung_resumed_past_its_level_reserves_on_the_new_track() { - let ladder = ladder(); - ladder.high.write(frame(0)).unwrap(); - let before = ladder.high.reservation().unwrap(); - let apply = |target| { - let mut guard = ladder.high.inner.lock().unwrap(); - guard.as_mut().unwrap().apply(target).unwrap(); - }; - apply(Target::Disabled); - // 3 Mbps takes H.264 level 2.1, past the 2.0 the 2 Mbps rung declared. - ladder.high.set_bitrate(3_000_000).unwrap(); - apply(Target::Enabled(Rate::from_kbps(3_000))); - ladder.high.write(frame(1)).unwrap(); - + assert!(ungranted.is_err()); let names = ladder .broadcast - .with_state(|state| { - Ok(state - .catalog - .snapshot() - .video - .renditions - .into_keys() - .collect::>()) - }) - .unwrap(); - assert!(names.contains(&"high/1".to_string()), "{names:?} has no new track"); - let after = ladder.high.reservation().unwrap(); - assert!(!Arc::ptr_eq(&before, &after), "the claim still watches the ended track"); - ladder.high.finish().unwrap(); - } - - /// A disabled rung advertises itself disabled, drops the frames written to it, and - /// resumes on a keyframe in a new group, its advertised maximum untouched throughout. - #[tokio::test] - async fn a_disabled_rung_stops_encoding_until_it_resumes() { - let ladder = ladder(); - let track = ladder - .broadcast - .consume() - .unwrap() - // Every group, however old, since the track is read only once it has ended. - .subscribe_track( - "high".into(), - Some(crate::consumer::MoqSubscription { - delay: crate::consumer::MoqDelay { - min: Some(std::time::Duration::ZERO), - max: None, - }, - ..crate::consumer::moq_subscription_defaults() - }), - ) - .await + .with_state(|state| Ok(state.catalog.snapshot().video.renditions.len())) .unwrap(); - ladder.high.write(frame(0)).unwrap(); - let bitrate = |ladder: &Ladder| { - ladder - .broadcast - .with_state(|state| Ok(state.catalog.snapshot().video.renditions["high"].bitrate)) - .unwrap() - }; - let advertised = bitrate(&ladder); - - let apply = |target| { - let mut guard = ladder.high.inner.lock().unwrap(); - guard.as_mut().unwrap().apply(target).unwrap(); - }; - apply(Target::Disabled); - assert!(!enabled(&ladder.broadcast, "high")); - assert!(enabled(&ladder.broadcast, "low"), "only the rung disables"); - for i in 1..10 { - ladder.high.write(frame(i)).unwrap(); - } - ladder.high.cut().unwrap(); - // Within the level the rung declared: a ceiling past it reopens the encoder at a new - // level, which is a new codec string and so a new track. - ladder.high.set_bitrate(1_500_000).unwrap(); - - apply(Target::Enabled(Rate::from_kbps(1_500))); - assert!(enabled(&ladder.broadcast, "high")); - ladder.high.write(frame(10)).unwrap(); - ladder.high.finish().unwrap(); - assert_eq!(bitrate(&ladder), advertised, "the advertised maximum never moves"); - - // The opening keyframe, the disable's discontinuity marker, then the resumed - // keyframe, each its own group: nothing written while disabled was encoded. - let mut groups = Vec::new(); - while let Some(group) = track.recv_group().await.unwrap() { - let mut frames = 0; - while group.read_frame().await.unwrap().is_some() { - frames += 1; - } - groups.push(frames); - } - assert_eq!(groups, [1, 1, 1]); + assert_eq!(names, 2, "a refused rung advertised a track"); } } diff --git a/rs/moq-video/src/encode/backend/probe.rs b/rs/moq-video/src/encode/backend/probe.rs index 90eec4fc5f1..68fd40d51b9 100644 --- a/rs/moq-video/src/encode/backend/probe.rs +++ b/rs/moq-video/src/encode/backend/probe.rs @@ -36,6 +36,9 @@ static BITRATES: Mutex> = Mutex::new(Vec::new()); /// A bitrate the probe refuses once, standing in for a transient codec failure. static REFUSE: Mutex> = Mutex::new(None); +/// Whether the next flush fails, standing in for a codec that loses its tail. +static LOSE_TAIL: Mutex = Mutex::new(false); + /// Serializes the tests that read [`LOG`], which is process-wide. nextest gives /// each test its own process, but `cargo test` does not, and a shared log that /// only holds up under one runner is a trap for whoever adds the next test. @@ -47,6 +50,7 @@ pub(crate) fn exclusive() -> std::sync::MutexGuard<'static, ()> { let _ = take(); BITRATES.lock().unwrap().clear(); REFUSE.lock().unwrap().take(); + *LOSE_TAIL.lock().unwrap() = false; guard } @@ -81,6 +85,11 @@ pub(crate) fn refuse_once(bitrate: u64) { *REFUSE.lock().unwrap() = Some(bitrate); } +/// Fail the next flush, losing the frame held back. +pub(crate) fn lose_tail() { + *LOSE_TAIL.lock().unwrap() = true; +} + pub(crate) struct Probe { /// The frame held back by one. held: Option, @@ -136,6 +145,9 @@ impl Backend for Probe { fn flush(&mut self) -> Result<(), Error> { record("flush"); + if std::mem::take(&mut *LOSE_TAIL.lock().unwrap()) { + return Err(Error::codec(anyhow::anyhow!("the probe lost its tail"))); + } self.ready.extend(self.held.take()); Ok(()) } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 7ad32b14822..faa3ecdc4d4 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -950,4 +950,148 @@ mod tests { assert_eq!(read(consumer).await, [1, 1, 1]); } + + /// A rung resumed at a ceiling past its declared level publishes a new track, and its + /// bandwidth claim moves there with it rather than watching the track that ended. + #[tokio::test] + #[cfg(feature = "openh264")] + async fn a_resume_past_the_declared_level_moves_the_claim() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut options = Options::new(config(encoder::Kind::Software, Rate::from_mbps(2))); + options.track = Some("video".into()); + let mut producer = Producer::new(&broadcast, catalog, options).await.unwrap(); + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + let before = producer.reservation().clone(); + producer.apply(rate::Target::Disabled).await.unwrap(); + // 3 Mbps takes H.264 level 2.1, past the 2.0 the 2 Mbps rendition declared. + producer.set_bitrate(Rate::from_mbps(3)).await.unwrap(); + producer.apply(rate::Target::Enabled(Rate::from_mbps(3))).await.unwrap(); + producer.write(frame(&mut micros)).await.unwrap(); + + assert_eq!(producer.demand().name(), "video/1"); + assert!( + !Arc::ptr_eq(&before, producer.reservation()), + "the claim still watches the ended track" + ); + let _finished = producer.finish().await.unwrap(); + assert_eq!(Arc::strong_count(&before), 1, "a finished producer still holds a claim"); + } + + /// A disable that fails partway leaves the catalog and the encoder agreeing, so viewers + /// never wait on a rendition the catalog calls enabled while its frames are dropped. + #[tokio::test] + #[cfg(feature = "openh264")] + async fn a_failed_disable_leaves_the_catalog_and_encoder_agreeing() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let config = config(encoder::Kind::Software, Rate::from_mbps(1)); + let rendition = config.probe().await.unwrap(); + let unlimited = moq_net::bandwidth::Allocator::unlimited(); + let mut producer = Producer::with_track(track.clone(), &catalog, config, rendition, unlimited, None).unwrap(); + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + // The track breaks, so the break marker the disable writes fails. + track.abort(moq_net::Error::Cancel).unwrap(); + let disabled = producer.apply(rate::Target::Disabled).await; + + assert!(disabled.is_err(), "the break marker should fail on a broken track"); + let encoding = matches!(producer.encoding, Encoding::Open(_)); + assert_eq!( + encoding, + enabled(&catalog, "video"), + "the catalog and the encoder disagree" + ); + } + + /// A lost tail must reach the subscriber as an abort: finishing the track anyway would + /// report a truncated stream as a complete one, and only the publisher would ever know. + #[tokio::test] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_lost_tail_aborts_the_track() { + let _probe = probe::exclusive(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let unlimited = moq_net::bandwidth::Allocator::unlimited(); + let mut producer = probed(&broadcast, &catalog, "video", Rate::from_mbps(1), unlimited, None); + let mut track = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut micros = 0; + + // The probe holds this frame back, so only the flush can publish it. + producer.write(frame(&mut micros)).await.unwrap(); + probe::lose_tail(); + assert!(producer.finish().await.is_err()); + + let Err(err) = track.recv_group().await else { + panic!("expected an abort, not a clean end"); + }; + assert!( + err.to_string().contains("the probe lost its tail"), + "the abort should carry the drain failure: {err}" + ); + } + + /// `set_bitrate` lowers the ceiling while the grant is climbing back toward the old one: + /// nothing retunes past the new ceiling, and the encoder recovers to it. + #[tokio::test(start_paused = true)] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_lowered_ceiling_caps_a_recovering_grant() { + let _probe = probe::exclusive(); + let old = Rate::from_mbps(4); + let lowered = Rate::from_mbps(2); + let estimate = moq_net::bandwidth::Producer::new(); + estimate.set(Some(Rate::from_mbps(1))).unwrap(); + let bandwidth = moq_net::bandwidth::Allocator::new(estimate.consume()); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut producer = probed(&broadcast, &catalog, "video", old, bandwidth, None); + let _viewer = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + assert!(producer.bitrate().unwrap() <= Rate::from_mbps(1)); + // The link recovers, and the encoder starts climbing back toward the old ceiling. + tokio::time::advance(Duration::from_secs(1)).await; + estimate.set(Some(old)).unwrap(); + for _ in 0..3 { + tokio::time::advance(Duration::from_millis(100)).await; + producer.write(frame(&mut micros)).await.unwrap(); + } + + producer.set_bitrate(lowered).await.unwrap(); + let _ = probe::bitrates(); + for _ in 0..600 { + tokio::time::advance(Duration::from_millis(100)).await; + producer.write(frame(&mut micros)).await.unwrap(); + } + let applied = probe::bitrates(); + assert!(applied.iter().all(|&bps| bps <= lowered.as_bps()), "{applied:?}"); + assert_eq!(producer.bitrate(), Some(lowered)); + assert_eq!(producer.reservation().rate(), Some(lowered)); + } } From f0f77109e05ca76a72d2a5a463038eb25bb018f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 13:56:57 -0800 Subject: [PATCH 05/14] fix(moq-video): a manual retune hands the encoder back to the grant set_bitrate to an unchanged ceiling left the grant at its old target, so a producer adapted below the ceiling stayed at the ceiling. moq-boy aborts its producer with the cause when it stops, and the docs say grant following needs the connection's allocator. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-video.md | 3 +- rs/moq-boy/src/video.rs | 3 ++ rs/moq-video/src/encode/producer.rs | 48 ++++++++++++++++++++++++++++- rs/moq-video/src/lib.rs | 4 ++- 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index ac59ccc7061..be702f43999 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -44,7 +44,8 @@ watches, and return to the configured bitrate while nobody does. To publish frames from your own source, `encode::Producer` takes raw frames, owns the encoder, and follows the same send estimate between frames, like -`moq_audio::encode::Producer`. Chain renditions through `Options::lower` for +`moq_audio::encode::Producer`, once `Options::bandwidth` carries the +connection's allocator; the default holds the configured bitrate. Chain renditions through `Options::lower` for simulcast: a higher rung stops encoding and advertises itself disabled while the connection cannot fund it, then resumes on a keyframe. `encode::Publisher` is the layer below, for frames you encoded yourself. Where the platform diff --git a/rs/moq-boy/src/video.rs b/rs/moq-boy/src/video.rs index 3fbf367772e..7825691c66b 100644 --- a/rs/moq-boy/src/video.rs +++ b/rs/moq-boy/src/video.rs @@ -146,6 +146,7 @@ async fn encoder_thread( && let Err(e) = producer.cut().await { tracing::error!(error = %e, "H.264 encoder cannot cut a group; stopping encoder"); + producer.abort(moq_net::Error::Transport(e.to_string())); return; } let start = Instant::now(); @@ -161,6 +162,8 @@ async fn encoder_thread( // track is gone, which is terminal: stop rather than flooding logs every frame. if let Err(e) = producer.write(moq_video::Frame::new(surface, ts)).await { tracing::error!(error = %e, "video encode failed; stopping encoder"); + // Subscribers see the cause rather than an unfinished track's `Dropped`. + producer.abort(moq_net::Error::Transport(e.to_string())); return; } encode_duration.store(start.elapsed().as_micros() as u64, Ordering::Relaxed); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index faa3ecdc4d4..0267553b555 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -315,9 +315,13 @@ impl Producer { *ceiling = bitrate; } self.reservation.update(bitrate); - // A new ceiling starts the grant over from it, since the encoder applied it itself. + // The grant follows from the rate the encoder applied itself, even when the ceiling is + // unchanged and the policy with it, so a grant below it retunes back down. if let Some(grant) = self.grant.as_mut() { grant.set_policy(policy(bitrate, self.lower.as_ref())); + if !matches!(self.encoding, Encoding::Disabled) { + grant.assume(rate::Target::Enabled(bitrate)); + } } Ok(()) } @@ -1094,4 +1098,46 @@ mod tests { assert_eq!(producer.bitrate(), Some(lowered)); assert_eq!(producer.reservation().rate(), Some(lowered)); } + + /// A manual retune to the unchanged ceiling hands the encoder back to the grant rather than + /// leaving it at the ceiling the grant cannot fund. + #[tokio::test(start_paused = true)] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_manual_retune_returns_to_the_grant() { + let _probe = probe::exclusive(); + let ceiling = Rate::from_mbps(4); + let estimate = moq_net::bandwidth::Producer::new(); + estimate.set(Some(Rate::from_mbps(1))).unwrap(); + let bandwidth = moq_net::bandwidth::Allocator::new(estimate.consume()); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut producer = probed(&broadcast, &catalog, "video", ceiling, bandwidth, None); + let _viewer = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut micros = 0; + + producer.write(frame(&mut micros)).await.unwrap(); + assert!(producer.bitrate().unwrap() <= Rate::from_mbps(1)); + + producer.set_bitrate(ceiling).await.unwrap(); + assert_eq!(producer.bitrate(), Some(ceiling)); + for _ in 0..10 { + tokio::time::advance(Duration::from_millis(100)).await; + producer.write(frame(&mut micros)).await.unwrap(); + } + assert!( + producer.bitrate().unwrap() <= Rate::from_mbps(1), + "{:?}", + producer.bitrate() + ); + } } diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 3e4922c7e79..6eb39c6c00b 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -33,7 +33,9 @@ //! limits it to while a subscriber is. Requires the `capture` feature. //! - [`encode::Producer`] encodes [`Frame`]s you supply (from capture, a //! decoder, or your own pixels via [`Surface::rgba`]) and publishes them, -//! following the connection's bandwidth grant between frames. +//! following the connection's bandwidth grant between frames when +//! `encode::Options::bandwidth` carries the connection's allocator (the +//! default holds the configured bitrate). //! - [`encode::Encoder`] encodes without publishing, and //! [`encode::Publisher`] publishes frames you encoded yourself. //! - [`decode`] subscribes to an H.264, H.265, or AV1 track and decodes it to From 193363a82d1cfe26871a7cec0f226a793b10ae70 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 14:56:56 -0800 Subject: [PATCH 06/14] fix(moq-video): a rung whose resume retune is refused waits for its full ceiling Maintainer decision on #8964: rather than open at a ceiling the grant cannot fund, the rung stays disabled and from then on resumes only once the grant reaches its full ceiling, which needs no retune. Adds the follow-up quest for an unwatched rendition to rest. Co-Authored-By: Claude Opus 5.5 --- quest/m3/video-unwatched-rest.md | 39 +++++++++++++++ rs/moq-video/src/encode/producer.rs | 77 +++++++++++++++++++++++++++-- 2 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 quest/m3/video-unwatched-rest.md diff --git a/quest/m3/video-unwatched-rest.md b/quest/m3/video-unwatched-rest.md new file mode 100644 index 00000000000..aaf4c2ebcfa --- /dev/null +++ b/quest/m3/video-unwatched-rest.md @@ -0,0 +1,39 @@ +# [S] An unwatched video rendition rests + +## Goal + +A `moq_video::encode::Producer` rendition with no `lower` stops spending on +frames nobody watches: while its track has no subscriber it does not encode at +its full ceiling, and the next viewer still joins on a keyframe without a burst +of frames encoded at a rate the grant would not have funded. + +## Plan + +Accepted 2026-11-16 (maintainer, on +[moq-dev/moq#8964](https://github.com/moq-dev/moq/pull/8964#issuecomment-6931517209)): +#8964 lifted the capture driver's rule into `encode::Producer`, so moq-ffi's +`MoqVideoProducer`, moq-c's `moq_encode_video*` and moq-boy now also encode an +unwatched rendition at its ceiling (`Producer::rest` in +`rs/moq-video/src/encode/producer.rs`), where the bindings used to follow the +grant regardless. Its groups wait in the track cache, so the cost is encoder +work and a join that starts on a group encoded at the ceiling. m3, as a fix to +behaviour #8964 shipped. + +Recommended: while unwatched, close the encoder like a disabled rung but keep +the rendition advertised enabled; the first write after a subscriber arrives +reopens it, which opens on a keyframe on every backend, at the rate the grant +then funds. A simulcast rung keeps following the grant as it does now, since it +needs the grant's slice to resume. `publish_capture` keeps reporting its state. + +Tests: an unwatched producer encodes nothing and holds no encoder; a viewer +arriving gets a group starting on a keyframe at the granted rate; a viewer +leaving and returning does the same. + +## Waiting + +- Maintainer: how an unwatched rendition rests. Options: (1) recommended, close + the encoder until a subscriber arrives, which saves the most and costs the + first viewer one encoder open; (2) keep encoding at the policy floor (a tenth + of the ceiling), which keeps a group cached for an instant join but still + encodes every frame; (3) keep encoding at the last granted rate, which is + what the bindings did before #8964. diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 0267553b555..b6f66cdfb78 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -142,6 +142,9 @@ pub struct Producer { /// Whether nothing has been published since the last break marker, which a second break /// would only repeat. marked: bool, + /// Whether a resume's encoder refused the funded rate, so this rung now resumes only at its + /// full ceiling, which needs no retune, rather than overshoot a partial grant. + ceiling_only: bool, } impl Producer { @@ -226,6 +229,7 @@ impl Producer { target: rate::Target::Enabled(ceiling), watched: false, marked: false, + ceiling_only: false, }) } @@ -613,12 +617,27 @@ impl Producer { self.publisher.report_bitrate(bitrate); Ok(()) } + // A resume. Left disabled, the grant ramps on as if it were on, and the + // target that reaches the ceiling resumes it. (rate::Target::Enabled(bitrate), Encoding::Disabled) => { - let sink = self.reopen(bitrate).await?; - if let Err(err) = self.publisher.set_enabled(true) { - self.target = rate::Target::Disabled; - return Err(err); + let ceiling = *self.ceiling.read(); + let bitrate = bitrate.min(ceiling); + if self.ceiling_only && bitrate < ceiling { + return Ok(()); } + let mut sink = Sink::open(&self.config).await?; + if bitrate < ceiling + && let Err(err) = sink.set_bitrate(bitrate).await + { + // Opening at the ceiling would overshoot the grant, and retrying would reopen + // an encoder every ramp step on a backend that can never retune. + tracing::warn!(error = %err, bitrate = bitrate.as_bps(), "resumed encoder refused the funded bitrate; resuming only at the ceiling"); + self.ceiling_only = true; + return Ok(()); + } + self.publisher.set_enabled(true)?; + self.target = rate::Target::Enabled(bitrate); + self.publisher.report_bitrate(bitrate); self.encoding = Encoding::Open(sink); Ok(()) } @@ -848,6 +867,56 @@ mod tests { let _low = low.finish().await.unwrap(); } + /// A rung whose resumed encoder refuses the funded rate stays disabled rather than open at a + /// ceiling the grant cannot fund, and from then on resumes only at its full ceiling. + #[tokio::test] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_refused_resume_waits_for_the_ceiling() { + let _probe = probe::exclusive(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let unlimited = moq_net::bandwidth::Allocator::unlimited(); + let low = probed( + &broadcast, + &catalog, + "low", + Rate::from_kbps(500), + unlimited.clone(), + None, + ); + let ceiling = Rate::from_mbps(2); + let mut high = probed(&broadcast, &catalog, "high", ceiling, unlimited, Some(low.rung())); + let mut micros = 0; + + high.write(frame(&mut micros)).await.unwrap(); + high.apply(rate::Target::Disabled).await.unwrap(); + let _ = probe::take(); + + let funded = Rate::from_mbps(1); + probe::refuse_once(funded.as_bps()); + high.apply(rate::Target::Enabled(funded)).await.unwrap(); + assert!(!enabled(&catalog, "high"), "a refused resume stays disabled"); + assert_eq!(high.bitrate(), None); + + // A grant that would now retune fine still funds less than the ceiling. + let _ = probe::take(); + high.apply(rate::Target::Enabled(funded)).await.unwrap(); + assert!(!enabled(&catalog, "high")); + assert_eq!(events("open"), 0, "a partial grant no longer reopens the encoder"); + + high.apply(rate::Target::Enabled(ceiling)).await.unwrap(); + assert!(enabled(&catalog, "high")); + assert_eq!(high.bitrate(), Some(ceiling)); + let resumed = probe::take(); + assert!(resumed.iter().all(|(event, _)| *event != "set_bitrate"), "{resumed:?}"); + + let _high = high.finish().await.unwrap(); + let _low = low.finish().await.unwrap(); + } + /// A rung must sit above the rendition it falls back to: one that does not fails before its /// track exists. #[tokio::test] From 5d2dde6c2477fb42b83b982430fc3cefb48359a2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 17:33:47 -0800 Subject: [PATCH 07/14] fix(moq-video): a deferred rung resume keeps the grant in step Maintainer decision on #8964 (review round 3): a rung that skips a partial resume tells the grant it is off, and resumes at its ceiling on the frame its slice covers it, so lowering the ceiling can no longer strand it. Records the unwatched-rest decision in its follow-up quest. Co-Authored-By: Claude Opus 5.5 --- quest/m3/video-unwatched-rest.md | 17 ++--- rs/moq-video/src/encode/producer.rs | 106 +++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 12 deletions(-) diff --git a/quest/m3/video-unwatched-rest.md b/quest/m3/video-unwatched-rest.md index aaf4c2ebcfa..5800bf7e63f 100644 --- a/quest/m3/video-unwatched-rest.md +++ b/quest/m3/video-unwatched-rest.md @@ -19,7 +19,13 @@ grant regardless. Its groups wait in the track cache, so the cost is encoder work and a join that starts on a group encoded at the ceiling. m3, as a fix to behaviour #8964 shipped. -Recommended: while unwatched, close the encoder like a disabled rung but keep +Decided 2026-11-16 (maintainer, on +[moq-dev/moq#8964](https://github.com/moq-dev/moq/pull/8964#issuecomment-6932840632)): +✅ rest by closing the encoder until a subscriber arrives (rejected: encoding +at the policy floor, which still encodes every frame; encoding at the last +granted rate, the bindings' behaviour before #8964). + +While unwatched, close the encoder like a disabled rung but keep the rendition advertised enabled; the first write after a subscriber arrives reopens it, which opens on a keyframe on every backend, at the rate the grant then funds. A simulcast rung keeps following the grant as it does now, since it @@ -28,12 +34,3 @@ needs the grant's slice to resume. `publish_capture` keeps reporting its state. Tests: an unwatched producer encodes nothing and holds no encoder; a viewer arriving gets a group starting on a keyframe at the granted rate; a viewer leaving and returning does the same. - -## Waiting - -- Maintainer: how an unwatched rendition rests. Options: (1) recommended, close - the encoder until a subscriber arrives, which saves the most and costs the - first viewer one encoder open; (2) keep encoding at the policy floor (a tenth - of the ceiling), which keeps a group cached for an instant join but still - encodes every frame; (3) keep encoding at the last granted rate, which is - what the bindings did before #8964. diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index b6f66cdfb78..2d94b7fd3cd 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -539,6 +539,8 @@ impl Producer { } } + self.resume_at_ceiling().await; + let noop = kio::Waiter::noop(); // Each target is one the grant would not hand out again without a new grant, a deadline, // or a failure, all of which leave it pending until the next write. @@ -562,10 +564,43 @@ impl Producer { } return; } + // A deferred resume told the grant the rung is still off, so it would only offer the + // same resume again: wait for the next write. + if matches!(target, rate::Target::Enabled(_)) && matches!(self.encoding, Encoding::Disabled) { + return; + } tracing::debug!(?target, "followed the bandwidth grant"); } } + /// Resume a rung that only resumes at its ceiling, once its slice of the grant covers it. + async fn resume_at_ceiling(&mut self) { + if !self.ceiling_only || self.grant.is_none() || !matches!(self.encoding, Encoding::Disabled) { + return; + } + let ceiling = *self.ceiling.read(); + if !self.reservation.rate().is_some_and(|rate| rate >= ceiling) { + return; + } + let target = rate::Target::Enabled(ceiling); + match self.apply(target).await { + Ok(()) => { + if let Some(grant) = self.grant.as_mut() { + grant.assume(target); + } + } + Err(err) => tracing::warn!(error = %err, bitrate = ceiling.as_bps(), "failed to resume at the ceiling"), + } + } + + /// Leave a rung disabled and tell the grant so, which then offers its resume again from the + /// current estimate. + fn defer_resume(&mut self) { + if let Some(grant) = self.grant.as_mut() { + grant.assume(rate::Target::Disabled); + } + } + /// Return an encoder nobody watches to its ceiling, and start its rate control over from /// there for the next viewer. async fn rest(&mut self) { @@ -617,12 +652,13 @@ impl Producer { self.publisher.report_bitrate(bitrate); Ok(()) } - // A resume. Left disabled, the grant ramps on as if it were on, and the - // target that reaches the ceiling resumes it. + // A resume. One deferred leaves the grant agreeing the rung is off, and + // `resume_at_ceiling` resumes it once its slice covers the ceiling. (rate::Target::Enabled(bitrate), Encoding::Disabled) => { let ceiling = *self.ceiling.read(); let bitrate = bitrate.min(ceiling); if self.ceiling_only && bitrate < ceiling { + self.defer_resume(); return Ok(()); } let mut sink = Sink::open(&self.config).await?; @@ -633,6 +669,7 @@ impl Producer { // an encoder every ramp step on a backend that can never retune. tracing::warn!(error = %err, bitrate = bitrate.as_bps(), "resumed encoder refused the funded bitrate; resuming only at the ceiling"); self.ceiling_only = true; + self.defer_resume(); return Ok(()); } self.publisher.set_enabled(true)?; @@ -917,6 +954,71 @@ mod tests { let _low = low.finish().await.unwrap(); } + /// A rung waiting for its ceiling resumes from the grant: not while its slice covers only + /// part of the ceiling, and at once when the slice covers it, including after `set_bitrate` + /// lowers the ceiling to what the slice already funds. + #[tokio::test(start_paused = true)] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_deferred_resume_follows_the_grant_to_the_ceiling() { + let _probe = probe::exclusive(); + let estimate = moq_net::bandwidth::Producer::new(); + estimate.set(Some(Rate::from_kbps(1_500))).unwrap(); + let bandwidth = moq_net::bandwidth::Allocator::new(estimate.consume()); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let low = probed( + &broadcast, + &catalog, + "low", + Rate::from_kbps(500), + moq_net::bandwidth::Allocator::unlimited(), + None, + ); + let ceiling = Rate::from_mbps(2); + let mut high = probed(&broadcast, &catalog, "high", ceiling, bandwidth, Some(low.rung())); + let _viewer = broadcast + .consume() + .track("high") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut micros = 0; + + high.write(frame(&mut micros)).await.unwrap(); + high.apply(rate::Target::Disabled).await.unwrap(); + let funded = Rate::from_mbps(1); + probe::refuse_once(funded.as_bps()); + high.apply(rate::Target::Enabled(funded)).await.unwrap(); + assert_eq!(high.bitrate(), None); + let _ = probe::take(); + + // The slice funds part of the ceiling: the grant offers resumes, and none opens. + for _ in 0..30 { + tokio::time::advance(Duration::from_millis(100)).await; + high.write(frame(&mut micros)).await.unwrap(); + } + assert!(!enabled(&catalog, "high")); + assert_eq!(events("open"), 0); + + // A ceiling the slice covers resumes it on the next frame, at that ceiling. + let lowered = Rate::from_mbps(1); + high.set_bitrate(lowered).await.unwrap(); + high.write(frame(&mut micros)).await.unwrap(); + assert!(enabled(&catalog, "high")); + assert_eq!(high.bitrate(), Some(lowered)); + let resumed = probe::take(); + assert!(resumed.iter().any(|(event, _)| *event == "open"), "{resumed:?}"); + assert!(resumed.iter().all(|(event, _)| *event != "set_bitrate"), "{resumed:?}"); + + let _high = high.finish().await.unwrap(); + let _low = low.finish().await.unwrap(); + } + /// A rung must sit above the rendition it falls back to: one that does not fails before its /// track exists. #[tokio::test] From 1f83ae51d1c119ecfc98d17dafed71585668c7bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 19:10:32 -0800 Subject: [PATCH 08/14] docs(moq-video): an unwatched rendition encodes at its ceiling, outside the grant Co-Authored-By: Claude Opus 5.5 --- rs/moq-video/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 6eb39c6c00b..7103ac06442 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -35,7 +35,9 @@ //! decoder, or your own pixels via [`Surface::rgba`]) and publishes them, //! following the connection's bandwidth grant between frames when //! `encode::Options::bandwidth` carries the connection's allocator (the -//! default holds the configured bitrate). +//! default holds the configured bitrate). While nobody subscribes, a +//! rendition with no `encode::Options::lower` ignores the grant and encodes +//! at its ceiling, so a later viewer can start on a group cached at that rate. //! - [`encode::Encoder`] encodes without publishing, and //! [`encode::Publisher`] publishes frames you encoded yourself. //! - [`decode`] subscribes to an H.264, H.265, or AV1 track and decodes it to From 7300e8e4b1625ab2ce55b22898bd5cbb841ccd99 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 19:49:21 -0800 Subject: [PATCH 09/14] quest: a session's handle is gone at its terminal callback, by contract CodeRabbit (r4594947229): session.rs drops the entry before the CLOSED callback, so the plan no longer calls that release a library bug. Co-Authored-By: Claude Opus 5.5 --- .../m0/test-flakes-3/c-server-accepts-session.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/quest/m0/test-flakes-3/c-server-accepts-session.md b/quest/m0/test-flakes-3/c-server-accepts-session.md index 2511af0a645..78901c032bb 100644 --- a/quest/m0/test-flakes-3/c-server-accepts-session.md +++ b/quest/m0/test-flakes-3/c-server-accepts-session.md @@ -14,11 +14,14 @@ main 73d65edbf7: `moq_session_close(client)`, after the accepted session closed and its tail drained, returns -21 where the test expects 0. The client's session likely already ended on its own once the server side -closed, so its id is gone by the time the test closes it. Find which: if a -closed peer is meant to leave the handle valid until `moq_session_close`, as -the session's terminal callback contract suggests, the early release is a -library bug, fixed there with a regression test. Only if releasing it is the -contract does the test change, by waiting on the client's terminal callback -before it closes the handle. +closed, so its id is gone by the time the test closes it. That release is the +contract: `rs/moq-c/src/session.rs` removes a session's entry before its +terminal `CLOSED` callback, so `moq_session_close` on a handle whose terminal +callback has fired (or is about to) returns a negative code. Find why the client +ends: if a dialed client is meant to keep reconnecting after its server closes +the accepted session, its early end is a library bug, fixed there with a +regression test. Otherwise the test changes: it waits on the client's terminal +callback and drops the `moq_session_close(client) == 0` assertion, or asserts +the close is negative after that callback, as `moq_server_close` already does. Public API: none. Wire: none. From d03ca0fc8444210502f988a84caa8f1fe061bd1d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 19:51:29 -0800 Subject: [PATCH 10/14] quest: moq-c's grants parity test closes its client after the server too It fails 3 of 8 runs alone at main a79bbae919, the same shape as server_accepts_a_session. Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-3/c-server-accepts-session.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/quest/m0/test-flakes-3/c-server-accepts-session.md b/quest/m0/test-flakes-3/c-server-accepts-session.md index 78901c032bb..28493932672 100644 --- a/quest/m0/test-flakes-3/c-server-accepts-session.md +++ b/quest/m0/test-flakes-3/c-server-accepts-session.md @@ -1,8 +1,8 @@ -# [S] moq-c's server session test closes its client after the server did +# [S] moq-c's session tests close their client after the server did ## Goal -`rs/moq-c/src/test.rs::server_accepts_a_session` passes every run. +`rs/moq-c/src/test.rs::server_accepts_a_session` and `rs/moq-c/src/test/parity.rs::session_tokens_and_grants` pass every run. ## Plan @@ -24,4 +24,9 @@ regression test. Otherwise the test changes: it waits on the client's terminal callback and drops the `moq_session_close(client) == 0` assertion, or asserts the close is negative after that callback, as `moq_server_close` already does. +`session_tokens_and_grants` has the same shape: it closes the accepted session, +waits for its terminal callback, then asserts `moq_session_close(client) == 0` +(`parity.rs:530`). It fails 3 of 8 runs alone at main a79bbae919, found while +merging main into #8964. Fix both the same way. + Public API: none. Wire: none. From fccc6ea7464450fa5d35693cd880af5ddf120e6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 11:58:24 -0800 Subject: [PATCH 11/14] fix(moq-video): a failed resume at the ceiling waits a retry step A rung that resumes only at its ceiling retried a failed open on every frame. It now waits moq-mux's RETRY, as a failed grant target does. The encoder docs no longer tell a rate controller to stop adapting on BitrateUnsupported, since the producer retries instead. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/rate.rs | 2 +- rs/moq-video/src/encode/backend/probe.rs | 24 +++++++- rs/moq-video/src/encode/backend/v4l2.rs | 4 +- rs/moq-video/src/encode/encoder.rs | 3 +- rs/moq-video/src/encode/producer.rs | 77 +++++++++++++++++++++++- rs/moq-video/src/encode/sink.rs | 6 +- 6 files changed, 105 insertions(+), 11 deletions(-) diff --git a/rs/moq-mux/src/rate.rs b/rs/moq-mux/src/rate.rs index d25993fd5e2..8ae93f49109 100644 --- a/rs/moq-mux/src/rate.rs +++ b/rs/moq-mux/src/rate.rs @@ -30,7 +30,7 @@ const RAMP: Duration = Duration::from_secs(4); /// The soonest a failed change is retried: the time the ramp takes to clear the /// hysteresis band, so a sender that keeps failing is asked at most this often. -const RETRY: Duration = Duration::from_millis(RAMP.as_millis() as u64 / HYSTERESIS); +pub const RETRY: Duration = Duration::from_millis(RAMP.as_millis() as u64 / HYSTERESIS); /// How a bandwidth estimate maps onto the bitrate a sender should produce at. /// diff --git a/rs/moq-video/src/encode/backend/probe.rs b/rs/moq-video/src/encode/backend/probe.rs index 68fd40d51b9..fa2cd4d9e3b 100644 --- a/rs/moq-video/src/encode/backend/probe.rs +++ b/rs/moq-video/src/encode/backend/probe.rs @@ -36,6 +36,9 @@ static BITRATES: Mutex> = Mutex::new(Vec::new()); /// A bitrate the probe refuses once, standing in for a transient codec failure. static REFUSE: Mutex> = Mutex::new(None); +/// How many of the next opens fail, standing in for a codec that is busy or gone. +static FAIL_OPENS: Mutex = Mutex::new(0); + /// Whether the next flush fails, standing in for a codec that loses its tail. static LOSE_TAIL: Mutex = Mutex::new(false); @@ -50,6 +53,7 @@ pub(crate) fn exclusive() -> std::sync::MutexGuard<'static, ()> { let _ = take(); BITRATES.lock().unwrap().clear(); REFUSE.lock().unwrap().take(); + *FAIL_OPENS.lock().unwrap() = 0; *LOSE_TAIL.lock().unwrap() = false; guard } @@ -85,6 +89,22 @@ pub(crate) fn refuse_once(bitrate: u64) { *REFUSE.lock().unwrap() = Some(bitrate); } +/// Fail the next `count` opens. +pub(crate) fn fail_opens(count: u32) { + *FAIL_OPENS.lock().unwrap() = count; +} + +/// Record an open, failing it while [`fail_opens`] has any left. +fn opened() -> Result<(), Error> { + record("open"); + let mut failing = FAIL_OPENS.lock().unwrap(); + if *failing == 0 { + return Ok(()); + } + *failing -= 1; + Err(Error::Unsupported("probe open refused".into())) +} + /// Fail the next flush, losing the frame held back. pub(crate) fn lose_tail() { *LOSE_TAIL.lock().unwrap() = true; @@ -99,7 +119,7 @@ pub(crate) struct Probe { impl Probe { pub(crate) fn open(_config: &Config) -> Result, Error> { - record("open"); + opened()?; Ok(Box::new(Self { held: None, ready: VecDeque::new(), @@ -108,7 +128,7 @@ impl Probe { } pub(crate) fn open_no_cut(_config: &Config) -> Result, Error> { - record("open"); + opened()?; Ok(Box::new(Self { held: None, ready: VecDeque::new(), diff --git a/rs/moq-video/src/encode/backend/v4l2.rs b/rs/moq-video/src/encode/backend/v4l2.rs index 20df8cec1ae..a71413fbacb 100644 --- a/rs/moq-video/src/encode/backend/v4l2.rs +++ b/rs/moq-video/src/encode/backend/v4l2.rs @@ -595,8 +595,8 @@ impl Backend for V4l2 { fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { // Settable on a streaming encoder and applied without an IDR, which is // exactly what the congestion controller wants. A driver that refuses gets - // `BitrateUnsupported`, which tells the control loop to stop adapting - // rather than to stop encoding. + // `BitrateUnsupported`, which leaves it encoding at its current rate rather + // than stopping it. set_bitrate(&self.device, bitrate).map_err(|err| { tracing::debug!(encoder = NAME, %err, "driver refused a bitrate change"); Error::BitrateUnsupported(NAME) diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index 411864ad112..8536e77c6fa 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -475,8 +475,7 @@ impl Encoder { /// /// Returns [`Error::BitrateUnsupported`] if this backend can't retune while /// running. That's not fatal: the encoder keeps running at its current rate, - /// so a caller driving a control loop should stop adapting rather than stop - /// encoding. + /// so a caller driving a control loop keeps encoding, and may try again later. pub fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { if bitrate == self.bitrate { return Ok(()); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index e1117a40c7a..8ea9f3a3679 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -145,6 +145,8 @@ pub struct Producer { /// Whether a resume's encoder refused the funded rate, so this rung now resumes only at its /// full ceiling, which needs no retune, rather than overshoot a partial grant. ceiling_only: bool, + /// The soonest a resume at the ceiling that failed is tried again, a grant's retry step later. + resume_retry: Option, } impl Producer { @@ -230,6 +232,7 @@ impl Producer { watched: false, marked: false, ceiling_only: false, + resume_retry: None, }) } @@ -598,14 +601,23 @@ impl Producer { if !self.reservation.rate().is_some_and(|rate| rate >= ceiling) { return; } + let now = kio::clock::now(); + if self.resume_retry.is_some_and(|retry| now < retry) { + return; + } let target = rate::Target::Enabled(ceiling); match self.apply(target).await { Ok(()) => { + self.resume_retry = None; if let Some(grant) = self.grant.as_mut() { grant.assume(target); } } - Err(err) => tracing::warn!(error = %err, bitrate = ceiling.as_bps(), "failed to resume at the ceiling"), + // Retried a step later, as a failed grant target is, rather than on every frame. + Err(err) => { + tracing::warn!(error = %err, bitrate = ceiling.as_bps(), "failed to resume at the ceiling"); + self.resume_retry = now.checked_add(rate::RETRY); + } } } @@ -1035,6 +1047,69 @@ mod tests { let _low = low.finish().await.unwrap(); } + /// A resume at the ceiling whose encoder fails to open is tried again a grant's retry step + /// later, not on every frame. + #[tokio::test(start_paused = true)] + #[expect( + clippy::await_holding_lock, + reason = "the guard only keeps other tests out of the process-wide probe log, and this runtime has one thread, so holding it across an await blocks nothing" + )] + async fn a_failed_resume_waits_a_retry_step() { + let _probe = probe::exclusive(); + let estimate = moq_net::bandwidth::Producer::new(); + estimate.set(Some(Rate::from_mbps(10))).unwrap(); + let bandwidth = moq_net::bandwidth::Allocator::new(estimate.consume()); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let low = probed( + &broadcast, + &catalog, + "low", + Rate::from_kbps(500), + moq_net::bandwidth::Allocator::unlimited(), + None, + ); + let ceiling = Rate::from_mbps(2); + let mut high = probed(&broadcast, &catalog, "high", ceiling, bandwidth, Some(low.rung())); + let _viewer = broadcast + .consume() + .track("high") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut micros = 0; + + // Disabled, and resuming only at its ceiling, which the slice covers. + high.write(frame(&mut micros)).await.unwrap(); + high.apply(rate::Target::Disabled).await.unwrap(); + probe::refuse_once(Rate::from_mbps(1).as_bps()); + high.apply(rate::Target::Enabled(Rate::from_mbps(1))).await.unwrap(); + let _ = probe::take(); + + // An encoder that keeps failing to open, written to well inside one retry step: the + // ceiling resume and the grant's own resume each try once, not once per frame. + probe::fail_opens(u32::MAX); + for _ in 0..8 { + high.write(frame(&mut micros)).await.unwrap(); + tokio::time::advance(moq_mux::rate::RETRY / 10).await; + } + let opens = events("open"); + assert!((1..=2).contains(&opens), "{opens} opens in one retry step"); + assert!(!enabled(&catalog, "high")); + + // Once the encoder opens again, the next retry resumes the rung at its ceiling. + probe::fail_opens(0); + tokio::time::advance(moq_mux::rate::RETRY).await; + high.write(frame(&mut micros)).await.unwrap(); + assert!(enabled(&catalog, "high")); + assert_eq!(high.bitrate(), Some(ceiling)); + + let _high = high.finish().await.unwrap(); + let _low = low.finish().await.unwrap(); + } + /// A rung must sit above the rendition it falls back to: one that does not fails before its /// track exists. #[tokio::test] diff --git a/rs/moq-video/src/encode/sink.rs b/rs/moq-video/src/encode/sink.rs index 86b99755c8a..7fa213faf97 100644 --- a/rs/moq-video/src/encode/sink.rs +++ b/rs/moq-video/src/encode/sink.rs @@ -126,7 +126,7 @@ impl Sink { /// Retune the encoder, waiting for the backend's verdict. See /// [`Encoder::set_bitrate`](super::Encoder::set_bitrate) for what a failure - /// means (not fatal: stop adapting, keep encoding). + /// means (not fatal: the encoder keeps its rate and keeps encoding). pub async fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { self.0.set_bitrate(bitrate).await } @@ -173,8 +173,8 @@ mod threaded { /// refusal has to reach the caller, since the alternative is a group /// boundary that silently never happens. Cut { resp: oneshot::Sender> }, - /// Retune to a new bitrate, reporting whether the backend took it so the - /// caller can stop adapting against an encoder that can't. The round trip + /// Retune to a new bitrate, reporting whether the backend took it, so the + /// caller's rate control knows the rate it is encoding at. The round trip /// is affordable because the rate control policy only sends one of these /// when the target moves meaningfully, not per frame. SetBitrate { From 4703f6998c6d806ca5758ebf93a83c7e2bea5351 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 14:44:38 -0800 Subject: [PATCH 12/14] fix(moq-ffi): a video producer's reservation handle no longer holds its claim `MoqVideoProducer::reservation` handed out a strong Arc to the producer's claim, so a caller keeping the handle kept the room after `finish` or a track move. The handle now borrows the claim weakly and reads no rate once the producer releases it. Also limit set_bitrate's "not fatal" docs to BitrateUnsupported. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/bandwidth.rs | 50 +++++++++++++++++++++++------ rs/moq-ffi/src/test.rs | 45 ++++++++++++++++++++++++++ rs/moq-ffi/src/video.rs | 13 ++++---- rs/moq-video/src/encode/producer.rs | 5 +-- rs/moq-video/src/encode/sink.rs | 8 +++-- 5 files changed, 100 insertions(+), 21 deletions(-) diff --git a/rs/moq-ffi/src/bandwidth.rs b/rs/moq-ffi/src/bandwidth.rs index 20786e52332..ada2cad78a9 100644 --- a/rs/moq-ffi/src/bandwidth.rs +++ b/rs/moq-ffi/src/bandwidth.rs @@ -54,18 +54,43 @@ impl MoqAllocator { /// means no estimate, so hold the current rate, and `Some(0)` is a real zero rate. #[derive(uniffi::Object)] pub struct MoqReservation { - /// Shared with the producer that holds the claim, such as a video producer's. - inner: Arc, + claim: Claim, +} + +/// Who keeps a [`MoqReservation`]'s claim alive. +enum Claim { + /// From [`MoqAllocator::reserve`]: the handle is the claim. + Owned(Arc), + /// A producer's own claim, such as a video producer's. Weak, so a handle the caller keeps + /// cannot hold the room after the producer finishes or moves to another track. + Borrowed(std::sync::Weak), } impl MoqReservation { - pub(crate) fn new(inner: impl Into>) -> Self { - Self { inner: inner.into() } + pub(crate) fn new(inner: moq_net::bandwidth::Reservation) -> Self { + Self { + claim: Claim::Owned(Arc::new(inner)), + } + } + + /// A view of a claim `owner` holds, released when the owner drops it. + pub(crate) fn borrowed(owner: &Arc) -> Self { + Self { + claim: Claim::Borrowed(Arc::downgrade(owner)), + } + } + + /// The claim, or `None` once its owner released it. + fn get(&self) -> Option> { + match &self.claim { + Claim::Owned(inner) => Some(inner.clone()), + Claim::Borrowed(inner) => inner.upgrade(), + } } #[cfg(test)] pub(crate) fn consume(&self) -> moq_net::bandwidth::Consumer { - self.inner.consume() + self.get().expect("reservation released").consume() } } @@ -75,16 +100,20 @@ impl MoqReservation { /// /// `None` means no estimate: hold the current rate. `Some(0)` is a real zero /// rate. While nothing subscribes to the track, this is the slice it would get - /// back, without claiming it from the others. + /// back, without claiming it from the others. `None` too once a producer that + /// owns this claim releases it. pub fn rate(&self) -> Option { let _guard = crate::ffi::enter(); - self.inner.rate().map(moq_net::bandwidth::Rate::as_bps) + self.get()?.rate().map(moq_net::bandwidth::Rate::as_bps) } - /// Change the ceiling, keeping the same claim. + /// Change the ceiling, keeping the same claim. Does nothing once a producer that + /// owns this claim releases it. pub fn update(&self, max_bps: u64) { let _guard = crate::ffi::enter(); - self.inner.update(moq_net::bandwidth::Rate::from_bps(max_bps)); + if let Some(inner) = self.get() { + inner.update(moq_net::bandwidth::Rate::from_bps(max_bps)); + } } } @@ -250,7 +279,8 @@ impl MoqRateControl { /// else could raise the grant to resume it. A missing grant (no estimate) holds the target. pub fn update(&self, reservation: &MoqReservation) -> Option { let _guard = crate::ffi::enter(); - let grant = reservation.inner.consume(); + // A released claim has no grant, which holds the target like a missing estimate. + let grant = reservation.get()?.consume(); let mut control = self.inner.lock().unwrap(); control.set_probed(grant.probed()); control.update(grant.peek(), kio::clock::now()).map(Into::into) diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 9f7a6acf640..31507e376a1 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -8204,6 +8204,51 @@ async fn set_bitrate_caps_a_later_bandwidth_grant() { broadcast.close().unwrap(); } +/// `finish` releases the producer's claim even while the caller keeps its reservation handle, +/// so a sibling gets the room back. +#[tokio::test] +async fn finish_releases_a_reservation_the_caller_keeps() { + use crate::bandwidth::MoqAllocator; + + let estimate = moq_net::bandwidth::Producer::new(); + estimate + .set(Some(moq_net::bandwidth::Rate::from_bps(4_000_000))) + .unwrap(); + let bandwidth = std::sync::Arc::new(MoqAllocator::new(moq_net::bandwidth::Allocator::new( + estimate.consume(), + ))); + let broadcast = MoqBroadcastProducer::new().unwrap(); + let (input, output) = software_camera(4_000_000); + let video = crate::video::MoqVideoProducer::new( + &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), + crate::video::MoqVideoProducerOptions { + input, + output, + track: Some("camera".into()), + bandwidth: Some(bandwidth.clone()), + lower: None, + }, + ) + .unwrap(); + let reservation = video.reservation().expect("published against an allocator"); + + let consumer = broadcast.consume().unwrap(); + let _sub = consumer.subscribe_track("camera".into(), None).await.unwrap(); + let sibling = broadcast.publish_track("sibling".into(), None).unwrap(); + let _sibling_sub = sibling.consume(None).unwrap(); + let sibling = bandwidth.reserve(&sibling.demand().unwrap(), 4_000_000); + // Video outranks a default track, so its tier takes the whole estimate. + assert_eq!(reservation.rate(), Some(4_000_000)); + assert_eq!(sibling.rate(), Some(0)); + + video.finish().unwrap(); + assert_eq!(sibling.rate(), Some(4_000_000)); + assert_eq!(reservation.rate(), None); + reservation.update(1_000_000); + + broadcast.close().unwrap(); +} + async fn one_shot_peers() -> (Arc, Arc, Arc) { let server = MoqServer::new(MoqServerConfig { bind: Some("127.0.0.1:0".into()), diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 44d0d6d648f..2faf3e3330d 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -435,7 +435,7 @@ impl MoqVideoProducer { /// /// Errors if this backend cannot retune while running. That is not fatal: the /// encoder keeps running at its current rate, so stop adapting rather than - /// stop publishing. + /// stop publishing. Any other error means the encoder stopped. pub fn set_bitrate(&self, bitrate: u64) -> Result<(), MoqError> { let _guard = crate::ffi::runtime().enter(); self.with(|producer| { @@ -446,14 +446,15 @@ impl MoqVideoProducer { } /// This encoder's bandwidth reservation, if it was published against a - /// [`MoqAllocator`]. Dropping the handle does not release the claim; the - /// producer holds it until [`finish`](Self::finish). A frame that changes the - /// decoder identity publishes under a new track name and reserves against that - /// track instead, so read this again after [`write`](Self::write). + /// [`MoqAllocator`]. The producer owns the claim, not the handle: dropping the + /// handle keeps it, and [`finish`](Self::finish) releases it however many handles + /// remain, after which they read no rate. A frame that changes the decoder + /// identity publishes under a new track name and reserves against that track + /// instead, releasing the old claim, so read this again after [`write`](Self::write). pub fn reservation(&self) -> Option> { self.bandwidth.as_ref()?; let guard = self.inner.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - Some(Arc::new(MoqReservation::new(guard.as_ref()?.reservation().clone()))) + Some(Arc::new(MoqReservation::borrowed(guard.as_ref()?.reservation()))) } /// Abort this track with an application error code, but only while nobody wants it. diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 8ea9f3a3679..482f08cb321 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -307,8 +307,9 @@ impl Producer { /// keyframe is forced. The reservation moves with it, so a later grant never retunes above it. /// /// Some backends cannot raise past the bitrate they opened at (openh264), so configure the - /// highest you will ask for and adapt downwards. Errors, changing nothing, when the encoder - /// cannot retune while running; it keeps encoding at its current rate. + /// highest you will ask for and adapt downwards. [`Error::BitrateUnsupported`], when the + /// encoder cannot retune while running, changes nothing: it keeps encoding at its current + /// rate. Any other error means the encoder stopped. pub async fn set_bitrate(&mut self, bitrate: Rate) -> Result<(), Error> { if let Encoding::Open(sink) = &mut self.encoding { sink.set_bitrate(bitrate).await?; diff --git a/rs/moq-video/src/encode/sink.rs b/rs/moq-video/src/encode/sink.rs index 7fa213faf97..7f937046dc7 100644 --- a/rs/moq-video/src/encode/sink.rs +++ b/rs/moq-video/src/encode/sink.rs @@ -124,9 +124,11 @@ impl Sink { self.0.receive() } - /// Retune the encoder, waiting for the backend's verdict. See - /// [`Encoder::set_bitrate`](super::Encoder::set_bitrate) for what a failure - /// means (not fatal: the encoder keeps its rate and keeps encoding). + /// Retune the encoder, waiting for the backend's verdict. + /// + /// [`Error::BitrateUnsupported`] is not fatal: the encoder keeps its rate and keeps + /// encoding (see [`Encoder::set_bitrate`](super::Encoder::set_bitrate)). + /// [`Error::CodecGone`] means the encode thread stopped, so the sink cannot continue. pub async fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { self.0.set_bitrate(bitrate).await } From 91658448bad386f4651062186ef250662a263d03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 14:58:49 -0800 Subject: [PATCH 13/14] fix(moq-ffi): the audio producer's reservation handle no longer holds its claim Same leak as video: `MoqAudioProducer::reservation` cloned the owning handle, so `finish` could not release a claim the caller kept. It now holds the native claim and hands out borrowed views. Also narrow set_bitrate's docs: a refused rate (openh264's raise past its opening rate included) leaves the encoder running; only CodecGone means it stopped. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/audio.rs | 13 +++++--- rs/moq-ffi/src/bandwidth.rs | 12 +++---- rs/moq-ffi/src/test.rs | 52 +++++++++++++++++++++++++++++ rs/moq-ffi/src/video.rs | 7 ++-- rs/moq-video/src/encode/producer.rs | 6 ++-- rs/moq-video/src/encode/sink.rs | 4 +-- 6 files changed, 73 insertions(+), 21 deletions(-) diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index a7910809757..9c000aba518 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -402,7 +402,8 @@ impl TryFrom for moq_audio::Frame { #[derive(uniffi::Object)] pub struct MoqAudioProducer { inner: std::sync::Mutex>, - reservation: std::sync::Mutex>>, + /// The claim itself; [`reservation`](Self::reservation) hands out only views of it. + reservation: std::sync::Mutex>>, /// Held so the reservation's registry outlives extra bandwidth handles. _bandwidth: Option>, children: crate::producer::Children, @@ -463,10 +464,12 @@ impl MoqAudioProducer { } /// This encoder's bandwidth reservation, if it was published against a - /// [`MoqAllocator`]. Dropping the handle does not release the claim; the - /// producer holds it until [`finish`](Self::finish). + /// [`MoqAllocator`]. The producer owns the claim, not the handle: dropping the + /// handle keeps it, and [`finish`](Self::finish) releases it however many handles + /// remain, after which they read no rate. pub fn reservation(&self) -> Option> { - self.reservation.lock().unwrap().clone() + let guard = self.reservation.lock().unwrap(); + Some(Arc::new(MoqReservation::borrowed(guard.as_ref()?))) } /// Abort this track with an application error code, but only while nobody wants it. @@ -575,7 +578,7 @@ impl MoqAudioProducer { // means the Rust producer will reserve and follow whenever it starts to. let reservation = bandwidth .as_ref() - .map(|bandwidth| bandwidth.reserve_demand(&producer.demand(), producer.bitrate().as_bps())); + .map(|bandwidth| Arc::new(bandwidth.inner().reserve(&producer.demand(), producer.bitrate()))); children.adopt(Arc::new(MoqAudioProducer { inner: std::sync::Mutex::new(Some(producer)), diff --git a/rs/moq-ffi/src/bandwidth.rs b/rs/moq-ffi/src/bandwidth.rs index ada2cad78a9..c49fa3d9e4b 100644 --- a/rs/moq-ffi/src/bandwidth.rs +++ b/rs/moq-ffi/src/bandwidth.rs @@ -24,13 +24,6 @@ impl MoqAllocator { pub(crate) fn inner(&self) -> &moq_net::bandwidth::Allocator { &self.allocator } - - pub(crate) fn reserve_demand(&self, demand: &moq_net::track::Demand, max_bps: u64) -> Arc { - Arc::new(MoqReservation::new( - self.allocator - .reserve(demand, moq_net::bandwidth::Rate::from_bps(max_bps)), - )) - } } #[uniffi::export] @@ -44,7 +37,10 @@ impl MoqAllocator { /// [`MoqReservation::update`] moves the ceiling without claiming twice. pub fn reserve(&self, demand: &MoqTrackDemand, max_bps: u64) -> Arc { let _guard = crate::ffi::enter(); - self.reserve_demand(demand.inner(), max_bps) + Arc::new(MoqReservation::new( + self.allocator + .reserve(demand.inner(), moq_net::bandwidth::Rate::from_bps(max_bps)), + )) } } diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 31507e376a1..961f5942873 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -8249,6 +8249,58 @@ async fn finish_releases_a_reservation_the_caller_keeps() { broadcast.close().unwrap(); } +/// The audio producer's claim goes with its `finish` too, whatever handles the caller keeps. +#[tokio::test] +async fn audio_finish_releases_a_reservation_the_caller_keeps() { + use crate::audio::*; + use crate::bandwidth::MoqAllocator; + + let estimate = moq_net::bandwidth::Producer::new(); + estimate + .set(Some(moq_net::bandwidth::Rate::from_bps(4_000_000))) + .unwrap(); + let bandwidth = std::sync::Arc::new(MoqAllocator::new(moq_net::bandwidth::Allocator::new( + estimate.consume(), + ))); + let broadcast = MoqBroadcastProducer::new().unwrap(); + let audio = MoqAudioProducer::new( + &crate::media::MoqMediaCatalogProducer::new(broadcast.clone()).unwrap(), + MoqAudioProducerOptions { + input: MoqAudioEncoderInput { + format: MoqAudioSampleFormat::F32, + sample_rate: 48_000, + channels: 1, + }, + output: MoqAudioEncoderOutput { + codec: MoqAudioCodec::opus(), + sample_rate: None, + channels: None, + bitrate: None, + frame_duration: None, + }, + track: Some("microphone".into()), + bandwidth: Some(bandwidth.clone()), + }, + ) + .unwrap(); + let reservation = audio.reservation().expect("published against an allocator"); + + let consumer = broadcast.consume().unwrap(); + let _sub = consumer.subscribe_track("microphone".into(), None).await.unwrap(); + let sibling = broadcast.publish_track("sibling".into(), None).unwrap(); + let _sibling_sub = sibling.consume(None).unwrap(); + let sibling = bandwidth.reserve(&sibling.demand().unwrap(), 4_000_000); + let claimed = reservation.rate().expect("an estimate"); + assert!(claimed > 0); + assert_eq!(sibling.rate(), Some(4_000_000 - claimed)); + + audio.finish().unwrap(); + assert_eq!(sibling.rate(), Some(4_000_000)); + assert_eq!(reservation.rate(), None); + + broadcast.close().unwrap(); +} + async fn one_shot_peers() -> (Arc, Arc, Arc) { let server = MoqServer::new(MoqServerConfig { bind: Some("127.0.0.1:0".into()), diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 2faf3e3330d..e4cba5da054 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -433,9 +433,10 @@ impl MoqVideoProducer { /// and the grant's ceiling move with it, so a later grant cannot retune above this /// value. /// - /// Errors if this backend cannot retune while running. That is not fatal: the - /// encoder keeps running at its current rate, so stop adapting rather than - /// stop publishing. Any other error means the encoder stopped. + /// Errors if the backend refuses the rate: some cannot retune while running, and + /// openh264 cannot raise past its opening rate. That is not fatal: the encoder + /// keeps running at its current rate, so stop adapting rather than stop + /// publishing. An error saying the codec session ended means the encoder stopped. pub fn set_bitrate(&self, bitrate: u64) -> Result<(), MoqError> { let _guard = crate::ffi::runtime().enter(); self.with(|producer| { diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 482f08cb321..07334ee99e1 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -307,9 +307,9 @@ impl Producer { /// keyframe is forced. The reservation moves with it, so a later grant never retunes above it. /// /// Some backends cannot raise past the bitrate they opened at (openh264), so configure the - /// highest you will ask for and adapt downwards. [`Error::BitrateUnsupported`], when the - /// encoder cannot retune while running, changes nothing: it keeps encoding at its current - /// rate. Any other error means the encoder stopped. + /// highest you will ask for and adapt downwards. A refused rate, such as + /// [`Error::BitrateUnsupported`] or openh264's raise past its opening rate, changes nothing: + /// the encoder keeps encoding at its current rate. Only [`Error::CodecGone`] means it stopped. pub async fn set_bitrate(&mut self, bitrate: Rate) -> Result<(), Error> { if let Encoding::Open(sink) = &mut self.encoding { sink.set_bitrate(bitrate).await?; diff --git a/rs/moq-video/src/encode/sink.rs b/rs/moq-video/src/encode/sink.rs index 7f937046dc7..5707190040c 100644 --- a/rs/moq-video/src/encode/sink.rs +++ b/rs/moq-video/src/encode/sink.rs @@ -126,8 +126,8 @@ impl Sink { /// Retune the encoder, waiting for the backend's verdict. /// - /// [`Error::BitrateUnsupported`] is not fatal: the encoder keeps its rate and keeps - /// encoding (see [`Encoder::set_bitrate`](super::Encoder::set_bitrate)). + /// A refused rate, such as [`Error::BitrateUnsupported`], is not fatal: the encoder keeps + /// its rate and keeps encoding (see [`Encoder::set_bitrate`](super::Encoder::set_bitrate)). /// [`Error::CodecGone`] means the encode thread stopped, so the sink cannot continue. pub async fn set_bitrate(&mut self, bitrate: moq_net::bandwidth::Rate) -> Result<(), Error> { self.0.set_bitrate(bitrate).await From af43ec37b1c4059f61cef2aa24fd2f0f0c7c494d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 15:22:50 -0800 Subject: [PATCH 14/14] fix(moq-c): an encoder's reservation handle views its claim, released at finish Mirrors moq-ffi's Claim split: moq_encode_{video,audio}_reservation hand out a weak view, so a handle the caller keeps no longer holds the room after moq_encode_*_finish. moq_allocator_reserve's handle still owns it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-c/src/audio.rs | 6 +-- rs/moq-c/src/bandwidth.rs | 55 +++++++++++++++++------- rs/moq-c/src/test.rs | 90 +++++++++++++++++++++++++++++++++++++++ rs/moq-c/src/video.rs | 6 +-- 4 files changed, 136 insertions(+), 21 deletions(-) diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index 3d8962e6aff..59fe686f69a 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -433,15 +433,15 @@ pub unsafe extern "C" fn moq_encode_audio( /// This encoder's bandwidth reservation, or 0 if it was published without one. /// -/// Closing the returned handle does not release the encoder's claim; that lasts -/// until [moq_encode_audio_finish]. +/// The handle views the encoder's claim: closing it does not release the claim, and +/// keeping it does not hold it past [moq_encode_audio_finish], after which it reads no rate. #[unsafe(no_mangle)] pub extern "C" fn moq_encode_audio_reservation(producer: u32) -> i32 { ffi::enter(move || { let producer = ffi::parse_id(producer)?; let mut state = State::lock(); match state.audio.reservation(producer)? { - Some(reservation) => Ok(i32::from(state.bandwidth.hold(reservation)?)), + Some(reservation) => Ok(i32::from(state.bandwidth.borrow(&reservation)?)), None => Ok(0), } }) diff --git a/rs/moq-c/src/bandwidth.rs b/rs/moq-c/src/bandwidth.rs index 98fa9bc3ad4..022687bccde 100644 --- a/rs/moq-c/src/bandwidth.rs +++ b/rs/moq-c/src/bandwidth.rs @@ -10,7 +10,7 @@ use crate::{Error, Id, NonZeroSlab, State, ffi}; #[derive(Default)] pub(crate) struct Bandwidth { allocators: NonZeroSlab, - reservations: NonZeroSlab>, + reservations: NonZeroSlab, consumers: NonZeroSlab>, viewers: NonZeroSlab, controls: NonZeroSlab, @@ -33,6 +33,15 @@ impl RateControl { } } +/// Who keeps a reservation handle's claim alive. +enum Claim { + /// From [`moq_allocator_reserve`]: the handle is the claim. + Owned(Arc), + /// A producer's own claim. Weak, so a handle the caller keeps cannot hold the room + /// after the producer finishes or moves to another track. + Borrowed(std::sync::Weak), +} + impl Bandwidth { pub fn insert(&mut self, allocator: moq_net::bandwidth::Allocator) -> Result { self.allocators.insert(allocator) @@ -50,15 +59,20 @@ impl Bandwidth { pub fn reserve(&mut self, allocator: Id, demand: &moq_net::track::Demand, max_bps: u64) -> Result { let allocator = self.allocator(allocator)?; let reservation = allocator.reserve(demand, moq_net::bandwidth::Rate::from_bps(max_bps)); - self.reservations.insert(Arc::new(reservation)) + self.reservations.insert(Claim::Owned(Arc::new(reservation))) } - pub fn hold(&mut self, reservation: Arc) -> Result { - self.reservations.insert(reservation) + /// A handle viewing a claim `owner` holds, released when the owner drops it. + pub fn borrow(&mut self, owner: &Arc) -> Result { + self.reservations.insert(Claim::Borrowed(Arc::downgrade(owner))) } - pub fn reservation(&self, id: Id) -> Result, Error> { - self.reservations.get(id).cloned().ok_or(Error::NotFound) + /// The handle's claim, or `None` once the producer that owned it released it. + pub fn reservation(&self, id: Id) -> Result>, Error> { + Ok(match self.reservations.get(id).ok_or(Error::NotFound)? { + Claim::Owned(inner) => Some(inner.clone()), + Claim::Borrowed(inner) => inner.upgrade(), + }) } pub fn reservation_close(&mut self, id: Id) -> Result<(), Error> { @@ -115,6 +129,7 @@ pub extern "C" fn moq_allocator_reserve(allocator: u32, track: u32, max_bps: u64 /// `present` is false when there is no estimate: hold the current rate. `present` /// true and `bps` 0 is a real zero rate. While nothing subscribes to the track, /// `bps` is the slice it would get back, without claiming it from the others. +/// `present` is false too once the producer that owns this claim releases it. /// /// # Safety /// - `bps` and `present` must be valid pointers. @@ -124,7 +139,11 @@ pub unsafe extern "C" fn moq_reservation_rate(reservation: u32, bps: *mut u64, p let reservation = ffi::parse_id(reservation)?; let bps = unsafe { bps.as_mut() }.ok_or(Error::InvalidPointer)?; let present = unsafe { present.as_mut() }.ok_or(Error::InvalidPointer)?; - match State::lock().bandwidth.reservation(reservation)?.rate() { + match State::lock() + .bandwidth + .reservation(reservation)? + .and_then(|inner| inner.rate()) + { Some(rate) => { *bps = rate.as_bps(); *present = true; @@ -138,15 +157,15 @@ pub unsafe extern "C" fn moq_reservation_rate(reservation: u32, bps: *mut u64, p }) } -/// Change the ceiling, keeping the same claim. +/// Change the ceiling, keeping the same claim. Does nothing once the producer that +/// owns this claim releases it. #[unsafe(no_mangle)] pub extern "C" fn moq_reservation_update(reservation: u32, max_bps: u64) -> i32 { ffi::enter(move || { let reservation = ffi::parse_id(reservation)?; - State::lock() - .bandwidth - .reservation(reservation)? - .update(moq_net::bandwidth::Rate::from_bps(max_bps)); + if let Some(inner) = State::lock().bandwidth.reservation(reservation)? { + inner.update(moq_net::bandwidth::Rate::from_bps(max_bps)); + } Ok(()) }) } @@ -154,7 +173,8 @@ pub extern "C" fn moq_reservation_update(reservation: u32, max_bps: u64) -> i32 /// Release a reservation, handing its share back to siblings. /// /// Closing an accessor returned by a video or audio producer does not release -/// the encoder's claim; that lasts until the producer is finished. +/// the encoder's claim, and keeping it open does not hold it: the claim lasts +/// until the producer is finished. #[unsafe(no_mangle)] pub extern "C" fn moq_reservation_close(reservation: u32) -> i32 { ffi::enter(move || { @@ -472,7 +492,8 @@ impl From for moq_rate_target { /// Create a rate control following this reservation, starting at the policy's maximum bitrate. /// -/// A zero `max_bps` is rejected with `MOQ_ERROR_INVALID_CONFIG`. +/// A zero `max_bps` is rejected with `MOQ_ERROR_INVALID_CONFIG`, and a claim its producer +/// already released with `MOQ_ERROR_NOT_FOUND`. #[unsafe(no_mangle)] pub extern "C" fn moq_rate_control_create(reservation: u32, policy: moq_rate_policy) -> i32 { ffi::enter(move || { @@ -484,7 +505,11 @@ pub extern "C" fn moq_rate_control_create(reservation: u32, policy: moq_rate_pol .has_lower .then(|| moq_net::bandwidth::Rate::from_bps(policy.lower_bps)); let mut state = State::lock(); - let grant = state.bandwidth.reservation(ffi::parse_id(reservation)?)?.consume(); + let grant = state + .bandwidth + .reservation(ffi::parse_id(reservation)?)? + .ok_or(Error::NotFound)? + .consume(); state.bandwidth.controls.insert(RateControl { inner: moq_mux::rate::Control::new(native), grant, diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 9aee7aa76ce..e47796562a5 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -8303,6 +8303,96 @@ fn encode_video_bitrate_caps_the_reservation() { assert_eq!(moq_origin_close(origin), 0); } +/// Finishing an encoder releases its claim, whatever reservation handle the caller keeps. +#[test] +fn encode_finish_releases_a_reservation_the_caller_keeps() { + let (bandwidth, estimate) = test_allocator(); + estimate + .set(Some(moq_net::bandwidth::Rate::from_bps(4_000_000))) + .unwrap(); + let origin = id(moq_origin_create()); + let path = b"finish-releases"; + let broadcast = publish_broadcast(origin, path); + let consume = request_broadcast(origin, path); + + let input = moq_video_encoder_input { + format: moq_video_pixel_format::MOQ_VIDEO_PIXEL_FORMAT_RGBA as u32, + width: 320, + height: 240, + framerate: moq_video_frame_rate { + numerator: 30, + denominator: 1, + }, + }; + let output = moq_video_encoder_output { + codec: moq_video_codec::MOQ_VIDEO_CODEC_H264 as u32, + bitrate: 4_000_000, + join_us: 0, + kind: moq_video_encoder_kind::MOQ_VIDEO_ENCODER_KIND_SOFTWARE as u32, + encoder: std::ptr::null(), + encoder_len: 0, + }; + let video = id(unsafe { + moq_encode_video( + broadcast, + &input, + &output, + &moq_video_encode_options { bandwidth, lower: 0 }, + ) + }); + let audio_input = moq_audio_encoder_input { + format: moq_audio_sample_format::MOQ_AUDIO_SAMPLE_FORMAT_F32 as u32, + sample_rate: 48_000, + channels: 1, + }; + let audio_output = moq_audio_encoder_output { + codec: c"opus".as_ptr(), + codec_len: 4, + sample_rate: 0, + channels: 0, + bitrate: 0, + frame_duration_us: 0, + }; + let audio = + id(unsafe { moq_encode_audio(broadcast, c"audio".as_ptr(), 5, &audio_input, &audio_output, bandwidth) }); + let video_claim = id(moq_encode_video_reservation(video)); + let audio_claim = id(moq_encode_audio_reservation(audio)); + + let (_, video_cb) = subscribe_named_track(consume, b"0.avc3"); + let (_, audio_cb) = subscribe_named_track(consume, b"audio"); + let sibling_track = publish_named_track(broadcast, b"sibling"); + let (sibling_sub, sibling_cb) = subscribe_named_track(consume, b"sibling"); + let sibling = id(moq_allocator_reserve(bandwidth, sibling_track, 4_000_000)); + // Media outranks a default track, so its tier takes the whole estimate. + wait_rate(sibling, 0); + + assert_eq!(moq_encode_video_finish(video), 0); + assert_eq!(moq_encode_audio_finish(audio), 0); + assert_eq!(rate(sibling), Some(4_000_000)); + for claim in [video_claim, audio_claim] { + assert_eq!(rate(claim), None); + assert_eq!(moq_reservation_update(claim, 1_000_000), 0); + let policy = moq_rate_policy { + max_bps: 1_000_000, + lower_bps: 0, + has_lower: false, + }; + assert!(moq_rate_control_create(claim, policy) < 0); + assert_eq!(moq_reservation_close(claim), 0); + } + + assert_eq!(moq_reservation_close(sibling), 0); + // Finishing ended the media subscriptions; only the sibling's is still open. + assert_eq!(moq_consume_track_cancel(sibling_sub), 0); + for cb in [video_cb, audio_cb, sibling_cb] { + let _ = cb.recv_terminal(); + } + assert_eq!(moq_allocator_close(bandwidth), 0); + assert_eq!(moq_consume_close(consume), 0); + assert_eq!(moq_publish_close(broadcast), 0); + assert_eq!(moq_origin_close(origin), 0); +} + /// A simulcast rung needs a bandwidth allocator and a bitrate above its lower rendition's. #[test] fn encode_video_rung_needs_a_grant_and_a_band() { diff --git a/rs/moq-c/src/video.rs b/rs/moq-c/src/video.rs index e15e368ede1..cfb11e8625f 100644 --- a/rs/moq-c/src/video.rs +++ b/rs/moq-c/src/video.rs @@ -736,15 +736,15 @@ pub unsafe extern "C" fn moq_encode_video( /// This encoder's bandwidth reservation, or 0 if it was published without one. /// -/// Closing the returned handle does not release the encoder's claim; that lasts -/// until [moq_encode_video_finish]. +/// The handle views the encoder's claim: closing it does not release the claim, and +/// keeping it does not hold it past [moq_encode_video_finish], after which it reads no rate. #[unsafe(no_mangle)] pub extern "C" fn moq_encode_video_reservation(producer: u32) -> i32 { ffi::enter(move || { let producer = ffi::parse_id(producer)?; let mut state = State::lock(); match state.video.reservation(producer)? { - Some(reservation) => Ok(i32::from(state.bandwidth.hold(reservation)?)), + Some(reservation) => Ok(i32::from(state.bandwidth.borrow(&reservation)?)), None => Ok(0), } })