From 8ed756a3154488fffc133031c0c56ec8971c1350 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 14:45:35 -0700 Subject: [PATCH 01/29] chore: bump package versions (#3401) Co-authored-by: GPT-5 --- bun.lock | 12 ++++++------ doc/lib/kt/index.md | 2 +- doc/lib/kt/moq.md | 2 +- doc/lib/swift/index.md | 2 +- doc/lib/swift/moq.md | 2 +- js/flate/package.json | 2 +- js/json/package.json | 2 +- js/net/package.json | 2 +- js/publish/package.json | 2 +- js/signals/package.json | 2 +- js/watch/package.json | 2 +- kt/README.md | 2 +- kt/gradle.properties | 2 +- py/moq-rs/pyproject.toml | 2 +- swift/README.md | 2 +- swift/VERSION | 2 +- uv.lock | 2 +- 17 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bun.lock b/bun.lock index 7525a6df0e6..3eec4b5c6ef 100644 --- a/bun.lock +++ b/bun.lock @@ -86,7 +86,7 @@ }, "js/flate": { "name": "@moq/flate", - "version": "0.1.2", + "version": "0.1.3", "dependencies": { "pako": "^3.0.1", }, @@ -123,7 +123,7 @@ }, "js/json": { "name": "@moq/json", - "version": "0.3.2", + "version": "0.3.3", "dependencies": { "@moq/flate": "workspace:^", "@moq/net": "workspace:^", @@ -187,7 +187,7 @@ }, "js/net": { "name": "@moq/net", - "version": "0.3.3", + "version": "0.3.4", "dependencies": { "@moq/qmux": "^0.3.3", "@moq/signals": "workspace:*", @@ -209,7 +209,7 @@ }, "js/publish": { "name": "@moq/publish", - "version": "0.4.5", + "version": "0.4.6", "dependencies": { "@moq/hang": "workspace:^", "@moq/net": "workspace:^", @@ -227,7 +227,7 @@ }, "js/signals": { "name": "@moq/signals", - "version": "0.2.2", + "version": "0.2.3", "devDependencies": { "@types/bun": "^1.4.0", "@types/react": "^19.2.18", @@ -273,7 +273,7 @@ }, "js/watch": { "name": "@moq/watch", - "version": "0.5.2", + "version": "0.5.3", "dependencies": { "@moq/hang": "workspace:^", "@moq/msf": "workspace:^", diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 51ef3dae6c5..98558c91a81 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -26,7 +26,7 @@ The raw UniFFI bindings (`uniffi.moq.*`) plus the native binaries (JNI on Androi ```kotlin // build.gradle.kts dependencies { - implementation("dev.moq:moq:0.4.4") + implementation("dev.moq:moq:0.4.5") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") } ``` diff --git a/doc/lib/kt/moq.md b/doc/lib/kt/moq.md index 04266bd57b9..a628a8b6472 100644 --- a/doc/lib/kt/moq.md +++ b/doc/lib/kt/moq.md @@ -16,7 +16,7 @@ Full API reference: [javadoc.io/doc/dev.moq/moq](https://javadoc.io/doc/dev.moq/ ```kotlin // build.gradle.kts dependencies { - implementation("dev.moq:moq:0.4.4") + implementation("dev.moq:moq:0.4.5") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") } ``` diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 54079be8676..5a2535c0bd7 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -39,7 +39,7 @@ Add the `Moq` wrapper; SPM resolves `MoqFFI` (and its XCFramework) transitively: ```swift dependencies: [ - .package(url: "https://github.com/moq-dev/moq-swift", from: "0.4.5"), + .package(url: "https://github.com/moq-dev/moq-swift", from: "0.4.6"), ], targets: [ .target( diff --git a/doc/lib/swift/moq.md b/doc/lib/swift/moq.md index 9743b17d6c1..6e6670ae8ca 100644 --- a/doc/lib/swift/moq.md +++ b/doc/lib/swift/moq.md @@ -14,7 +14,7 @@ Full API reference: [Swift Package Index](https://swiftpackageindex.com/moq-dev/ ## Install ```swift -.package(url: "https://github.com/moq-dev/moq-swift", from: "0.4.5"), +.package(url: "https://github.com/moq-dev/moq-swift", from: "0.4.6"), ``` Add `Moq` to your target's dependencies: diff --git a/js/flate/package.json b/js/flate/package.json index b32a70110d9..717ba9b351f 100644 --- a/js/flate/package.json +++ b/js/flate/package.json @@ -1,7 +1,7 @@ { "name": "@moq/flate", "type": "module", - "version": "0.1.2", + "version": "0.1.3", "description": "Group-scoped DEFLATE: a stream of self-delimited frames sharing one compression window.", "license": "(MIT OR Apache-2.0)", "repository": "github:moq-dev/moq", diff --git a/js/json/package.json b/js/json/package.json index b6ab02a48b6..f1a4b1b0fd2 100644 --- a/js/json/package.json +++ b/js/json/package.json @@ -1,7 +1,7 @@ { "name": "@moq/json", "type": "module", - "version": "0.3.2", + "version": "0.3.3", "description": "JSON publishing over MoQ tracks: snapshot/delta (RFC 7396 merge patch) objects, or append-log NDJSON streams.", "license": "(MIT OR Apache-2.0)", "repository": "github:moq-dev/moq", diff --git a/js/net/package.json b/js/net/package.json index f10e6b1ad37..39dbd9a80c4 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -1,7 +1,7 @@ { "name": "@moq/net", "type": "module", - "version": "0.3.3", + "version": "0.3.4", "description": "The networking layer for Media over QUIC: real-time pub/sub with built-in caching, fan-out, and prioritization.", "license": "(MIT OR Apache-2.0)", "repository": "github:moq-dev/moq", diff --git a/js/publish/package.json b/js/publish/package.json index da85c908e88..cb0e886ce35 100644 --- a/js/publish/package.json +++ b/js/publish/package.json @@ -1,7 +1,7 @@ { "name": "@moq/publish", "type": "module", - "version": "0.4.5", + "version": "0.4.6", "jsr": false, "description": "Publish Media over QUIC broadcasts", "license": "(MIT OR Apache-2.0)", diff --git a/js/signals/package.json b/js/signals/package.json index 6850c9ba38a..9ad14b0a740 100644 --- a/js/signals/package.json +++ b/js/signals/package.json @@ -1,7 +1,7 @@ { "name": "@moq/signals", "type": "module", - "version": "0.2.2", + "version": "0.2.3", "description": "Reactive and safe signals", "license": "(MIT OR Apache-2.0)", "repository": "github:moq-dev/moq", diff --git a/js/watch/package.json b/js/watch/package.json index 1b3ba634529..5243ad33333 100644 --- a/js/watch/package.json +++ b/js/watch/package.json @@ -1,7 +1,7 @@ { "name": "@moq/watch", "type": "module", - "version": "0.5.2", + "version": "0.5.3", "jsr": false, "description": "Watch Media over QUIC broadcasts", "license": "(MIT OR Apache-2.0)", diff --git a/kt/README.md b/kt/README.md index 05af3041c0b..0692e263008 100644 --- a/kt/README.md +++ b/kt/README.md @@ -14,7 +14,7 @@ Most apps want `dev.moq:moq`. Reach for `dev.moq:moq-ffi` directly only if you w ```kotlin // build.gradle.kts dependencies { - implementation("dev.moq:moq:0.4.4") + implementation("dev.moq:moq:0.4.5") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") } ``` diff --git a/kt/gradle.properties b/kt/gradle.properties index 0e5ed8dada0..6fc7069ed5b 100644 --- a/kt/gradle.properties +++ b/kt/gradle.properties @@ -14,7 +14,7 @@ moqffi.version=0.0.0-dev # Wrapper (`dev.moq:moq`) version. Source of truth, bumped by hand. Independent # of the crate: release-kt-lib.yml publishes a new version only when this # changes. Must stay >= the last published wrapper version. -moq.version=0.4.4 +moq.version=0.4.5 # Publishing is disabled by default. CI flips this when MAVEN_CENTRAL_USERNAME # and SIGNING_KEY secrets are configured. See README.md for setup details. diff --git a/py/moq-rs/pyproject.toml b/py/moq-rs/pyproject.toml index da902379ca2..40ad1931235 100644 --- a/py/moq-rs/pyproject.toml +++ b/py/moq-rs/pyproject.toml @@ -14,7 +14,7 @@ name = "moq-rs" # isn't already there (no tag needed; the registry is the gate). # Starts at 0.3.0 to open a new line above the old lockstep 0.2.x releases that # bundled the bindings. -version = "0.4.6" +version = "0.4.7" description = "Media over QUIC: real-time pub/sub with built-in caching, fan-out, and prioritization, with a Pythonic API. Import as `moq`." readme = "README.md" license = "MIT OR Apache-2.0" diff --git a/swift/README.md b/swift/README.md index d4cb503481f..93a8a0cb2d9 100644 --- a/swift/README.md +++ b/swift/README.md @@ -16,7 +16,7 @@ The Swift integration ships as two SPM packages, each mirrored to its own repo: ## Install ```swift -.package(url: "https://github.com/moq-dev/moq-swift", from: "0.4.5"), +.package(url: "https://github.com/moq-dev/moq-swift", from: "0.4.6"), ``` SPM resolves `MoqFFI` (and its prebuilt `MoqFFI.xcframework`, attached to the matching `moq-ffi-v*` GitHub Release) transitively. You only depend on `moq-swift`. diff --git a/swift/VERSION b/swift/VERSION index 0bfccb08040..ef52a648073 100644 --- a/swift/VERSION +++ b/swift/VERSION @@ -1 +1 @@ -0.4.5 +0.4.6 diff --git a/uv.lock b/uv.lock index 7c665645af9..7222b1b81da 100644 --- a/uv.lock +++ b/uv.lock @@ -509,7 +509,7 @@ source = { editable = "py/moq-ffi" } [[package]] name = "moq-rs" -version = "0.4.6" +version = "0.4.7" source = { editable = "py/moq-rs" } dependencies = [ { name = "moq-ffi" }, From 4cf216149851206ccfbd54c9d78f3f7cc4b0f6a0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 15:01:54 -0700 Subject: [PATCH 02/29] fix(moq-mux): slice the TS export on the PCR grid (#3351) Co-authored-by: Claude Opus 5 --- .github/workflows/nightly.yml | 15 + rs/moq-cli/src/subscribe.rs | 198 +++++-- rs/moq-mux/src/container/ts/export.rs | 545 +++++++++++++------ rs/moq-mux/src/container/ts/export_test.rs | 248 ++++++++- rs/moq-mux/src/pace.rs | 107 +++- test/justfile | 4 +- test/ts/README.md | 123 ++++- test/ts/pcr-timing.py | 605 +++++++++++++++++++++ test/ts/run.sh | 110 +++- 9 files changed, 1710 insertions(+), 245 deletions(-) create mode 100755 test/ts/pcr-timing.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 08682009935..3a297fec09a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -78,6 +78,21 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just obs ci + # The TS exporter's PCR release timing. Everything else about the stream + # can be graded from a capture, but *when* the bytes were handed over + # cannot: the grader has to sit on the exporter's stdout for the length of + # the run, stamping each packet. That is a real-time window no per-PR gate + # should pay for, so it lives here (see test/ts/README.md). + # + # 120 s rather than the 20 s default, because the drift term needs it. The + # exporter builds its mux buffer once and holds it, and a window that ends + # while it is still filling cannot tell that from a pipe running slow: the + # lag reaches ~480 ms over the first ~48 s and is flat after. Nightly is the + # one arm with the wall-clock time to see the flat part. + - name: TS timing + if: ${{ !cancelled() }} + run: nix develop --command just test ts --live --duration 120 + # `just rs windows` is the only thing that compiles moq-video's Media # Foundation capture/encode/decode, its D3D11 frames, and every # `#[cfg(target_os = "windows")]` test module. The Linux gate skips all of it, diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 6c66e689519..2cb8a7aa144 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -302,11 +302,11 @@ impl Subscribe { .with_latency(self.args.max_latency); // A TS byte stream carries no per-frame timing, so delivery time is the only - // carrier of each frame's spacing: the exporter emits its PCR grid as - // standalone frames stamped at their slot boundaries, on the contract that - // the caller writes the bytes at the time the stamp asserts. Draining on - // arrival instead collapses the clock into position clusters no downstream - // stage can repair (#2984). See [`Delivery`] for how the pacing stays + // carrier of each frame's spacing: the exporter slices its output on the PCR + // grid and stamps each slice at its slot boundary, on the contract that the + // caller writes the bytes at the time the stamp asserts. Draining on arrival + // instead collapses the clock into position clusters no downstream stage can + // repair (#2984). See [`Delivery`] for how the pacing stays // bounded; it needs to know whether each frame was waited for, hence the // hand-rolled poll instead of `ts.next()`. let mut delivery = Delivery::new(self.args.max_latency); @@ -364,15 +364,31 @@ impl Subscribe { /// schedule overshoots that epoch by more than the lead, it is delivered /// immediately and becomes the live edge /// ([`Pacer::hurry`](moq_mux::Pacer::hurry)). The anchor then rides the newest -/// frame through a backlog, so pacing resumes from the live edge once the -/// export makes us wait again. +/// frame through a backlog, so pacing resumes from the live edge. /// -/// The epoch is deliberately conservative: a ready frame may in truth have -/// arrived later, during a pacing sleep, but one-frame polling can't observe -/// that, and crediting sleep intervals would let a pre-queued backlog restart -/// the budget on every sleep. The cost is bounded: a hurry can collapse at -/// most the lead-sized interval ahead of the epoch, and smoothing resumes at -/// the next actual wait. +/// A hurry moves the epoch with it, because the frame it delivers *is* the live +/// edge and the distance it is measured from has to be too. `hurry` returns `now`, +/// so the credit below can never fire on the frame that hurried; leaving the epoch +/// behind would put every later frame the same overshoot past it and shed the whole +/// stream at the arrival cadence, which the buffered export would never correct +/// since it never makes us wait. A sink that is genuinely slow still sheds on every +/// stall: each one leaves the next frame far enough past the epoch to overshoot +/// again. +/// +/// Reaching a scheduled instant also advances the epoch, and has to. The TS export +/// holds a mux buffer: it always has the next grid slot ready, so it stops making +/// us wait, and an epoch that only moved on a wait would freeze for good and take +/// the budget with it (measured: a hurry roughly every second, each shedding the +/// pacing it was there to protect). Sleeping to the schedule is the proof that +/// replaces the wait, since nothing the export queued while we slept could have +/// gone out any earlier. What that gives up is a producer running faster than real +/// time indefinitely: the sink keeps pace with it and falls further behind live +/// without the budget noticing. Bounding *that* is the export's own +/// `--latency-max`, which sheds media rather than compressing the clock. +/// +/// The budget is the lead plus whatever standing lag the pacer has absorbed +/// ([`Pacer::slack`](moq_mux::Pacer::slack)), which is a distance it is holding on +/// purpose rather than lag to shed. struct Delivery { pacer: moq_mux::Pacer, /// The delivery-lag bound, and the pacer's lead: both are the export's @@ -407,12 +423,31 @@ impl Delivery { self.arrived = now; } + // The pacer's own slack is not lag: it is the producer's standing delivery + // distance, which the pacer discovered and is holding on purpose. Counting + // it here would shed the margin on a fixed cadence and put the writes back + // on the arrival clock, which is the whole thing this is here to avoid. + let budget = self.lead + self.pacer.slack(); let mut send_at = self.pacer.pace(frame.timestamp, now.into_std()); - if send_at.saturating_duration_since(self.arrived.into_std()) > self.lead { + if send_at.saturating_duration_since(self.arrived.into_std()) > budget { send_at = self.pacer.hurry(frame.timestamp, now.into_std()); + // A hurry makes this frame the live edge, so the epoch it is measured + // against becomes now as well. `hurry` returns `now`, so the credit below + // can't do it, and a stale epoch would overshoot the budget on every + // later frame, latching the shed on for good. + self.arrived = now; } - tokio::time::sleep_until(tokio::time::Instant::from_std(send_at)).await; + let send_at = tokio::time::Instant::from_std(send_at); + tokio::time::sleep_until(send_at).await; + // Reaching a scheduled instant is proof we are not behind, and it is the only + // such proof once the export holds a buffer: it always has the next slot + // ready, so it stops making us wait and the arrival epoch would freeze for + // good, taking the budget with it. Credit the scheduled instant rather than + // `now`, so an overshoot isn't credited as headroom. + if send_at > now { + self.arrived = send_at; + } out.write_all(&frame.payload).await?; out.flush().await?; Ok(()) @@ -469,52 +504,139 @@ mod tests { assert_eq!(out.len(), 3 * 188, "every payload was written"); } - /// A backlog delivered faster than real time (a tune-in group replaying from - /// its keyframe) must not be slept through step by step: each frame is within - /// the lead of the previous sleep's end, but total delivery lag versus the - /// frames' arrival would grow with the backlog's length and then stand for - /// the pipe's lifetime. Lag is measured against the last wait instead, so the - /// drain hurries once it overshoots the latency budget. + /// A sink that cannot reach its schedule still sheds the lag, which is the case + /// the budget is left guarding. + /// + /// The epoch advances on a wait, on reaching a scheduled instant, or on a hurry. + /// A sink falling behind reaches none of its instants and is never made to wait, + /// so only the hurry moves it, and each fresh stall puts the next frame far + /// enough past that epoch to overshoot the budget and hurry again. + /// + /// What this no longer covers, deliberately, is a one-off pre-queued backlog + /// (a tune-in group replaying from its keyframe). That now paces out at the + /// media rate instead of being shed, because it is indistinguishable from the + /// TS export's own mux buffer from here: both hand over a frame that is ready + /// and ahead of the schedule. The size of such a backlog is bounded by the + /// export's `--latency-max`, which is where it belongs. #[tokio::test(start_paused = true)] - async fn backlog_lag_is_bounded_by_the_latency_budget() { + async fn a_sink_that_cannot_keep_up_sheds_the_lag() { let mut delivery = Delivery::new(Duration::from_millis(500)); let mut out = Vec::new(); let start = tokio::time::Instant::now(); - - // Frames spanning 1200ms of media, all available at once (waited = false - // after the first): pacing may hold each for at most the 500ms budget - // past the last wait, not restart the budget after every sleep. delivery .deliver(&frame(0, Timescale::MICRO), true, &mut out) .await .unwrap(); + + // The writer stalls for 2s (a blocked pipe): wall clock runs on, the media + // clock does not, and every frame after this is overdue on arrival. + tokio::time::advance(Duration::from_secs(2)).await; + delivery .deliver(&frame(400_000, Timescale::MICRO), false, &mut out) .await .unwrap(); - assert_eq!(start.elapsed(), Duration::from_millis(400), "within the budget: paced"); + assert_eq!( + start.elapsed(), + Duration::from_secs(2), + "an overdue frame writes at once: the overshoot hurries and makes it the live edge" + ); + + // Shedding happens at the re-anchor, once, not on every frame after it. This + // one is 200ms of media past the new edge and 200ms past the epoch the hurry + // set with it, so it is inside the budget and paces. + delivery + .deliver(&frame(600_000, Timescale::MICRO), false, &mut out) + .await + .unwrap(); + assert_eq!(start.elapsed(), Duration::from_secs(2) + Duration::from_millis(200)); + + // A sink that goes on stalling goes on shedding, which is the property this + // test is here for: each stall leaves the next frame far enough past the + // epoch to overshoot the budget again, however recently the last hurry moved + // it. Media steps 25ms per iteration while the wall clock steps 2s. + for slot in 1..=3u64 { + tokio::time::advance(Duration::from_secs(2)).await; + let before = start.elapsed(); + delivery + .deliver(&frame(600_000 + slot * 25_000, Timescale::MICRO), false, &mut out) + .await + .unwrap(); + assert_eq!(start.elapsed(), before, "stall {slot} must shed, not pace"); + } + } + + /// A hurry has to move the arrival epoch with it, or it latches. + /// + /// `hurry` returns `now`, so the credit for reaching a scheduled instant can + /// never fire on the frame that hurried. Left there, the epoch stays wherever it + /// was before the shed while the schedule walks forward from the new edge, so the + /// next frame overshoots the budget too, and so does every one after it. The + /// buffered export never waits, so nothing else would move the epoch back. + #[tokio::test(start_paused = true)] + async fn pacing_resumes_after_a_hurry_without_a_wait() { + let mut delivery = Delivery::new(Duration::from_millis(500)); + let mut out = Vec::new(); + let start = tokio::time::Instant::now(); delivery - .deliver(&frame(800_000, Timescale::MICRO), false, &mut out) + .deliver(&frame(0, Timescale::MICRO), true, &mut out) .await .unwrap(); + + // Stall long enough that the schedule outruns the budget and sheds. + tokio::time::advance(Duration::from_secs(2)).await; delivery - .deliver(&frame(1_200_000, Timescale::MICRO), false, &mut out) + .deliver(&frame(400_000, Timescale::MICRO), false, &mut out) .await .unwrap(); - assert_eq!( - start.elapsed(), - Duration::from_millis(400), - "past the budget: the drain hurries instead of sleeping" - ); + let hurried = start.elapsed(); + assert_eq!(hurried, Duration::from_secs(2), "the overshoot sheds"); + + // Grid slots from the new edge, every one already queued and none waited on. + // Against a stale epoch each is a whole stall past it, so each would hurry and + // write at once, pinning `start.elapsed()` at `hurried` for the whole loop. + for slot in 1..=8u64 { + delivery + .deliver(&frame(400_000 + slot * 25_000, Timescale::MICRO), false, &mut out) + .await + .unwrap(); + assert_eq!( + start.elapsed(), + hurried + Duration::from_millis(slot * 25), + "slot {slot} must be paced, not shed" + ); + } + } + + /// The TS export holds a mux buffer, so it always has the next grid slot ready + /// and stops making the sink wait. Reaching a scheduled instant has to advance + /// the epoch too, or the budget freezes and hurries on a fixed cadence, shedding + /// the pacing it exists to protect. + #[tokio::test(start_paused = true)] + async fn a_buffered_producer_keeps_pacing() { + let mut delivery = Delivery::new(Duration::from_millis(500)); + let mut out = Vec::new(); - // The hurry made the newest frame the live edge, so pacing resumes - // relative to it once the export makes us wait again. + let start = tokio::time::Instant::now(); delivery - .deliver(&frame(1_240_000, Timescale::MICRO), true, &mut out) + .deliver(&frame(0, Timescale::MICRO), true, &mut out) .await .unwrap(); - assert_eq!(start.elapsed(), Duration::from_millis(440)); + + // A grid slot every 25ms, each already queued: an epoch that only moved on a + // wait would freeze here and hurry once the schedule passed 500ms. + for slot in 1..=40u64 { + delivery + .deliver(&frame(slot * 25_000, Timescale::MICRO), false, &mut out) + .await + .unwrap(); + assert_eq!( + start.elapsed(), + Duration::from_millis(slot * 25), + "slot {slot} must be paced, not shed" + ); + } } } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 1f0174e24b0..4a332b22f65 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1,11 +1,13 @@ //! MPEG-TS muxer. //! -//! [`Export`] subscribes to a MoQ broadcast and produces MPEG-TS, yielding one -//! [`Frame`] per media frame: PAT/PMT program tables followed by one PES packet, -//! packetized into 188-byte TS packets. Each frame keeps its media timestamp so -//! the caller can pace delivery on the media clock. The PCR rides its own -//! adaptation-field-only packets on a fixed media-time grid ([`Export::write_pcr`]). -//! Video is carried as Annex-B, audio as ADTS AAC. +//! [`Export`] subscribes to a MoQ broadcast and produces MPEG-TS: PAT/PMT program +//! tables and PES packets, packetized into 188-byte TS packets, with the PCR +//! riding its own adaptation-field-only packets on a fixed media-time grid. +//! Output is sliced on that grid rather than per media frame ([`Export::emit`]): +//! each [`Frame`] is one slot's clock packet plus the bytes belonging to it, +//! stamped at the slot boundary, so the clock a receiver recovers from byte +//! position agrees with the values, and a pacing caller releases each slot at +//! the instant it asserts. Video is carried as Annex-B, audio as ADTS AAC. //! //! Video flows through [`ExportSource`], which normalizes every H.264/H.265 //! source to length-prefixed NALU plus a resolved avcC/hvcC (parsing in-band @@ -14,7 +16,7 @@ //! length-prefixed -> Annex-B conversion, re-injecting the parameter sets as //! inline NALs on every keyframe. CMAF tracks are rejected with a clear error. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, VecDeque}; use std::task::Poll; use std::time::Duration; @@ -46,9 +48,9 @@ const PMT_PID: u16 = 0x1000; const FIRST_ES_PID: u16 = 0x1001; /// Re-emit PAT/PMT at least this often (wall-clock of the media) for tune-in. const PSI_INTERVAL: Duration = Duration::from_millis(500); -/// Emit a PCR on every crossing of this media-time grid ([`Export::write_pcr`]). +/// Emit a PCR on every crossing of this media-time grid ([`Export::emit`]). /// TR 101 290 flags a gap over 40 ms; broadcast muxes emit every 25-40 ms. -const PCR_INTERVAL: Duration = Duration::from_millis(25); +pub(super) const PCR_INTERVAL: Duration = Duration::from_millis(25); /// How many missed PCR slots to backfill at most: one second's worth. Frames /// coarser than the grid cross several slots at a time and every one is filled so /// the ramp stays uniform, down to a 1 fps cadence. Past this cap the media @@ -59,11 +61,11 @@ const PCR_BACKFILL: u128 = 40; /// Subscribe to a broadcast and produce an MPEG-TS byte stream. /// -/// Use [`next`](Self::next) to pull one [`Frame`] per media frame: its `payload` -/// is the TS packets, stamped with the source `timestamp` and `keyframe` flag. -/// The leading PAT/PMT rides on the first frame (so it inherits a real -/// timestamp), and is re-emitted at video keyframes and periodically for -/// mid-stream tune-in. Returns `None` when the broadcast ends. +/// Use [`next`](Self::next) to pull one [`Frame`] per PCR grid slot: its `payload` +/// is the TS packets belonging to that slot, stamped at the slot's boundary. The +/// leading PAT/PMT rides on the first frame (so it inherits a real timestamp), and +/// is re-emitted at video keyframes and periodically for mid-stream tune-in. +/// Returns `None` when the broadcast ends. pub struct Export { source: crate::Source, catalog: Option>, @@ -87,8 +89,28 @@ pub struct Export { psi: Option, /// Media timestamp of the last PAT/PMT emission ([`due`]). last_psi: Option, - /// Grid slot of the last PCR emission ([`Self::write_pcr`]). + /// Grid slot of the last PCR emission ([`Self::emit`]). last_pcr: Option, + /// TS packets muxed into the span that is still open. + pending: Vec, + /// Offsets into [`pending`](Self::pending) where a keyframe's packets begin, so + /// the output frame carrying one keeps the flag. + keyframes: Vec, + /// Output frames ready to hand out, one per grid slot the last span covered. + queue: VecDeque, + /// Continuity counter of the last packet emitted on the PCR PID. A clock packet + /// carries no payload, so it repeats whatever preceded it on the wire rather + /// than advancing the counter ([`Export::pcr_at`]). + pcr_cc: Option, + /// Media time the next span's bytes are transmitted from ([`Export::emit`]). + clock: Option, + /// Earliest media timestamp muxed into the open span: the decode time its bytes + /// have to arrive before, so it bounds how far the clock may run. + low: Option, + /// Media timestamp that opened the current span. Reordered (B-frame) timestamps + /// step backwards all the time, so a span closes on a timestamp passing this + /// high-water mark rather than on every frame. + watermark: Option, /// Tune-in point: the first video keyframe's timestamp, captured when the program /// tables are built. Non-video frames before it are dropped so the keyframe leads /// the stream. @@ -221,6 +243,13 @@ impl Export { psi: None, last_psi: None, last_pcr: None, + pending: Vec::new(), + keyframes: Vec::new(), + queue: VecDeque::new(), + pcr_cc: None, + clock: None, + low: None, + watermark: None, video_start: None, }) } @@ -233,12 +262,14 @@ impl Export { /// Get the next muxed frame. /// - /// Each [`Frame`] carries the TS packets for one media frame in `payload`, - /// stamped with that frame's media `timestamp` and `keyframe` flag so a - /// transport can pace delivery on the media clock. The leading PAT/PMT rides - /// on the first frame (inheriting its timestamp), and is re-emitted at video - /// keyframes and periodically for mid-stream tune-in. Returns `None` when the - /// broadcast ends. `duration` is always `None`: the muxer has no use for it. + /// Each [`Frame`] carries one slice of the PCR grid in `payload`: the clock + /// packets that slice opens with, followed by the muxed bytes belonging to it. + /// It is stamped with the media time that slice starts at, so a transport can + /// pace delivery on the media clock, and `keyframe` marks the slice a video + /// keyframe begins in. The leading PAT/PMT rides on the first slice, and is + /// re-emitted at video keyframes and periodically for mid-stream tune-in. + /// Returns `None` when the broadcast ends. `duration` is always `None`: the + /// muxer has no use for it. pub async fn next(&mut self) -> crate::Result> { kio::wait(|waiter| self.poll_next(waiter)).await } @@ -256,48 +287,13 @@ impl Export { } } - // 2. Pull a frame into every idle track. ExportSource has already - // transformed Annex-B avc3/hev1 into length-prefixed form and resolved - // the avcC/hvcC. Before the program tables are written, drop slices that - // arrive before their codec config resolves: a receiver joining mid-GOP - // can't use them, and parking them would stop us polling for the keyframe - // that carries the parameter sets. - let waiting_for_header = self.psi.is_none(); - let video_start = self.video_start; - for track in self.tracks.values_mut() { - if track.pending.is_some() || track.finished { - continue; - } - let is_video = matches!(track.kind, Kind::Video(_)); - loop { - match track.source.poll_read(waiter)? { - Poll::Ready(Some(frame)) => { - if waiting_for_header && !track.source.header_ready() { - continue; - } - // Tune-in alignment: drop non-video frames before the first video - // keyframe (see `video_start`) so the in-band SPS/PPS leads the stream. - if let Some(start) = video_start - && !is_video && frame.timestamp < start - { - continue; - } - track.pending = Some(frame); - break; - } - Poll::Ready(None) => { - track.finished = true; - break; - } - Poll::Pending => break, - } - } - } + // 2. Pull a frame into every idle track. + self.fill(waiter)?; // 3. Build the program tables once the layout is resolved and every // track's codec config is ready. The tables aren't emitted here: PSI has - // no media time of its own, so `write_frame` prepends them to the first - // frame instead, letting the leading PAT/PMT inherit a real timestamp. + // no media time of its own, so `mux` prepends them to the first frame's + // packets instead, letting the leading PAT/PMT inherit a real timestamp. if self.psi.is_none() { if self.tracks.is_empty() { // No tracks yet. If the catalog is also done, the broadcast is empty. @@ -332,32 +328,87 @@ impl Export { } } - // 4. Emit the smallest-timestamp pending frame as a PES packet (the first - // one carries the buffered PAT/PMT). The program clock leads it: each grid - // slot the frame's timestamp has crossed goes out first as its own frame, - // stamped at the slot boundary, so the caller's pacer places the PCR at the - // time it asserts instead of bunching it with the frame that revealed it. - if let Some(name) = self.pick_next_track() { - let timestamp = self.tracks[&name].pending.as_ref().unwrap().timestamp; - if let Some(out) = self.write_pcr(timestamp)? { + // 4. Mux the smallest-timestamp pending frame into the open span (the first + // one carries the buffered PAT/PMT). Nothing goes out until a later + // timestamp measures that span: only then is it known how many bytes it + // carried, which is what puts the clock packets at the byte position their + // own values imply and lets the caller's pacer release each at the instant + // it asserts. See [`Self::advance`]. + loop { + if let Some(out) = self.queue.pop_front() { return Poll::Ready(Ok(Some(out))); } + let Some(name) = self.pick_next_track() else { break }; let frame = self.tracks.get_mut(&name).unwrap().pending.take().unwrap(); - let out = self.write_frame(&name, frame)?; - return Poll::Ready(Ok(Some(out))); + self.advance(frame.timestamp)?; + self.mux(&name, frame)?; + // Refill the track we just drained: the next span is measured by its + // successor, and without the refill nothing would be polling for it. + self.fill(waiter)?; } - // 5. End of stream once every track has drained and the catalog is closed. - if self.catalog.is_none() && !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished) { - return Poll::Ready(Ok(None)); + // 5. Once every track has drained, no later timestamp is coming to measure + // the open span, so its bytes go out whole. That's independent of the + // catalog: a retained track finishes while the broadcast stays live, and + // holding its tail until the catalog closed would strand it indefinitely. + let drained = !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished); + if drained { + self.emit(None)?; + if let Some(out) = self.queue.pop_front() { + return Poll::Ready(Ok(Some(out))); + } } - if self.catalog.is_none() && self.tracks.is_empty() { + + // End of stream once the catalog is closed too: nothing more can appear. + if self.catalog.is_none() && (drained || self.tracks.is_empty()) { return Poll::Ready(Ok(None)); } Poll::Pending } + /// Pull a frame into every idle track. + /// + /// [`ExportSource`] has already transformed Annex-B avc3/hev1 into + /// length-prefixed form and resolved the avcC/hvcC. Before the program tables + /// are written, drop slices that arrive before their codec config resolves: a + /// receiver joining mid-GOP can't use them, and parking them would stop us + /// polling for the keyframe that carries the parameter sets. + fn fill(&mut self, waiter: &kio::Waiter) -> crate::Result<()> { + let waiting_for_header = self.psi.is_none(); + let video_start = self.video_start; + for track in self.tracks.values_mut() { + if track.pending.is_some() || track.finished { + continue; + } + let is_video = matches!(track.kind, Kind::Video(_)); + loop { + match track.source.poll_read(waiter)? { + Poll::Ready(Some(frame)) => { + if waiting_for_header && !track.source.header_ready() { + continue; + } + // Tune-in alignment: drop non-video frames before the first video + // keyframe (see `video_start`) so the in-band SPS/PPS leads the stream. + if let Some(start) = video_start + && !is_video && frame.timestamp < start + { + continue; + } + track.pending = Some(frame); + break; + } + Poll::Ready(None) => { + track.finished = true; + break; + } + Poll::Pending => break, + } + } + } + Ok(()) + } + fn update_catalog(&mut self, mut catalog: Catalog) -> anyhow::Result<()> { self.source.retain_valid(&mut catalog); @@ -712,11 +763,13 @@ impl Export { .map(|(_, _, name)| name.clone()) } - /// Packetize one media frame into an output [`Frame`], re-emitting PAT/PMT - /// before video keyframes (and periodically) so receivers can tune in - /// mid-stream. The returned frame keeps the source `timestamp` and `keyframe` - /// flag so the caller can pace it. - fn write_frame(&mut self, name: &str, frame: Frame) -> anyhow::Result { + /// Packetize one media frame into the open span, re-emitting PAT/PMT before + /// video keyframes (and periodically) so receivers can tune in mid-stream. + /// + /// The bytes are buffered rather than returned: which grid slots they belong + /// to isn't known until a later timestamp measures the span (see + /// [`Self::advance`]). + fn mux(&mut self, name: &str, frame: Frame) -> anyhow::Result<()> { let track = self.tracks.get(name).context("missing track")?; let pid = track.pid; let kind = track.kind.clone(); @@ -824,111 +877,184 @@ impl Export { self.write_pes(&mut out, &unit, &es_payload)?; } } - Ok(Frame { + if keyframe { + self.keyframes.push(self.pending.len()); + } + self.pending.extend_from_slice(&out); + self.low = Some(self.low.map_or(timestamp, |low| low.min(timestamp))); + Ok(()) + } + + /// Close the open span if `ts` passes the watermark, laying its bytes out. + /// + /// `ts` belongs to the frame about to be muxed. Passing the watermark means the + /// span that timestamp opened is done: everything buffered since is exactly the + /// bytes it carried, and the distance from it measures how long it ran. A + /// reordered (B-frame) timestamp that trails the watermark closes nothing, and + /// its bytes join the open span, which is where they are transmitted anyway. + /// + /// This is why nothing goes out on arrival, and why it can't. A span's bytes + /// have to reach a receiver before the units in it decode, so they ride the + /// grid slots *preceding* the span, which are only known once the span has + /// closed. That is the mux buffer: the exporter runs two spans behind the media + /// clock, by a constant amount, so a caller pacing on the stamps still releases + /// every slot at the interval its own PCR value asserts. + fn advance(&mut self, ts: Timestamp) -> anyhow::Result<()> { + let Some(watermark) = self.watermark else { + self.watermark = Some(ts); + return Ok(()); + }; + if ts <= watermark { + return Ok(()); + } + self.watermark = Some(ts); + let span = ts.as_nanos().saturating_sub(watermark.as_nanos()); + self.emit(Some(span)) + } + + /// Lay the open span's bytes across the grid slots that run up to its decode + /// time, one [`Frame`] per slot. `span` is how long the span ran, or `None` at + /// end of stream, where nothing measured it. + /// + /// Each frame opens with the clock packets whose slot boundary it starts at, + /// then carries the share of the bytes its slice of the interval earns. Three + /// things follow, and they are the whole point of muxing this way. The packet + /// count between consecutive PCRs is proportional to the difference between + /// their values, so a consumer holding only the byte stream (which is every + /// MPEG-TS tool) recovers the same clock from byte position that the values + /// assert. Each frame is stamped at its own slot boundary, so a pacing caller + /// releases the clock at the instant it asserts rather than when the media that + /// revealed it arrived. And the interval ends at the span's earliest media + /// timestamp, so every byte precedes the decode time of the unit it belongs to. + /// + /// The PES units cannot carry the clock themselves. Frames arrive in decode + /// order, so the authored DTS is a saw: a reference frame leaps a whole reorder + /// span ahead and each B-frame nudges one tick past it. A PCR sampled from it + /// freezes and jumps, and no downstream CBR stage can repair that, because a + /// groomer can only place the clock samples it receives. So the PCR asserts its + /// own uniform ramp instead: absolute grid slots on the media timeline (shared by + /// every exporter of the broadcast, like [`due`]), each backed off by the largest + /// decode-clock reserve of any track so every PES unit, whichever rendition it + /// belongs to, decodes at or after the clock that precedes it. + fn emit(&mut self, span: Option) -> anyhow::Result<()> { + let bytes = std::mem::take(&mut self.pending); + let keyframes = std::mem::take(&mut self.keyframes); + let Some(to) = self.low.take() else { return Ok(()) }; + let packets = bytes.len() / TsPacket::SIZE; + + // Transmit from where the last span stopped up to this one's decode time, so + // the intervals abut and the clock neither repeats nor reverses. The first + // has nothing to abut, so it takes the span's own measured length. + // + // The clock only ever moves forward. A track skewed far enough behind that it + // decodes before the clock already reached it can't be placed ahead of its own + // decode time, and `with_latency` owns that skew, so its bytes go out at the + // clock rather than dragging it backwards. + let start = match self.clock { + Some(clock) => clock.as_nanos(), + None => to.as_nanos().saturating_sub(span.unwrap_or(0)), + }; + let end = to.as_nanos().max(start); + let from = stamp(start)?; + self.clock = Some(stamp(end)?); + + // The slot `start` sits in; its boundary is at or before every byte here. + let open = start / PCR_INTERVAL.as_nanos(); + // The last boundary strictly inside the interval: `end` opens the next one's. + let last = slot_before(end, PCR_INTERVAL).max(open); + // Slots still owed a clock packet. Backfill every missed one so the ramp + // stays uniform when frames are coarser than the grid, but cap it: past the + // cap the media itself gapped, and a dense clock history for a span that + // carried no bytes helps nobody. + let first = self + .last_pcr + .map_or(open, |l| l + 1) + .max((last + 1).saturating_sub(PCR_BACKFILL)); + // Spread the bytes only across an interval the grid can describe. Past the + // cap they were muxed before a media gap, so they belong at its start rather + // than smeared across silence. + let spread = last - open < PCR_BACKFILL; + let width = end - start; + + // The first frame opens at `from` rather than a boundary, and carries every + // clock packet whose slot has already begun. + let pcr_pid = self.psi.as_ref().context("PSI not built")?.pcr_pid; + let mut payload = Vec::new(); + for index in first..=open { + let before = counter_before(&bytes, 0, pcr_pid, self.pcr_cc); + payload.extend_from_slice(&self.pcr_at(index, before)?); + } + let mut cut = 0; + let mut at = from; + + // One frame per boundary inside the interval. `first` can only exceed + // `open + 1` when the backfill cap skipped the slots below it, and that cap + // bounds this range to [`PCR_BACKFILL`] iterations however long the gap was. + for index in (open + 1).max(first)..=last { + let boundary = slot_stamp(index)?; + let next = if spread && width > 0 { + (packets as u128 * (boundary.as_nanos() - start) / width) as usize + } else { + packets + }; + payload.extend_from_slice(&bytes[cut * TsPacket::SIZE..next * TsPacket::SIZE]); + self.push(at, payload, &keyframes, cut, next); + cut = next; + at = boundary; + let before = counter_before(&bytes, cut * TsPacket::SIZE, pcr_pid, self.pcr_cc); + payload = self.pcr_at(index, before)?; + } + + payload.extend_from_slice(&bytes[cut * TsPacket::SIZE..]); + self.push(at, payload, &keyframes, cut, packets); + if let Some(cc) = counter_before(&bytes, bytes.len(), pcr_pid, None) { + self.pcr_cc = Some(cc); + } + Ok(()) + } + + /// Queue one output frame, unless it would be empty. `from`..`to` are the packet + /// indices it carries, which decide whether a keyframe begins in it. + fn push(&mut self, timestamp: Timestamp, payload: Vec, keyframes: &[usize], from: usize, to: usize) { + if payload.is_empty() { + return; + } + let (from, to) = (from * TsPacket::SIZE, to * TsPacket::SIZE); + let keyframe = keyframes.iter().any(|&at| at >= from && at < to); + self.queue.push_back(Frame { timestamp, duration: None, - payload: Bytes::from(out), + payload: Bytes::from(payload), keyframe, - }) + }); } - /// Emit the program clock: one adaptation-field-only packet on the PCR PID per - /// [`PCR_INTERVAL`] slot of the media timeline, each returned as its own - /// [`Frame`] stamped at its slot boundary so the caller's pacer delivers the - /// packet at the time it asserts. + /// The clock packet for grid slot `index`, and record that the slot is served. /// - /// The PES units cannot carry the clock. Frames arrive in decode order, so the - /// authored DTS is a saw: a reference frame leaps a whole reorder span ahead and - /// each B-frame nudges one tick past it. A PCR sampled from it freezes and jumps - /// (measured as 85% of intervals within 11 us of each other, the rest collected - /// into gaps of hundreds of ms), and no downstream CBR stage can repair that, - /// because a groomer can only place the clock samples it receives. So the PCR - /// asserts its own uniform ramp instead: absolute grid slots on the media - /// timeline (shared by every exporter of the broadcast, like [`due`]), each - /// backed off by the largest decode-clock reserve of any track so every PES - /// unit, whichever rendition it belongs to, decodes at or after the clock that - /// precedes it. - /// - /// `timestamp` is the next media frame's; `None` when its slot has already been - /// served (the caller then emits the media frame itself). One slot per call: - /// missed slots drain over successive polls, each at its own boundary, capped at - /// [`PCR_BACKFILL`]. The clock always leads, including ahead of the very first - /// frame's PAT/PMT: a receiver joins a TS mid-stream at an arbitrary packet, so - /// there is no "first" on the wire, and holding the clock back would leave the - /// leading PES without one. - fn write_pcr(&mut self, timestamp: Timestamp) -> anyhow::Result> { + /// The value backs off by the largest reserve of any track, not just the PCR + /// track's: every rendition's PES must decode at or after the clock, and each + /// video track backs its DTS off by its own catalog jitter. Back off through the + /// 33-bit wrap rather than saturating: a timeline that starts inside the reserve + /// would otherwise clamp its first slots to zero and break the uniform step. The + /// wire field is a circular clock, so the masked wrapped value is the correct + /// mod-2^33 back-off. + fn pcr_at(&mut self, index: u128, before: Option) -> anyhow::Result> { let pcr_pid = self.psi.as_ref().context("PSI not built")?.pcr_pid; - let current = slot(timestamp, PCR_INTERVAL); - let index = match self.last_pcr { - None => current, - Some(last) => { - // A reordered (B-frame) timestamp steps backwards into a slot already - // served; the clock only ever moves forward. - if current <= last { - return Ok(None); - } - // Backfill every missed slot so the ramp stays uniform when frames are - // coarser than the grid, but cap it: past the cap the media itself - // gapped, and a dense clock history for a span that carried no bytes - // helps nobody. - (last + 1).max(current.saturating_sub(PCR_BACKFILL - 1)) - } - }; - // The largest reserve of any track, not just the PCR track's: every - // rendition's PES must decode at or after the clock, and each video track - // backs its DTS off by its own catalog jitter. let reserve = self .tracks .values() .map(|t| t.dts_reserve) .max() .unwrap_or(DEFAULT_DTS_RESERVE); - // Back off through the 33-bit wrap rather than saturating: a timeline that - // starts inside the reserve would otherwise clamp its first slots to zero - // and break the uniform step. The wire field is a circular clock, so the - // masked wrapped value is the correct mod-2^33 back-off. let ticks = slot_ticks(index, PCR_INTERVAL).wrapping_sub(reserve); - let payload = self.pcr_packet(pcr_pid, ticks)?; + // Nothing has gone out on this PID yet, so there is no counter to repeat and + // any value starts a valid run; take the one before the next to be used. + let cc = match before { + Some(cc) => cc, + None => self.counters.entry(pcr_pid).or_default().as_u8().wrapping_sub(1), + }; self.last_pcr = Some(index); - Ok(Some(Frame { - timestamp: Timestamp::from_micros((index * PCR_INTERVAL.as_micros()) as u64)?, - duration: None, - payload: Bytes::from(payload), - keyframe: false, - })) - } - - /// One adaptation-field-only TS packet carrying `ticks` (continuous 90 kHz) as - /// its PCR, laid out by hand: the `mpeg2ts` serializer writes the six reserved - /// bits between PCR base and extension as zeros where ISO 13818-1 requires - /// ones, and strict analyzers flag that. There is no payload, so the field's - /// stuffing fills the packet and the continuity counter is not incremented - /// (ISO 13818-1 2.4.3.3): it repeats the previous packet's value, one behind - /// the stored next-to-use value. - fn pcr_packet(&mut self, pid: u16, ticks: u64) -> anyhow::Result> { - anyhow::ensure!(pid <= 0x1FFF, "PID out of range: {pid}"); - let cc = self.counters.entry(pid).or_default().as_u8().wrapping_sub(1) & ContinuityCounter::MAX; - let base = ticks & TS_TIMESTAMP_MASK; - let mut p = Vec::with_capacity(TsPacket::SIZE); - p.push(0x47); - p.push((pid >> 8) as u8); - p.push(pid as u8); - // adaptation_field_control = adaptation field only, no scrambling. - p.push(0x20 | cc); - // adaptation_field_length covers the rest of the packet. - p.push(183); - // PCR_flag alone. - p.push(0x10); - // program_clock_reference_base (33 bits), 6 reserved '1' bits, and a zero - // 9-bit extension (the grid is 90 kHz-exact, so there is nothing sub-tick). - p.push((base >> 25) as u8); - p.push((base >> 17) as u8); - p.push((base >> 9) as u8); - p.push((base >> 1) as u8); - p.push(((base as u8) << 7) | 0x7e); - p.push(0x00); - p.resize(TsPacket::SIZE, 0xff); - Ok(p) + pcr_packet(pcr_pid, ticks, cc) } /// Packetize a PES payload into 188-byte TS packets. @@ -1085,6 +1211,63 @@ impl Export { } } +/// One adaptation-field-only TS packet carrying `ticks` (continuous 90 kHz) as its +/// PCR, laid out by hand: the `mpeg2ts` serializer writes the six reserved bits +/// between PCR base and extension as zeros where ISO 13818-1 requires ones, and +/// strict analyzers flag that. +/// +/// There is no payload, so the field's stuffing fills the packet and the continuity +/// counter is not incremented (ISO 13818-1 2.4.3.3): `cc` is the counter of +/// whatever preceded this packet on the same PID, which it repeats. The clock rides +/// a PID that also carries media, and the packets around it were numbered when they +/// were muxed rather than when they go out, so this has to come from the wire order +/// rather than from the counter's current value. +fn pcr_packet(pid: u16, ticks: u64, cc: u8) -> anyhow::Result> { + anyhow::ensure!(pid <= 0x1FFF, "PID out of range: {pid}"); + let cc = cc & ContinuityCounter::MAX; + let base = ticks & TS_TIMESTAMP_MASK; + let mut p = Vec::with_capacity(TsPacket::SIZE); + p.push(0x47); + p.push((pid >> 8) as u8); + p.push(pid as u8); + // adaptation_field_control = adaptation field only, no scrambling. + p.push(0x20 | cc); + // adaptation_field_length covers the rest of the packet. + p.push(183); + // PCR_flag alone. + p.push(0x10); + // program_clock_reference_base (33 bits), 6 reserved '1' bits, and a zero + // 9-bit extension (the grid is 90 kHz-exact, so there is nothing sub-tick). + p.push((base >> 25) as u8); + p.push((base >> 17) as u8); + p.push((base >> 9) as u8); + p.push((base >> 1) as u8); + p.push(((base as u8) << 7) | 0x7e); + p.push(0x00); + p.resize(TsPacket::SIZE, 0xff); + Ok(p) +} + +/// The continuity counter a payload-less packet inserted at byte offset `cut` has to +/// repeat: the last one before it on `pid`, else the one carried in from the last +/// span, else one behind the first packet that follows it, which is what keeps the +/// run continuous where the clock leads the stream and nothing precedes it. +fn counter_before(bytes: &[u8], cut: usize, pid: u16, carried: Option) -> Option { + let on_pid = |p: &&[u8]| (u16::from(p[1] & 0x1f) << 8 | u16::from(p[2])) == pid; + let counter = |p: &[u8]| p[3] & ContinuityCounter::MAX; + bytes[..cut] + .rchunks_exact(TsPacket::SIZE) + .find(on_pid) + .map(counter) + .or(carried) + .or_else(|| { + bytes[cut..] + .chunks_exact(TsPacket::SIZE) + .find(on_pid) + .map(|p| counter(p).wrapping_sub(1) & ContinuityCounter::MAX) + }) +} + /// Optional PES header region carrying PTS only: 2 flag bytes + 1 length byte + 5 PTS bytes. const PES_OPTIONAL_LEN: usize = 3 + 5; /// Extra bytes when the optional region also carries a DTS (5 DTS bytes). @@ -1129,6 +1312,30 @@ fn slot(timestamp: Timestamp, interval: Duration) -> u128 { Duration::from(timestamp).as_nanos() / interval.as_nanos() } +/// Index of the last repetition slot to *begin* strictly before `nanos`. +/// +/// [`slot`] rounds down, so a position sitting exactly on a boundary belongs to the slot that +/// boundary opens. When the position is the exclusive end of an interval, that slot is the +/// next interval's, not this one's, hence the offset. +fn slot_before(nanos: u128, interval: Duration) -> u128 { + if nanos == 0 { + return 0; + } + (nanos - 1) / interval.as_nanos() +} + +/// A repetition slot's boundary as a media timestamp. +fn slot_stamp(index: u128) -> anyhow::Result { + stamp(index * PCR_INTERVAL.as_micros() * 1_000) +} + +/// A nanosecond position on the media timeline as a microsecond [`Timestamp`], the +/// scale the exporter stamps its own output in. +fn stamp(nanos: u128) -> anyhow::Result { + let micros = (nanos / 1_000).try_into().context("media timeline out of range")?; + Ok(Timestamp::from_micros(micros)?) +} + /// A repetition slot's boundary (`index * interval`) in 90 kHz ticks. fn slot_ticks(index: u128, interval: Duration) -> u64 { (index * interval.as_nanos() * 90_000 / 1_000_000_000) as u64 diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index f0ad7acb2b2..5779a64d784 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -16,6 +16,7 @@ use mpeg2ts::pes::{PesPacketReader, ReadPesPacket}; use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use crate::catalog::hang::Container as HangContainer; +use crate::container::ts::export::PCR_INTERVAL; use crate::container::ts::{Export, catalog as tscat}; use crate::container::{Frame, Producer}; use moq_net::Timestamp; @@ -728,35 +729,40 @@ async fn export_pcr_wraps_below_the_reserve_at_start() { let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); let frames = drain_frames(&mut exporter).await; - // Each PCR is its own single-packet frame, stamped at its slot boundary so the - // caller's pacer delivers the clock at the time it asserts. + // A slot's clock packets lead the frame carrying that slot's bytes, and the + // frame is stamped at the slot boundary so the caller's pacer delivers the + // clock at the time it asserts. let mut pcrs: Vec = Vec::new(); for (i, frame) in frames.iter().enumerate() { assert_packet_aligned(&frame.payload); - let packet = &frame.payload[..188]; - // adaptation-field-only packets (adaptation_field_control == 0b10) are the clock. - if packet[3] & 0x30 != 0x20 { - continue; + let mut head = None; + for packet in frame.payload.chunks(188) { + // adaptation-field-only packets (adaptation_field_control == 0b10) are the clock. + if packet[3] & 0x30 != 0x20 { + break; + } + assert_eq!(packet[5], 0x10, "PCR_flag alone, at {i}"); + // The six reserved bits between base and extension are ones (ISO 13818-1); + // the crate's serializer writes zeros here, which is why the packet is + // laid out by hand. + assert_eq!(packet[10] & 0x7e, 0x7e, "reserved bits must be ones, at {i}"); + let base = (u64::from(packet[6]) << 25) + | (u64::from(packet[7]) << 17) + | (u64::from(packet[8]) << 9) + | (u64::from(packet[9]) << 1) + | u64::from(packet[10] >> 7); + pcrs.push(base); + head = Some(base); } - assert_eq!(frame.payload.len(), 188, "a PCR frame is one packet, at {i}"); - assert_eq!(packet[5], 0x10, "PCR_flag alone, at {i}"); - // The six reserved bits between base and extension are ones (ISO 13818-1); - // the crate's serializer writes zeros here, which is why the packet is - // laid out by hand. - assert_eq!(packet[10] & 0x7e, 0x7e, "reserved bits must be ones, at {i}"); - let base = (u64::from(packet[6]) << 25) - | (u64::from(packet[7]) << 17) - | (u64::from(packet[8]) << 9) - | (u64::from(packet[9]) << 1) - | u64::from(packet[10] >> 7); - // The frame is paced at the slot the value asserts (plus the reserve). - let slot_ticks = frame.timestamp.as_micros() * 90_000 / 1_000_000; + // The frame is paced at the slot its last leading clock asserts (plus the + // reserve), which is the newest slot that has begun by then. + let Some(base) = head else { continue }; + let slot_ticks = frame.timestamp.as_micros() / 25_000 * 25_000 * 90 / 1_000; assert_eq!( base, (slot_ticks as u64).wrapping_sub(16) & WIRE, "pacing off value, at {i}" ); - pcrs.push(base); } const WIRE: u64 = (1 << 33) - 1; @@ -2277,3 +2283,205 @@ async fn late_join_matches_a_running_exporter_without_video() { let from = b.iter().map(|f| f.timestamp).filter(|&t| t > tune_in).min().unwrap(); assert_only_continuity_differs(&a, &b, from); } + +/// Every PCR packet in transport order: its packet index in the stream, its value +/// (90 kHz), and the media timestamp of the frame carrying it. +fn collect_pcrs(frames: &[Frame]) -> Vec<(usize, u64, u128)> { + let mut at = 0; + let mut pcrs = Vec::new(); + for frame in frames { + for packet in frame.payload.chunks(188) { + // adaptation-field-only packets (adaptation_field_control == 0b10) are the clock. + if packet[3] & 0x30 == 0x20 && packet[5] & 0x10 != 0 { + let base = (u64::from(packet[6]) << 25) + | (u64::from(packet[7]) << 17) + | (u64::from(packet[8]) << 9) + | (u64::from(packet[9]) << 1) + | u64::from(packet[10] >> 7); + pcrs.push((at, base, frame.timestamp.as_micros())); + } + at += 1; + } + } + pcrs +} + +/// A 4 s CBR-ish single-rendition H.264 feed at 25 fps, one second per group: the +/// shape of a broadcast contribution capture, and the one #3334 measured. +async fn export_cbr_video() -> Vec { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + + let track = broadcast + .create_track(broadcast.unique_name(".h264"), hang::container::track_info()) + .unwrap(); + { + let mut cfg = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: true, + }); + cfg.container = Container::Legacy; + catalog.lock().video.renditions.insert(track.name().to_string(), cfg); + } + let mut video = Producer::new(track, HangContainer::Legacy); + for i in 0..100u64 { + let keyframe = i % 25 == 0; + let mut nal = vec![if keyframe { 0x65u8 } else { 0x41 }]; + nal.extend(std::iter::repeat_n(0xAB, 7_000)); + if keyframe && i > 0 { + video.cut(None).unwrap(); + } + video + .write(Frame { + timestamp: Timestamp::from_micros(i * 40_000).unwrap(), + duration: None, + payload: if keyframe { + annexb(&[SPS, PPS, &nal]) + } else { + annexb(&[&nal]) + }, + keyframe, + }) + .unwrap(); + } + video.finish().unwrap(); + + let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); + drain_frames(&mut exporter).await +} + +/// #3334: the clock a receiver recovers from *byte position* has to agree with the +/// values, because a byte stream carries no other timing. Emitting each media frame +/// whole put every PCR between two frames instead of among the bytes it labels, so +/// the grid #2967 made exact could not be read off the wire: consecutive clock +/// packets sat one packet apart with the media they described heaped between the +/// clusters, and a downstream stage re-deriving PCR from position regenerated the +/// original clustered distribution. +#[tokio::test(start_paused = true)] +async fn pcr_rides_the_bytes_it_labels() { + let frames = export_cbr_video().await; + let pcrs = collect_pcrs(&frames); + assert!(pcrs.len() > 100, "expected the full feed, got {} PCRs", pcrs.len()); + + // Every step is one grid slot, so every gap should carry the same span of media + // and therefore a comparable number of packets. + let step = PCR_INTERVAL.as_micros() as u64 * 90 / 1_000; + for (i, w) in pcrs.windows(2).enumerate() { + assert_eq!(w[1].1.wrapping_sub(w[0].1) & ((1 << 33) - 1), step, "value step at {i}"); + } + + let gaps: Vec = pcrs.windows(2).map(|w| w[1].0 - w[0].0).collect(); + assert_eq!( + gaps.iter().filter(|&&gap| gap == 1).count(), + 0, + "no clock packet may sit adjacent to the previous one: {gaps:?}" + ); + let mut sorted = gaps.clone(); + sorted.sort_unstable(); + let median = sorted[sorted.len() / 2]; + // The leading group carries the parameter sets and program tables on top of its + // media, so allow generous headroom; what this rules out is the bimodal + // distribution (one packet, then hundreds) that made the clock unrecoverable. + assert!( + sorted[0] * 3 >= median && sorted[sorted.len() - 1] <= median * 3, + "packet gaps must track the interval the values assert, got {sorted:?}" + ); +} + +/// The other half of #3334: a PCR's *release* has to track the interval its own +/// value asserts. The exporter only stamps; the caller paces on the stamps (see +/// [`moq_mux::Pacer`]), so the property here is that consecutive clock packets are +/// stamped exactly one grid interval apart. Frames are emitted whole and a slot's +/// bytes are only all in hand once media past it has arrived, so a stamp that +/// tracked frame arrival was already in the past and its sleep was a no-op. +#[tokio::test(start_paused = true)] +async fn pcr_stamps_step_by_the_grid() { + let frames = export_cbr_video().await; + let pcrs = collect_pcrs(&frames); + + let steps: Vec = pcrs.windows(2).map(|w| w[1].2 as i128 - w[0].2 as i128).collect(); + let interval = PCR_INTERVAL.as_micros() as i128; + assert!( + steps.iter().all(|&step| step == interval), + "every clock packet must be stamped one grid interval past the last: {steps:?}" + ); +} + +/// The same positional property on a real reordered (B-frame) capture with a second +/// rendition, where the exporter has no uniform cadence to lean on: frames arrive in +/// decode order, and the two tracks advance the media clock at different rates. The +/// clock still lands among the bytes it labels rather than clustering, though the +/// gaps are no longer uniform: a span is however long it took the next timestamp to +/// arrive, which for interleaved tracks is nothing like the media a frame's bytes +/// represent. Evening that out needs the muxer to hold a byte buffer and drain it at +/// a measured rate, which this does not do. +#[tokio::test(start_paused = true)] +async fn pcr_stays_among_the_bytes_across_reordered_tracks() { + let data = include_bytes!("test_data/scte35/kyrion_dirtystart.ts"); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast).unwrap(); + let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve()); + import.decode(&BytesMut::from(&data[..])).unwrap(); + import.finish().unwrap(); + + let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); + let frames = drain_frames(&mut exporter).await; + let pcrs = collect_pcrs(&frames); + assert!(pcrs.len() > 50, "expected the full feed, got {} PCRs", pcrs.len()); + + let gaps: Vec = pcrs.windows(2).map(|w| w[1].0 - w[0].0).collect(); + assert_eq!( + gaps.iter().filter(|&&gap| gap == 1).count(), + 0, + "no clock packet may sit adjacent to the previous one: {gaps:?}" + ); + + // Stamps still step by exactly one slot, apart from the first interval, which + // covers the leading span rather than a whole slot. + let interval = PCR_INTERVAL.as_micros() as i128; + let off = pcrs + .windows(2) + .filter(|w| w[1].2 as i128 - w[0].2 as i128 != interval) + .count(); + assert!(off <= 1, "{off} clock packets are stamped off the grid"); +} + +/// A clock packet carries no payload, so ISO 13818-1 2.4.3.3 says it must repeat +/// the continuity counter of whatever preceded it on its PID rather than advance +/// it. The clock rides a PID that also carries media, and slicing on the grid puts +/// clock packets *inside* a frame's packet run, whose counters were assigned when +/// the frame was muxed rather than when the bytes go out. Numbering the clock +/// packet from the counter's current value there lands it a whole frame ahead, and +/// an analyzer reports a discontinuity on it and another on the payload packet +/// after it. +#[tokio::test(start_paused = true)] +async fn payload_less_clock_packets_repeat_the_counter() { + let frames = export_cbr_video().await; + let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + assert_packet_aligned(&ts); + + let mut last: std::collections::HashMap = std::collections::HashMap::new(); + let mut advanced = 0; + let mut discontinuities = 0; + for (i, packet) in ts.chunks(188).enumerate() { + let pid = u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]); + let cc = packet[3] & 0x0f; + let payload = packet[3] & 0x10 != 0; + if let Some(&prev) = last.get(&pid) { + if payload && cc != (prev + 1) & 0x0f { + discontinuities += 1; + eprintln!("discontinuity at packet {i} on pid {pid}: {prev} -> {cc}"); + } else if !payload && cc != prev { + advanced += 1; + eprintln!("payload-less packet {i} on pid {pid} advanced: {prev} -> {cc}"); + } + } + last.insert(pid, cc); + } + assert_eq!(advanced, 0, "payload-less packets must repeat the counter"); + assert_eq!(discontinuities, 0, "the counter must be continuous on every PID"); +} diff --git a/rs/moq-mux/src/pace.rs b/rs/moq-mux/src/pace.rs index 777a903a6bc..022f66347c6 100644 --- a/rs/moq-mux/src/pace.rs +++ b/rs/moq-mux/src/pace.rs @@ -11,8 +11,8 @@ use moq_net::Timestamp; /// timestamp on the contract that the caller delivers the bytes at the time the /// stamp asserts. That matters most for MPEG-TS, where the byte stream itself /// carries no per-frame timing: [`ts::Export`](crate::container::ts::Export) -/// emits its PCR grid as standalone frames stamped at their slot boundaries, and -/// a caller that drains them on arrival collapses the clock into position +/// slices its output on the PCR grid and stamps each slice at its slot boundary, +/// and a caller that drains them on arrival collapses the clock into position /// clusters no downstream stage can repair. "Deliver" is up to the transport: /// a paced sink sleeps until the returned instant before writing, while a /// transport with receiver-side buffering (e.g. SRT's TSBPD) stamps the payload @@ -21,9 +21,9 @@ use moq_net::Timestamp; /// `send_at = anchor + (ts - base)` maps the frame's media time onto the wall /// clock, where `base`'s media time and `anchor`'s wall instant were pinned to /// each other at the last re-anchor. Offsets are computed in nanoseconds, so the -/// frames may change [`Timescale`](moq_net::Timescale) mid-stream (the TS -/// exporter stamps PCR frames in microseconds and media frames at the source's -/// own scale). +/// frames may change [`Timescale`](moq_net::Timescale) mid-stream: each exporter +/// picks its own, and the TS one stamps grid boundaries in microseconds whatever +/// scale the source's frames carry. /// /// When `send_at` would lead `now` by more than the configured /// [`lead`](Self::with_lead), the media clock has outrun wall-clock by more than @@ -33,6 +33,15 @@ use moq_net::Timestamp; /// ever moves the anchor *forward*: a frame that merely arrives late (network /// or CPU jitter, or a reordered B-frame whose timestamp trails the edge) keeps /// its earlier media instant instead of collapsing to its arrival instant. +/// +/// The other direction is the standing lag. Every producer delivers some fixed +/// distance behind the media clock (a muxer's buffer, a hop of network), and the +/// first frame pins an offset that has no room for it, so every later frame is +/// due the instant it arrives, the sleep is a no-op, and the sink ends up writing +/// at whatever cadence frames turned up in. So the anchor slides back by however +/// much a frame fell behind, up to `lead` in total: it discovers that distance +/// rather than assuming one, and the bound is the buffer the caller already said +/// it would hold. #[derive(Default)] pub struct Pacer { /// How far ahead of `now` a frame may be scheduled before re-anchoring. @@ -40,6 +49,8 @@ pub struct Pacer { /// The wall instant and media time (in nanoseconds) pinned to each other at /// the last re-anchor; every frame paces relative to this pair. anchor: Option<(Instant, u128)>, + /// How far the anchor has slid back to absorb late arrivals, capped at `lead`. + slack: Duration, } impl Pacer { @@ -79,12 +90,41 @@ impl Pacer { match send_at { // `saturating_duration_since` is zero for an `at` in the past, which any // lead admits; the subtraction form can't overflow on a huge lead. - Some(at) if at.saturating_duration_since(now) <= self.lead => at, + Some(at) if at.saturating_duration_since(now) <= self.lead => self.absorb(at, now), // Media outran wall-clock (or overflowed the platform clock). _ => self.hurry(ts, now), } } + /// Slide the anchor back by however much `at` fell behind `now`, and return the + /// instant that leaves. + /// + /// This is how the pacer finds the producer's standing delivery lag (see the + /// type docs). The shift is capped so the total never exceeds the lead, and it + /// can only ever raise a past instant toward `now`, never past it, so absorbing + /// never schedules a frame later than it would have gone out anyway. + fn absorb(&mut self, at: Instant, now: Instant) -> Instant { + let behind = now.saturating_duration_since(at); + let shift = behind.min(self.lead.saturating_sub(self.slack)); + if shift.is_zero() { + return at; + } + self.slack += shift; + if let Some((anchor, _)) = self.anchor.as_mut() { + *anchor += shift; + } + at + shift + } + + /// How much of the producer's standing delivery lag the anchor has absorbed. + /// + /// A caller running its own lag budget has to know this: the pacer is holding + /// it deliberately (see the type docs), so counting it as lag would have the + /// caller shedding the very margin that makes its sleeps do anything. + pub fn slack(&self) -> Duration { + self.slack + } + /// Deliver `ts` at `now` and make it the live edge: later frames pace /// relative to this pair. /// @@ -96,6 +136,8 @@ impl Pacer { /// have arrived and hurries when that overshoots. pub fn hurry(&mut self, ts: Timestamp, now: Instant) -> Instant { self.anchor = Some((now, ts.as_nanos())); + // A fresh pin has no lag absorbed into it yet, so the budget starts over. + self.slack = Duration::ZERO; now } } @@ -178,6 +220,59 @@ mod tests { assert_eq!(pacer.pace(ms(40), start), start + Duration::from_millis(40)); } + /// A producer that delivers a constant distance behind the media clock (the TS + /// exporter's mux buffer, or a hop of network) would otherwise pin that distance + /// into the first frame's anchor and never get it back: every later frame is due + /// exactly when it arrives, so the sleep is a no-op and the sink writes at the + /// arrival cadence rather than the media one. + #[test] + fn absorbs_a_standing_delivery_lag() { + let start = Instant::now(); + let mut pacer = Pacer::default().with_lead(Duration::from_millis(500)); + assert_eq!(pacer.pace(ms(0), start), start, "the first frame anchors at now"); + + // Steady state: the producer is 40ms behind, so this frame is already due. + let now = start + Duration::from_millis(80); + assert_eq!(pacer.pace(ms(40), now), now, "a late frame still goes out at once"); + + // The anchor absorbed those 40ms, so the next frame has room to be paced. + assert_eq!( + pacer.pace(ms(80), now), + now + Duration::from_millis(40), + "the discovered lag becomes the sink's margin" + ); + } + + /// The absorbed lag is a buffer the caller holds, so it is capped by the lead + /// rather than growing with every outlier. + #[test] + fn absorbed_lag_is_capped_by_the_lead() { + let start = Instant::now(); + let mut pacer = Pacer::default().with_lead(Duration::from_millis(50)); + assert_eq!(pacer.pace(ms(0), start), start); + + // 200ms behind, but only 50ms of that may be taken. + let now = start + Duration::from_millis(240); + assert_eq!(pacer.pace(ms(40), now), start + Duration::from_millis(90)); + assert_eq!( + pacer.pace(ms(80), now), + start + Duration::from_millis(130), + "the anchor moved by the cap, not by the shortfall" + ); + } + + /// Zero lead (SRT, whose receiver owns the jitter buffer) absorbs nothing. + #[test] + fn zero_lead_absorbs_nothing() { + let start = Instant::now(); + let mut pacer = Pacer::default(); + assert_eq!(pacer.pace(ms(0), start), start); + + let now = start + Duration::from_millis(80); + assert_eq!(pacer.pace(ms(40), now), start + Duration::from_millis(40)); + assert_eq!(pacer.pace(ms(80), now), start + Duration::from_millis(80)); + } + #[test] fn hurry_makes_the_frame_the_live_edge() { let start = Instant::now(); diff --git a/test/justfile b/test/justfile index 5b775991811..1fe00caa513 100644 --- a/test/justfile +++ b/test/justfile @@ -90,7 +90,9 @@ smoke-negative *args: # ts` output with TSDuck plus a custom analyzer (PCR jitter/repetition, # burstiness, instantaneous bitrate, T-STD). Flags: `--analyze-only file.ts` # (skip the round-trip), `--strict` (fail on shape warnings), `--source file.ts` -# (use a real capture). See ts/README.md. +# (use a real capture), `--live` (grade PCR release timing and byte position off +# the pipe instead, which a capture cannot carry; nightly runs this arm). See +# ts/README.md. # Check the subscriber's MPEG-TS output for IRD compliance. ts *args: diff --git a/test/ts/README.md b/test/ts/README.md index a22896b4830..d40349b84f8 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -11,6 +11,10 @@ is VBR, inserts no null packets, and paces PCR once per media frame, so several broadcast-shape checks are expected to flag. The report quantifies exactly where and by how much. +Two instruments live here. `compliance.py` (via `run.sh`) grades a captured file +against the IRD model. [`pcr-timing.py`](#pcr-timing-pcr-timingpy) grades a live +pipe, which is the only way to see *when* the exporter released each PCR. + ## Running ```bash @@ -19,8 +23,19 @@ just test ts --source cap.ts # round-trip a real capture instead just test ts --analyze-only x.ts # skip the round-trip, analyze a file just test ts --strict # also fail on broadcast-shape warnings just test ts --with-eit # add a synthetic EPG first, report which SI survived +just test ts --live # grade PCR release timing off the live pipe ``` +`--live` swaps the analyzer, not the rig: the same round-trip runs, but the +subscriber's stdout goes straight into `pcr-timing.py` instead of a capture file. +That is the only arm that can see release timing at all, and it is what nightly +runs (see [CI](#ci)). + +The arm passes only when the grader's verdict *and* the publisher's exit status +are clean. The grader can only speak for what reached it, and the sample floor +rejects a window that came up short, so a publisher dying late in the run leaves +enough behind to pass every check: a broken round-trip reported as a good one. + `--analyze-only` needs only TSDuck + Python, so you can point it at any captured subscriber output: @@ -73,6 +88,100 @@ Thresholds are CLI flags forwarded through `run.sh` (e.g. `--tb-size-bytes`, `--video-leak-bps`, `--audio-leak-bps`). `--report-json ` writes the full machine-readable report. +## PCR timing (`pcr-timing.py`) + +A PCR makes three claims at once, and an instrument pointed at one of them +cannot see the other two: + +| Domain | What it claims | Graded from | +|---|---|---| +| value | consecutive PCR values are spaced within the repetition limit | the values | +| release | the bytes carrying a PCR were handed over when that PCR asserts | arrival stamps | +| position | a PCR packet sits among the media bytes it describes | packet offsets | + +`compliance.py` grades `value` from a file, deterministically and with no +wall-clock capture, which is the right basis for the model math it does. That +also means it cannot grade `release`: a change to *when* the exporter hands bytes +over is invisible to any harness that does not stamp arrivals. +`pcr-timing.py` reads a pipe and grades all three in one pass. + +```bash +# live: all three domains, reading the exporter directly +moq --client-connect http://localhost:4443 --broadcast live.hang export ts \ + | ./pcr-timing.py --live --seconds 45 + +# offline: value and position only (a file has no release timing left in it) +./pcr-timing.py capture.ts +``` + +It needs only `python3`, with no TSDuck, no source file and no declared mux rate, +because every check is graded against the stream's **own** PCR values. If two +consecutive PCRs are 25 ms apart in value then they must be ~25 ms apart in +arrival, whatever clock rate the stream is running at. The price of that basis is +the same one `compliance.py` pays: a PCR emitted at the wrong rate stays +internally consistent, so absolute rate is not what this grades. + +| Check | Severity | What it verifies | +|---|---|---| +| `sync` | hard | no invalid sync bytes / transport-error packets | +| `continuity` | hard | no discontinuities, and a payload-less packet must not advance the counter (ISO 13818-1 2.4.3.3) | +| `pcr-single-pid` | hard | every PCR rides one PID | +| `pcr-value-interval` | hard | no interval above `--repetition-ms` (default 40, TR 101 290), within one time base | +| `pcr-release-timing` | hard | no more than `--release-pct-max` of intervals arrive further than `--release-ms` from the interval their own values assert, and accumulated drift stays within `--drift-ms`, being the standing lag the sender is allowed to hold; a sample below `--live-min-pcr` PCRs or `--live-cover-pct` of the window is a failure, not a pass (`--live` only) | +| `pcr-position` | shape | share of PCR packets within `--adjacent-packets` of the previous one | + +Accumulated drift has two shapes and only one is a defect, so the check bounds +the total and reports the rate over the tail of the sample beside it. A sender +that buffers builds a standing lag once and then runs at the media rate: the lag +is a constant offset no receiver can see, and it cannot grow past the latency +budget the sender is allowed to hold, so set `--drift-ms` to that budget +(`export ts --latency-max`, 500 ms by default). A pipe that is not running at the +media rate never stops accumulating and so breaches any fixed bound given a long +enough sample. The tail rate is what tells the two apart, and it needs a sample +longer than the lag takes to build: measured against the grid-sliced exporter, +the lag reaches ~480 ms over the first ~48 s and then holds to within 0.02 ms/s +over the following 40 s, whereas a 20 s window catches it still filling at +\~8.7 ms/s and cannot distinguish that from a slow pipe. + +Two conditions are legal and must not be reported as defects. ISO 13818-1 2.4.3.4 +lets a source declare a new time base by setting `discontinuity_indicator`, after +which the next PCR is a fresh value rather than the next point on the old ramp, so +the difference across that boundary measures nothing: the value and release checks +drop that one interval and count it separately, drift is summed over the intervals +actually graded, and the jump is not mistaken for a 33-bit wrap. Counting it read a +signalled splice as an 820 ms repetition breach and a continuity error at once. + +Separately, "not measured" and "passed" are different answers and only one of them +belongs to a file. A file carries no arrival stamps, so release timing genuinely +cannot be graded and the check says so. Under `--live` the stamps were expected, and +a sample too small to mean anything is a truncated capture rather than a clean +stream, so it fails: without that, a producer which emitted two packets and exited +turned the gate green, and an exporter exiting early is common enough that the route +is a real one rather than a hypothetical. + +`pcr-position` is a shape check because `export ts` is VBR by design. It is worth +reporting even so: a consumer holding only the byte stream, which is every +MPEG-TS tool, recovers the clock from where the PCR packets sit, so a layout +that clusters them and heaps the media bytes between the clusters is one such a +consumer cannot follow, however exact the values are. + +When both `release` and `position` flag, the report cross-tabulates them: + +```text + release timing by byte position (report only) + adjacent + early 615 ( 34.0%) + adjacent + on time 408 ( 22.5%) + spaced + on time 641 ( 35.4%) + spaced + late 136 ( 7.5%) +``` + +Two invariants failing on the *same* PCRs is one cause rather than two, which two +aggregate percentages cannot show. It is report-only: it explains a failure, it +does not define one. + +`--report-json ` writes the full report, and `--strict` promotes the shape +check to hard. + ## EIT fixture `make-eit-fixture.sh` adds a synthetic DVB EPG to any transport stream, so the @@ -122,12 +231,22 @@ matrix (nightly, on demand, and on PRs touching `test/ts/`). TSDuck comes from the `nix develop` shell, so the run uses the same `tsp`/`tsanalyze` a local developer would. +`.github/workflows/nightly.yml` runs `just test ts --live --duration 120` as +well. It stays off the PR path because release timing needs a real-time window to +measure: the grader has to sit on the pipe for the length of the capture, which +no per-PR gate should be paying for, and a scheduled arm on a quiet runner is a +better place to notice the exporter drifting off its own clock anyway. The window +is 120 s rather than the 20 s default because that is what the drift term costs +to read: under a minute it catches the mux buffer still filling and cannot tell +that from a pipe running slow. + ## Caveats - Physical-layer TR 101 290 items (RF, real sync-byte loss) cannot be measured from a file; TSDuck notes the same limitation. -- Wall-clock delivery jitter/burstiness is intentionally out of scope: all timing - is derived from the stream's PCR, not from socket arrival times. +- Wall-clock delivery jitter/burstiness is out of scope *for `compliance.py`*: + all of its timing is derived from the stream's PCR, not from arrival times. + `pcr-timing.py --live` covers that axis separately, by stamping a pipe. - `tstd` models only the transport-buffer (TB) smoothing stage of the ISO 13818-1 T-STD, not the full multiplex/elementary decode buffers. Its leak rates are defaults, not level-derived, so treat overflow as a smell rather than proof. diff --git a/test/ts/pcr-timing.py b/test/ts/pcr-timing.py new file mode 100755 index 00000000000..4560b77bf2b --- /dev/null +++ b/test/ts/pcr-timing.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 +"""Grade an MPEG-TS stream's PCR against its own asserted clock. + + moq ... export ts | ./pcr-timing.py --live --seconds 45 + ./pcr-timing.py capture.ts + +A PCR is three claims at once, and a fix in one of them is invisible to an +instrument pointed at another: + + value the intervals between PCR values (ISO 13818-1 / TR 101 290) + release when the bytes carrying them were handed over (--live only) + position where the PCR packets sit among the media bytes + +Every check is an invariant on the stream, not an assertion about how the stream +was produced. `release` grades arrival against the PCR's *own* values rather than +against a wall clock, so it needs no reference clock, no source file and no +declared mux rate; the price is that a clock running at the wrong rate stays +internally consistent under it and has to be caught by `value` and `position`. + +Exit status is 0 when every hard check passes, 1 otherwise. `--strict` promotes +the report-only (shape) checks to hard. +""" + +import argparse +import collections +import json +import select +import statistics +import sys +import time + +PKT = 188 +SYNC = 0x47 +HARD, SHAPE = "hard", "shape" +TICKS_PER_MS = 27_000.0 +# The 33-bit PCR base counts 90 kHz ticks and the 9-bit extension counts 27 MHz ticks +# within one of them, so the whole field restarts after this many 27 MHz ticks, roughly +# every 26.5 hours. +PCR_MODULUS = (1 << 33) * 300 + + +def percentile(xs, p): + s = sorted(xs) + if not s: + return float("nan") + k = min(len(s) - 1, max(0, int(round(p / 100.0 * (len(s) - 1))))) + return s[k] + + +def parse_pcr(p): + """The 33+9-bit PCR of an adaptation field, in 27 MHz ticks, or None.""" + if (p[3] >> 4) & 0x3 not in (2, 3): # adaptation_field_control + return None + # A PCR occupies six bytes after the flags byte, so an adaptation field shorter than + # seven cannot hold one however the flag is set. Without this a stream that sets + # PCR_flag on a short field yields a value read out of stuffing or payload, and that + # sample then feeds all three timing checks. + if p[4] < 7 or not (p[5] & 0x10): # adaptation_field_length, PCR_flag + return None + base = (p[6] << 25) | (p[7] << 17) | (p[8] << 9) | (p[9] << 1) | (p[10] >> 7) + ext = ((p[10] & 0x01) << 8) | p[11] + return base * 300 + ext + + +class Scan: + """One pass over the packets, collecting only what the checks need.""" + + def __init__(self): + self.pcr = [] # (packet_index, ticks, arrival_or_None, pid) + self.packets = 0 + self.bad_sync = 0 + self.transport_error = 0 + self.cc_errors = [] + self.cc_on_empty = [] + self.wraps = 0 + self.new_base = set() # positions in self.pcr that state a new time base + self._cc = {} + self._dup = {} + self._new_base = set() # PIDs whose next PCR states a new time base + self._pcr_raw = {} + self._pcr_offset = {} + + def feed(self, p, arrival=None): + index = self.packets + self.packets += 1 + if p[0] != SYNC: + self.bad_sync += 1 + return + if p[1] & 0x80: + self.transport_error += 1 + pid = ((p[1] & 0x1F) << 8) | p[2] + + # ISO 13818-1 2.4.3.3 allows the counter to jump when the adaptation field sets + # discontinuity_indicator, and allows one packet to be sent twice, repeating its + # counter. Both are legal, and treating either as an error fails a conforming + # stream on a hard check. Read the flag before the PCR, because 2.4.3.4 makes the + # same flag mean the clock may jump too, and the value and release checks both need + # to know which of their intervals spans that. + cc = p[3] & 0x0F + payload = bool((p[3] >> 4) & 0x1) + adaptation = bool((p[3] >> 4) & 0x2) + discontinuity = adaptation and p[4] > 0 and bool(p[5] & 0x80) + + ticks = parse_pcr(p) + if ticks is not None: + # Unwrap, so the checks see a monotone timeline. A capture crossing the + # rollover would otherwise show one huge negative interval, which the value + # check accepts (it only rejects intervals that are too long) while the + # release check reads it as a day of lateness. A rollover drops the value by + # nearly the whole modulus, so only a drop past halfway is treated as one and + # a merely backwards PCR stays visible as the defect it is. + # + # A signalled discontinuity is the exception in both directions: the next PCR + # states a new time base, so a drop across it is neither a wrap to unwrap nor a + # backwards clock to report, and the interval spanning it is not a measurement + # of anything. Mark it instead, and let the checks drop that one interval. + if discontinuity or pid in self._new_base: + self._new_base.discard(pid) + self.new_base.add(len(self.pcr)) + self._pcr_raw.pop(pid, None) + else: + prev_raw = self._pcr_raw.get(pid) + if prev_raw is not None and ticks < prev_raw - PCR_MODULUS // 2: + self._pcr_offset[pid] = self._pcr_offset.get(pid, 0) + PCR_MODULUS + self.wraps += 1 + self._pcr_raw[pid] = ticks + self.pcr.append((index, ticks + self._pcr_offset.get(pid, 0), arrival, pid)) + elif discontinuity: + # The flag rode a packet with no PCR in it, so the new base arrives with the + # next PCR on this PID rather than with this packet. + self._new_base.add(pid) + + prev = self._cc.get(pid) + self._cc[pid] = cc + if prev is None or pid == 0x1FFF: + return + if discontinuity: + self._dup[pid] = False + return + if payload: + if cc == prev: + # Legal only once: a second repeat is a stuck counter, not a duplicate. + if self._dup.get(pid): + self.cc_errors.append((index, pid, (prev + 1) & 0x0F, cc)) + self._dup[pid] = not self._dup.get(pid) + elif cc == (prev + 1) & 0x0F: + self._dup[pid] = False + else: + self.cc_errors.append((index, pid, (prev + 1) & 0x0F, cc)) + self._dup[pid] = False + elif cc != prev: + self.cc_on_empty.append((index, pid, prev, cc)) + + +def scan_file(path): + scan = Scan() + with open(path, "rb") as f: + while True: + p = f.read(PKT) + if len(p) < PKT: + return scan + scan.feed(p) + + +def scan_live(seconds): + """Read stdin one packet at a time, stamping each read. + + The granularity is deliberate: a coarser read cannot distinguish a run of + packets released together from a run released on a cadence, which is the + distinction `release` exists to measure. Each stamp is taken after its read + returns, so it is an upper bound on when the writer released those bytes. + """ + scan = Scan() + fd = sys.stdin.buffer + deadline = time.monotonic() + seconds + + def read_exactly(n): + """Read n bytes, or return None once the capture window has expired. + + A plain read blocks until the producer sends something, so a producer that + holds the pipe open and stops writing suspends the loop indefinitely and + --seconds never takes effect. That is the likeliest state to be in while + diagnosing an exporter or a network stall, which is when the tool has to + return a report rather than hang. + """ + buf = b"" + while len(buf) < n: + left = deadline - time.monotonic() + if left <= 0 or not select.select([fd], [], [], left)[0]: + return None + chunk = fd.read1(n - len(buf)) + if not chunk: + return None + buf += chunk + return buf + + # Find the first sync byte, then stay aligned on it. + while True: + b = read_exactly(1) + if not b: + return scan + if b[0] == SYNC: + rest = read_exactly(PKT - 1) + if not rest: + return scan + scan.feed(b + rest, time.monotonic()) + break + + while time.monotonic() < deadline: + p = read_exactly(PKT) + if not p: + return scan + scan.feed(p, time.monotonic()) + return scan + + +# --- checks: each returns (name, severity, ok, headline, detail) ------------- + + +def check_sync(scan, args): + detail = { + "packets": scan.packets, + "bad_sync": scan.bad_sync, + "transport_error": scan.transport_error, + } + return ( + "sync", + HARD, + not (scan.bad_sync or scan.transport_error), + f"{scan.packets} packets, {scan.bad_sync} bad sync bytes, " + f"{scan.transport_error} with transport_error_indicator", + detail, + ) + + +def check_continuity(scan, args): + detail = { + "discontinuities": len(scan.cc_errors), + "empty_packets_advancing_cc": len(scan.cc_on_empty), + "first_few": scan.cc_errors[:5], + } + return ( + "continuity", + HARD, + not (scan.cc_errors or scan.cc_on_empty), + f"{len(scan.cc_errors)} continuity discontinuities, {len(scan.cc_on_empty)} payload-less " + f"packets advanced the counter (ISO 13818-1 2.4.3.3)", + detail, + ) + + +def check_pcr_single_pid(scan, args): + """Every PCR must ride the one PID the PMT declares.""" + pids = collections.Counter(pid for _, _, _, pid in scan.pcr) + return ( + "pcr-single-pid", + HARD, + len(pids) <= 1, + f"PCR carried on {len(pids)} PID(s): " + ", ".join(f"{k} ({v})" for k, v in pids.most_common()), + {"pids": {str(k): v for k, v in pids.items()}}, + ) + + +def check_value_interval(scan, args): + """PCR values must be spaced within the repetition limit. + + Within one time base. ISO 13818-1 2.4.3.4 lets a source declare a new one by setting + discontinuity_indicator, after which the next PCR is a fresh value rather than the next + point on the old ramp, so the difference across that boundary measures nothing. Counting + it fails a conforming stream: a signalled splice reads as one enormous interval, which is + the same class of false positive review found in the continuity check. + """ + iv = [ + (b[1] - a[1]) / TICKS_PER_MS + for i, (a, b) in enumerate(zip(scan.pcr, scan.pcr[1:])) + if (i + 1) not in scan.new_base + ] + spliced = sum(1 for i in range(1, len(scan.pcr)) if i in scan.new_base) + if not iv: + return ("pcr-value-interval", HARD, False, "no PCR intervals in the stream", {}) + over = [m for m in iv if m > args.repetition_ms] + # A PCR that goes backwards is as much a clock defect as one that arrives too late, + # and testing only the upper bound makes it invisible: the negative interval passes, + # and it drags the median and the over-limit share down with it. Rollover is already + # unwrapped in the scan, so a negative interval here is the stream's own doing. Zero + # is not: ISO 13818-1 2.4.3.3 allows a packet to be sent twice, and the duplicate + # repeats its PCR exactly, so testing `<= 0` would fail a conforming stream. + backwards = [m for m in iv if m < 0] + detail = { + "count": len(iv), + "median_ms": round(statistics.median(iv), 3), + "p95_ms": round(percentile(iv, 95), 3), + "max_ms": round(max(iv), 3), + "over_limit": len(over), + "over_limit_pct": round(100.0 * len(over) / len(iv), 2), + "sub_ms": sum(1 for m in iv if m < 1.0), + "non_positive": len(backwards), + "wraps_unwrapped": scan.wraps, + "signalled_discontinuities": spliced, + } + backwards_note = f", {len(backwards)} non-positive" if backwards else "" + if spliced: + backwards_note += f", {spliced} signalled discontinuity not counted" + return ( + "pcr-value-interval", + HARD, + not over and not backwards, + f"{len(over)}/{len(iv)} intervals over {args.repetition_ms:g} ms " + f"(median {detail['median_ms']:g} ms, worst {detail['max_ms']:g} ms{backwards_note})", + detail, + ) + + +def check_release(scan, args): + """The bytes carrying a PCR must be released at the time that PCR asserts. + + Graded against the stream's own values, so it holds at any clock rate: if two + consecutive PCRs are 25 ms apart in value they must be ~25 ms apart in + arrival. Two statistics, because they fail independently: per-interval error, + which is what a receiver PLL and any downstream re-timing stage sees, and + accumulated drift, which must stay bounded or the pipe is not running at the + media rate at all. + + Bounded is the whole of the drift requirement, and the bound is the sender's + latency budget. An exporter that buffers builds a standing lag once, at + startup, and then runs at the media rate; the lag is a constant offset, which + no receiver can see and which cannot grow past the budget the sender is + allowed to hold. A pipe running slow never stops accumulating and so breaches + any fixed bound given a long enough sample. So the total is the gate and the + lag's rate over the tail of the sample is reported beside it, which is what + distinguishes a lag that has settled from one still growing. + """ + stamped = [i for i, (_, _, arrival, _) in enumerate(scan.pcr) if arrival is not None] + pts = [(scan.pcr[i][1], scan.pcr[i][2]) for i in stamped] + # scan.new_base indexes scan.pcr; pts drops the unstamped entries, so remap. + live_new_base = {j for j, i in enumerate(stamped) if i in scan.new_base} + # A file has no arrival stamps in it, so there is nothing to grade and saying so is + # the honest verdict. Live is the opposite case: arrivals were expected, and too few + # of them is a truncated capture rather than a clean stream. Passing that is how a + # gate reports success on a producer that died, which is the state this tool exists + # to catch, so the two have to be told apart rather than sharing one branch. + if not args.live: + if len(pts) < 3: + return ("pcr-release-timing", HARD, True, "not measured (no arrival stamps)", {}) + else: + covered_ms = (pts[-1][0] - pts[0][0]) / TICKS_PER_MS if len(pts) >= 2 else 0.0 + want_ms = 1000.0 * args.seconds * args.live_cover_pct / 100.0 + short = len(pts) < args.live_min_pcr or covered_ms < want_ms + if short: + return ( + "pcr-release-timing", + HARD, + False, + f"insufficient live sample: {len(pts)} PCR with arrival stamps spanning " + f"{covered_ms / 1000.0:.3f} s, against a floor of {args.live_min_pcr} PCR and " + f"{want_ms / 1000.0:g} s ({args.live_cover_pct:g} % of the {args.seconds:g} s " + f"window). A truncated capture cannot be graded and must not pass", + { + "count": len(pts), + "covered_s": round(covered_ms / 1000.0, 3), + "required_pcr": args.live_min_pcr, + "required_s": round(want_ms / 1000.0, 3), + }, + ) + + # Same exclusion as the value check: an interval spanning a signalled new time base is + # not a release measurement, and counting it reports a splice as seconds of lateness. + graded = [ + (((b[1] - a[1]) * 1000.0) - ((b[0] - a[0]) / TICKS_PER_MS), (b[0] - a[0]) / TICKS_PER_MS) + for i, (a, b) in enumerate(zip(pts, pts[1:])) + if (i + 1) not in live_new_base + ] + err = [e for e, _ in graded] + span = [s for _, s in graded] + if not err: + return ("pcr-release-timing", HARD, False, "no gradable release intervals", {}) + magnitude = [abs(e) for e in err] + # Accumulated drift is the sum of the intervals actually graded. That telescopes to the + # endpoint difference on an unspliced sample, and stays correct on a spliced one, where + # the endpoints straddle a time base the stream never claimed to be running on. + drift = sum(err) + late = [e for e in err if e > args.release_ms] + early = [e for e in err if e < -args.release_ms] + detail = { + "count": len(err), + "median_abs_ms": round(statistics.median(magnitude), 3), + "p95_abs_ms": round(percentile(magnitude, 95), 3), + "p99_abs_ms": round(percentile(magnitude, 99), 3), + "max_abs_ms": round(max(magnitude), 3), + "outside_tolerance": len(late) + len(early), + "outside_tolerance_pct": round(100.0 * (len(late) + len(early)) / len(err), 2), + "released_early": len(early), + "released_late": len(late), + "total_drift_ms": round(drift, 1), + "drift_limit_ms": args.drift_ms, + } + # Whether the lag has settled is the difference between a buffer that filled and a + # pipe running slow, and the two are only separable over a sample longer than the + # fill. Grade the last third: a settled lag adds nothing to it. + cut = (2 * len(err)) // 3 + tail_span_ms = sum(span[cut:]) + if tail_span_ms > 0: + detail["tail_drift_rate_ms_per_s"] = round(1000.0 * sum(err[cut:]) / tail_span_ms, 3) + detail["tail_span_s"] = round(tail_span_ms / 1000.0, 1) + # Per-interval error and accumulated drift fail independently, and bounding only the + # first leaves the second unbounded: a consistent 1 ms error on a 25 ms grid sits well + # inside a 10 ms tolerance while reaching nearly two seconds over a 45 s sample. The + # docstring claimed drift was bounded; until now only the per-interval error was. + drifted = abs(drift) > args.drift_ms + drift_note = f" > ±{args.drift_ms:g} ms" if drifted else "" + rate = detail.get("tail_drift_rate_ms_per_s") + tail_note = "" if rate is None else f", {rate:+g} ms/s over the last {detail['tail_span_s']:g} s" + over = detail["outside_tolerance_pct"] > args.release_pct_max + return ( + "pcr-release-timing", + HARD, + not (over or drifted), + f"{detail['outside_tolerance']}/{len(err)} releases outside ±{args.release_ms:g} ms of the " + f"interval the PCR asserts ({detail['released_early']} early, {detail['released_late']} late; " + f"p95 {detail['p95_abs_ms']:g} ms, worst {detail['max_abs_ms']:g} ms; " + f"drift {detail['total_drift_ms']:g} ms{drift_note}{tail_note})", + detail, + ) + + +def check_position(scan, args): + """A PCR packet must sit among the media bytes whose arrival it describes. + + The grid is uniform in media time, so if position tracks time then the packet + gap between consecutive PCRs is roughly uniform too. That needs no assumption + that the stream is CBR, only that comparable spans of media time carry + comparable numbers of bytes. What it catches is a bimodal layout, PCR packets + laid back-to-back with the media bytes they label heaped between the clusters. + A consumer holding only the byte stream cannot recover the clock from such a + layout, and one that re-stamps PCR from byte position regenerates exactly the + clustering the value domain was fixed to remove. + """ + if len(scan.pcr) < 3: + return ("pcr-position", SHAPE, True, "not measured (too few PCRs)", {}) + gap = [b[0] - a[0] for a, b in zip(scan.pcr, scan.pcr[1:])] + adjacent = sum(1 for g in gap if g <= args.adjacent_packets) + median = statistics.median(gap) + detail = { + "count": len(gap), + "median_packets": median, + "p95_packets": percentile(gap, 95), + "max_packets": max(gap), + "adjacent": adjacent, + "adjacent_pct": round(100.0 * adjacent / len(gap), 2), + # Dispersion about the middle of the distribution: near 1 for a uniform + # layout, large for cluster-and-hole. Guard a zero median. + "p95_over_median": round(percentile(gap, 95) / median, 1) if median else None, + } + return ( + "pcr-position", + SHAPE, + detail["adjacent_pct"] <= args.adjacent_pct_max, + f"{detail['adjacent_pct']:g}% of PCR packets sit within {args.adjacent_packets} packet(s) of " + f"the previous one (median gap {median} packets, p95 {detail['p95_packets']}, worst {max(gap)})", + detail, + ) + + +CHECKS = [ + check_sync, + check_continuity, + check_pcr_single_pid, + check_value_interval, + check_release, + check_position, +] + + +def coincidence(scan, args): + """Cross-tabulate release error against byte position, for one pass's PCRs. + + `release` and `position` are separate invariants and a stream can fail either + alone, but when they fail *on the same PCRs* they have one cause rather than + two, and that is worth reporting rather than leaving to be inferred from two + aggregate percentages. Report-only: it explains a failure, it does not define + one. + """ + pts = [(i, ticks, arrival) for i, ticks, arrival, _ in scan.pcr if arrival is not None] + if len(pts) < 3: + return None + rows = collections.Counter() + for a, b in zip(pts, pts[1:]): + err = ((b[2] - a[2]) * 1000.0) - ((b[1] - a[1]) / TICKS_PER_MS) + when = "early" if err < -args.release_ms else "late" if err > args.release_ms else "on time" + where = "adjacent" if (b[0] - a[0]) <= args.adjacent_packets else "spaced" + rows[(where, when)] += 1 + return rows + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("path", nargs="?", help="TS file; omit with --live to read stdin") + ap.add_argument("--live", action="store_true", help="read stdin and stamp arrivals, grading release timing") + ap.add_argument("--seconds", type=float, default=45.0, help="live capture window (default 45)") + ap.add_argument( + "--repetition-ms", type=float, default=40.0, help="max PCR value interval, TR 101 290 (default 40)" + ) + ap.add_argument( + "--release-ms", + type=float, + default=10.0, + help="tolerance on release timing against the asserted interval (default 10)", + ) + ap.add_argument( + "--release-pct-max", + type=float, + default=0.0, + help="share of intervals allowed outside the release tolerance (default 0)", + ) + ap.add_argument( + "--live-min-pcr", + type=int, + default=20, + help="fewest PCR samples a --live run may grade before it is a truncated capture " + "rather than a result (default 20)", + ) + ap.add_argument( + "--live-cover-pct", + type=float, + default=50.0, + help="share of the --seconds window a --live sample must span, or it is treated as " + "truncated (default 50)", + ) + ap.add_argument( + "--drift-ms", + type=float, + default=500.0, + help="bound on accumulated release drift, being the standing lag the sender may " + "hold; set it to the sender's latency budget (moq export ts --latency-max, " + "itself 500ms by default)", + ) + ap.add_argument( + "--adjacent-packets", type=int, default=1, help="packet gap at or below which two PCRs count as clustered" + ) + ap.add_argument( + "--adjacent-pct-max", type=float, default=1.0, help="share of clustered PCRs before pcr-position flags" + ) + ap.add_argument("--strict", action="store_true", help="fail on shape checks too") + ap.add_argument("--report-json", help="write the full report here") + args = ap.parse_args() + + if args.live: + scan = scan_live(args.seconds) + elif args.path: + scan = scan_file(args.path) + else: + ap.error("give a path or --live") + + results = [check(scan, args) for check in CHECKS] + width = max(len(r[0]) for r in results) + + print(f"### PCR timing report - {scan.packets} packets, {len(scan.pcr)} PCR") + print() + failed = 0 + for name, severity, ok, headline, _ in results: + if ok: + verdict = "PASS" + elif severity == HARD or args.strict: + verdict = "FAIL" + failed += 1 + else: + verdict = "WARN" + print(f" {verdict:4} {name:<{width}} {headline}") + rows = coincidence(scan, args) + if rows: + total = sum(rows.values()) + print() + print(" release timing by byte position (report only)") + for where in ("adjacent", "spaced"): + for when in ("early", "on time", "late"): + n = rows[(where, when)] + if n: + print(f" {where:<8} + {when:<7} {n:6} ({100.0 * n / total:5.1f}%)") + + print() + print(f"{failed} check(s) failed" if failed else "all checks passed") + + if args.report_json: + with open(args.report_json, "w") as f: + json.dump( + { + "packets": scan.packets, + "pcr_count": len(scan.pcr), + "live": args.live, + "checks": [ + {"name": n, "severity": s, "ok": ok, "headline": h, "detail": d} + for n, s, ok, h, d in results + ], + }, + f, + indent=2, + ) + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/run.sh b/test/ts/run.sh index ca2a209eefe..fdbeb3cdb4d 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -15,6 +15,16 @@ # ./run.sh --analyze-only cap.ts # skip the round-trip, just analyze a file # ./run.sh --strict # fail on broadcast-shape warnings too # ./run.sh --with-eit # add a synthetic EPG first, report which SI survived +# ./run.sh --live # grade PCR release timing off the live pipe + +# `--live` swaps the analyzer, not the rig. compliance.py grades a captured file +# on the stream's own PCR clock, which is the right basis for the IRD model it +# builds and is why it needs no wall-clock capture -- and also why nothing it +# checks can see *when* the exporter handed the bytes over. pcr-timing.py reads +# the subscriber's stdout a packet at a time and stamps each read, so it grades +# release timing and byte position alongside the values. Nightly runs this arm +# (.github/workflows/nightly.yml); it is not a per-PR gate, because it needs a +# real-time window to measure at all. set -euo pipefail DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -28,6 +38,7 @@ PORT="${TSC_PORT:-4443}" PROFILE="${TSC_PROFILE:-debug}" STRICT="" WITH_EIT="" # add a synthetic EPG to the source and report which SI survived +LIVE="" # grade the exporter's stdout as it arrives, rather than a capture PASSTHRU=() # forwarded to compliance.py (thresholds, --report-json, ...) while [[ $# -gt 0 ]]; do @@ -60,6 +71,10 @@ while [[ $# -gt 0 ]]; do WITH_EIT=1 shift ;; + --live) + LIVE=1 + shift + ;; *) PASSTHRU+=("$1") shift @@ -167,6 +182,10 @@ if [[ -n "$SOURCE" ]]; then } else echo "### generating ~${DURATION}s broadcast-like clip with ffmpeg" + # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` + # paces on the source PCR, so a clip whose clock is coarse and whose rate is + # unconstrained is released unevenly and finishes early (measured: a 20 s clip + # in 17 s, and release jitter of its own). The harness then grades ffmpeg. ffmpeg -y -hide_banner -loglevel error \ -f lavfi -i "testsrc=size=1280x720:rate=25" \ -f lavfi -i "sine=frequency=1000:sample_rate=48000" \ @@ -174,7 +193,7 @@ else -c:v libx264 -profile:v high -preset veryfast -pix_fmt yuv420p \ -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 8M \ -c:a aac -b:a 128k \ - -f mpegts -pes_payload_size 0 "$SRC_TS" + -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" fi # No capture in this repository carries EIT, so the import path's EIT handling is @@ -201,21 +220,49 @@ fi # Start the subscriber first so it is waiting on the announce before the # publisher appears; a live broadcast has no history, so a late joiner would miss # the start of the stream (or the whole thing for a short clip). -echo "### capturing subscriber output (export ts)" -timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --client-connect "$URL" --broadcast "$BROADCAST" export ts >"$SUB_TS" 2>"$TMP/sub.log" & -SUB_PID=$! +# +# In --live mode the grader reads that stdout directly rather than a capture: it +# stamps each 188-byte read, which is the only way release timing survives at all +# (a file has none left in it). It stops itself after its own window, so the +# round-trip below still bounds the run. +if [[ -n "$LIVE" ]]; then + echo "### grading subscriber output live (export ts | pcr-timing.py)" + # Both halves matter and `wait` can only report one, so record each. The + # exporter's own status is not incidental here: it decides whether the grader + # saw the whole window or graded a stream that ended under it. + ( + # `set -e` would abort the subshell on the pipeline's own failure, which is + # exactly the status we are here to record. + set +e + timeout -k 3 $((DURATION + 20)) \ + "$MOQ" --client-connect "$URL" --broadcast "$BROADCAST" export ts 2>"$TMP/sub.log" | + python3 "$DIR/pcr-timing.py" --live --seconds "$DURATION" --release-pct-max 1 $STRICT \ + ${PASSTHRU[@]+"${PASSTHRU[@]}"} >"$TMP/timing.out" 2>&1 + printf '%s\n' "${PIPESTATUS[0]} ${PIPESTATUS[1]}" >"$TMP/timing.rc" + ) & + SUB_PID=$! +else + echo "### capturing subscriber output (export ts)" + timeout -k 3 $((DURATION + 20)) \ + "$MOQ" --client-connect "$URL" --broadcast "$BROADCAST" export ts >"$SUB_TS" 2>"$TMP/sub.log" & + SUB_PID=$! +fi sleep 1 # Pace on the source PCR (real media time), not a fixed bitrate: a synthetic clip # compresses tiny, so bitrate pacing would rush the whole stream out in a blink. +# `--wait-min 5` is what makes that pacing fine-grained enough to measure against: +# at the default, `regulate` releases in ~50 ms chunks, which is jitter of its own +# on top of whatever the exporter does (measured: p95 40 ms at the default, 1 ms +# at 5). It is the publisher's granularity, not the exporter's, so the harness has +# to be well inside it or it grades tsp. # Bounded like the subscriber: if the relay stalls after readiness, `moq import # ts` could otherwise block `wait "$PUB_PID"` forever. tsp/moq stderr lands in # pub.log, which the empty-capture handler below dumps on failure. echo "### publishing PCR-paced TS -> $BROADCAST" # shellcheck disable=SC2016 # $1..$4 are the child bash -c positionals, not ours. timeout -k 3 $((DURATION + 20)) bash -c ' - tsp -I file "$1" -P regulate --pcr-synchronous | + tsp -I file "$1" -P regulate --pcr-synchronous --wait-min 5 | "$2" --client-connect "$3" --broadcast "$4" import ts ' _ "$SRC_TS" "$MOQ" "$URL" "$BROADCAST" >"$TMP/pub.log" 2>&1 & PUB_PID=$! @@ -225,9 +272,6 @@ PUB_PID=$! # explain a truncated capture, so surface it alongside the logs on failure. wait "$PUB_PID" 2>/dev/null && PUB_RC=0 || PUB_RC=$? PUB_PID="" -sleep 3 -kill_tree "$SUB_PID" 2>/dev/null || true -SUB_PID="" # shellcheck disable=SC2329 # invoked from multiple failure paths below dump_logs() { @@ -236,6 +280,54 @@ dump_logs() { sed 's/^/ sub: /' "$TMP/sub.log" >&2 || true } +# ── live: the grader owns the verdict ─────────────────────────────────────── +if [[ -n "$LIVE" ]]; then + wait "$SUB_PID" 2>/dev/null || true + SUB_PID="" + if [[ ! -s "$TMP/timing.rc" ]]; then + echo "error: the live grader never reported a status" >&2 + dump_logs + exit 1 + fi + read -r EXPORT_RC GRADE_RC <"$TMP/timing.rc" + echo + cat "$TMP/timing.out" + # 124 is `timeout` reaching the end of the window, which is how the exporter is + # meant to stop; SIGPIPE (141) is the grader closing the pipe on its own window. + # Anything else ended the stream under the grader, so say so: it graded a + # shorter window than it was asked for, and the report alone doesn't show that. + # Not fatal, because the grader's verdict on what it did see is still the + # verdict, and a short window can only make the sample smaller, not kinder. + if [[ "$EXPORT_RC" -ne 0 && "$EXPORT_RC" -ne 124 && "$EXPORT_RC" -ne 141 ]]; then + echo >&2 + echo "warning: the exporter exited $EXPORT_RC before the window closed" >&2 + sed 's/^/ sub: /' "$TMP/sub.log" >&2 || true + fi + if [[ "$GRADE_RC" -ne 0 ]]; then + echo >&2 + echo "error: PCR timing analysis failed (see round-trip logs below)" >&2 + dump_logs + exit "$GRADE_RC" + fi + # The grader's verdict is not the whole run. It grades whatever reached it, and + # the sample floor only rejects a window that came up short: a publisher that + # dies late still leaves enough behind to pass every check. That is a broken + # round-trip reported as a good one, so the publisher's own status is a gate + # too. 124 is `timeout` killing a stalled `moq import ts`, which is a failure + # of the same kind rather than a clean end, so it is not excused here. + if [[ "$PUB_RC" -ne 0 ]]; then + echo >&2 + echo "error: the publisher exited $PUB_RC; the graded stream is not a whole round-trip" >&2 + dump_logs + exit 1 + fi + exit 0 +fi + +sleep 3 +kill_tree "$SUB_PID" 2>/dev/null || true +SUB_PID="" + if [[ ! -s "$SUB_TS" ]]; then echo "error: subscriber captured no data" >&2 dump_logs From 78f82e37e3763fc2f2c5bbc4cbfdb81d3ee99b62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 15:08:25 -0700 Subject: [PATCH 03/29] fix(obs): pin the plugin to OBS 32.2.2 and guard libobs CI compatibility (#3377) Co-authored-by: Claude Opus 5 Co-authored-by: GPT-5 --- cpp/obs/buildspec.json | 20 ++++---- cpp/obs/justfile | 46 ++++++++++++++++++- doc/bin/obs.md | 12 ++++- flake.nix | 31 +++++++++++-- justfile | 19 ++++++-- ...targets-obs-31-1-1-while-linux-ci-links.md | 30 ------------ quest/m0/README.md | 1 - 7 files changed, 106 insertions(+), 53 deletions(-) delete mode 100644 quest/m0/2868-obs-the-plugin-targets-obs-31-1-1-while-linux-ci-links.md diff --git a/cpp/obs/buildspec.json b/cpp/obs/buildspec.json index f4e18bad5b0..35dd8f23326 100644 --- a/cpp/obs/buildspec.json +++ b/cpp/obs/buildspec.json @@ -1,33 +1,33 @@ { "dependencies": { "obs-studio": { - "version": "31.1.1", + "version": "32.2.2", "baseUrl": "https://github.com/obsproject/obs-studio/archive/refs/tags", "label": "OBS sources", "hashes": { - "macos": "39751f067bacc13d44b116c5138491b5f1391f91516d3d590d874edd21292291", - "windows-x64": "2c8427c10b55ac6d68008df2e9a3e82f4647aaad18f105e30d4713c2de678ccf" + "macos": "35d3cd0979d65664fada7119fdb612eca7c34b61a1623a330caec74bf72626c4", + "windows-x64": "f15f001f1fa526405318835f44f9910046502f496ebc3a30d5296a5018b831aa" } }, "prebuilt": { - "version": "2025-07-11", + "version": "2026-07-15", "baseUrl": "https://github.com/obsproject/obs-deps/releases/download", "label": "Pre-Built obs-deps", "hashes": { - "macos": "495687e63383d1a287684b6e2e9bfe246bb8f156fe265926afb1a325af1edd2a", - "windows-x64": "c8c642c1070dc31ce9a0f1e4cef5bb992f4bff4882255788b5da12129e85caa7" + "macos": "4ecb4c598dfa853168df6c2a0c4e0ffec8495a81fbd1ba051ef88ecd5e0f7e53", + "windows-x64": "6f90e9598fa10cff5ad23cdcfae49b87868c07bf896b02cd464582b4ce2f2ba9" } }, "qt6": { - "version": "2025-07-11", + "version": "2026-07-15", "baseUrl": "https://github.com/obsproject/obs-deps/releases/download", "label": "Pre-Built Qt6", "hashes": { - "macos": "d3f5f04b6ea486e032530bdf0187cbda9a54e0a49621a4c8ba984c5023998867", - "windows-x64": "0e76bf0555dd5382838850b748d3dcfab44a1e1058441309ab54e1a65b156d0a" + "macos": "d4b8058612a7067e44b2205fe7925ee24e9ec6b15ac8c29d3e6230c70030b102", + "windows-x64": "7c7f985711d80467bdc1795b6592275a27d5b0e5a2c7a61db1f2c1d08d6a5579" }, "debugSymbols": { - "windows-x64": "11b7be92cf66a273299b8f3515c07a5cfb61614b59a4e67f7fc5ecba5e2bdf21" + "windows-x64": "471d0b2191c424a520a51d064f3741084f117e1ff6ee5af1d16aabcdbacc6659" } } }, diff --git a/cpp/obs/justfile b/cpp/obs/justfile index d62c571bb8e..65d5bc4863e 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -175,6 +175,26 @@ _includes: exit 1 fi + # obs.h reaches as of OBS 32, which used to be vendored + # under libobs/util/ and so came along with the headers above. The dev shell + # carries it (see flake.nix); the obs-deps bundle ships its own copy beside + # libobs, which is how the macOS and Windows builds get it. + simde_include="" + if command -v pkg-config >/dev/null && pkg-config --exists simde; then + simde_include=$(pkg-config --variable=includedir simde) + else + for candidate in .deps/obs-deps-*/include; do + if [ -f "$candidate/simde/x86/sse2.h" ]; then + simde_include="$candidate" + break + fi + done + fi + if [ -z "$simde_include" ]; then + echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 + exit 1 + fi + (cd ../.. && {{ cargo_compile }} check --locked --quiet -p libmoq) # Ask cargo where its build script ran rather than reconstructing the path. @@ -199,7 +219,7 @@ _includes: # One flag per line, path glued to the -I, so a caller can rebuild the # arguments without splitting on the spaces in a path. - printf -- '-I%s\n' src "$obs_include" "$moq_include" + printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" # Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under # ThreadSanitizer. @@ -310,6 +330,30 @@ check: echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 exit 1 fi + # And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS + # this repo doesn't pick. Without this the released macOS/Windows binaries + # could be a whole libobs API apart from what CI ever compiles. Patch + # releases carry no API change, so only the major.minor has to agree; that + # also keeps the guard quiet until flake.lock actually moves OBS, which is + # the change that opens the gap and so the one that should fail. + # OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg + # is skipped outside it, where there is no nixpkgs obs-studio to compare. + # MOQ_STRICT turns that skip into an error for the same reason it turns a + # missing tool into one: in CI a leg that checks nothing still reports green. + if [ -z "${OBS_LINKED_VERSION:-}" ]; then + if [ -n "${MOQ_STRICT:-}" ]; then + echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 + exit 1 + fi + else + spec_api=$(echo "$spec_obs" | cut -d. -f1,2) + linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) + if [ "$spec_api" != "$linked_api" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 + echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 + exit 1 + fi + fi if command -v cmake >/dev/null; then build_dir=$(mktemp -d) trap 'rm -rf "$build_dir"' EXIT diff --git a/doc/bin/obs.md b/doc/bin/obs.md index 2289760fba0..a134b4982a2 100644 --- a/doc/bin/obs.md +++ b/doc/bin/obs.md @@ -68,7 +68,7 @@ nix develop just obs compile ``` -This is the gate to run while working, because it needs headers rather than libraries, and the dev shell carries all of them on every platform. `libobs` comes from the `libobs-headers` package in `flake.nix`, which unpacks the headers from the same OBS release `buildspec.json` pins (`just obs check` fails if the two versions drift); `Qt6` and `ffmpeg` come from nixpkgs. It regenerates `target/include/moq.h` first, so a call to a `libmoq` function whose signature has since changed is a compile error rather than something you find out about later. +This is the gate to run while working, because it needs headers rather than libraries, and the dev shell carries all of them on every platform. `libobs` comes from the `libobs-headers` package in `flake.nix`, which unpacks the headers from the same OBS release `buildspec.json` pins; `Qt6` and `ffmpeg` come from nixpkgs. It regenerates `target/include/moq.h` first, so a call to a `libmoq` function whose signature has since changed is a compile error rather than something you find out about later. It compiles the Qt sources too, which the CMake build only does when `ENABLE_QT` and `ENABLE_FRONTEND_API` are on. `just check` runs it for you when a branch touches `cpp/obs/` or `rs/libmoq/`. @@ -78,6 +78,16 @@ It compiles the Qt sources too, which the CMake build only does when `ENABLE_QT` The filter reaches past `cpp/obs/` because this is the only place `libmoq.a` is linked from outside cargo, which needs the hand-maintained native-library lists in `rs/libmoq/native-libs/`. A dependency that starts pulling in a new native library leaves those stale, and every Rust gate stays green because cargo passes the flag itself. No list of paths catches all of those, so [`nightly.yml`](https://github.com/moq-dev/moq/blob/main/.github/workflows/nightly.yml) runs the same recipe diff-independently as the backstop. +### Which OBS version + +Three places name an OBS release, and `just obs check` compares all three: + +- `cpp/obs/buildspec.json` names the obs-deps bundle the macOS and Windows builds download, so this is what the released binaries link. +- `flake.nix`'s `libobs-headers` is what `just obs compile` type-checks against on every platform. It must equal `buildspec.json` exactly; both unpack the same OBS tag. +- nixpkgs' `obs-studio` is what `just obs ci` links on Linux. This one comes from `flake.lock` rather than from us, so only its `major.minor` has to match: a patch release carries no libobs API change, and the guard should fire on the nixpkgs bump that opens a real gap, not on every one. + +Bumping the first two together means new SHA-256 hashes for the OBS source archive (`.tar.gz` for macOS, `.zip` for Windows), the prebuilt obs-deps and Qt6 archives, and the nix `fetchzip` hash. The obs-deps version and hashes to use are the ones in the target OBS release's own `CMakePresets.json`, under the `dependencies` configure preset. + ### Tests `just obs test` compiles the plugin sources against stubbed `libobs`, `libmoq` and FFmpeg under ThreadSanitizer, and drives the callback orderings directly rather than waiting for them. There is one binary per source under test, because each test file defines its own stubs: diff --git a/flake.nix b/flake.nix index cf435f4b903..f28fd196a4f 100644 --- a/flake.nix +++ b/flake.nix @@ -336,14 +336,15 @@ # bundle. `just obs compile` type-checks the plugin against it. # # Keep `version` equal to buildspec.json's obs-studio version; `just obs - # check` fails when the two drift. + # check` fails when the two drift, and when either drifts a minor + # release from the nixpkgs obs-studio below. obs-headers = pkgs.stdenvNoCC.mkDerivation rec { pname = "libobs-headers"; - version = "31.1.1"; + version = "32.2.2"; src = pkgs.fetchzip { url = "https://github.com/obsproject/obs-studio/archive/refs/tags/${version}.tar.gz"; - hash = "sha256-ycfROxgm3wUVyC2d1r3vIr7yWb6ErYIoDZX8xZrc+Vk="; + hash = "sha256-miLe4MhiVhLlPvwzDjL31BwdcjVhgWvmcSzH9pgZV6U="; }; dontBuild = true; @@ -367,8 +368,8 @@ # Dependencies for the OBS plugin (`just obs build`, `just obs compile`, # `just obs check`). ffmpeg + cmake come from rustDeps. # - # Only the *build* is Linux-only: nixpkgs marks obs-studio broken on - # Darwin, so macOS and Windows link libobs/Qt6 from the OBS buildspec + # Only the *build* is Linux-only: nixpkgs' obs-studio lists no Darwin + # platform, so macOS and Windows link libobs/Qt6 from the OBS buildspec # bundle instead (see cpp/obs/buildspec.json and doc/bin/obs.md). # Type-checking needs headers rather than libraries, and those are # cross-platform -- obs-headers above, plus qt6.qtbase, which does build @@ -378,6 +379,13 @@ with pkgs; [ obs-headers + # libobs' public headers include as of OBS 32, + # which used to be vendored under libobs/util/ and so travelled with + # obs-headers. obs-deps ships it beside libobs for the macOS and + # Windows builds, and nixpkgs' obs-studio propagates it for the + # Linux one; listed here so the header-only path has it on Darwin + # too. Same 0.8.2 that obs-deps carries. + simde qt6.qtbase clang-tools gersemi @@ -387,6 +395,14 @@ ninja ]; + # The obs-studio `just obs ci` links against on Linux, which is a third + # version alongside buildspec.json and obs-headers and the only one this + # repo doesn't choose. Read the string on every platform, Darwin + # included, so `just obs check` compares it everywhere rather than only + # where the package builds. It moves when flake.lock does, so the guard + # fails on the nixpkgs bump that opens the gap. + obs-linked-version = pkgs.obs-studio.version; + # Apply our overlay to get the package definitions overlayPkgs = pkgs.extend self.overlays.default; in @@ -480,6 +496,11 @@ # on every platform so the plugin type-checks against the pinned OBS # release everywhere, rather than whatever the host happens to have. OBS_INCLUDE_DIR = "${obs-headers}/include/obs"; + + # What `just obs check` compares the two in-repo OBS pins against. + # Exported rather than read back out of nix, so the guard costs a + # variable lookup instead of a nested evaluation of this flake. + OBS_LINKED_VERSION = obs-linked-version; } // pkgs.lib.optionalAttrs (!pkgs.stdenv.hostPlatform.isDarwin) { ALSA_PLUGIN_DIR = "${alsaPlugins}/lib/alsa-lib"; diff --git a/justfile b/justfile index e2f8d3af908..df8ecce9f1c 100644 --- a/justfile +++ b/justfile @@ -247,10 +247,16 @@ _tools $FILES="": # here to prevent. scoped '^(go/|rs/moq-ffi/)' && tools+=(go uniffi-bindgen-go cargo rsync) scoped '^(dart/|rs/moq-ffi/)' && tools+=(cargo dart uniffi_bindgen_dart) - # cargo regenerates moq.h for the type-check; pkg-config locates Qt6 and - # ffmpeg. Every platform: the plugin type-checks against headers, and the - # dev shell ships those even on Darwin, where obs-studio can't build. - scoped '^(cpp/obs/|rs/libmoq/)' && tools+=(clang-format gersemi pkg-config cargo) + # Two obs recipes with two dispatch scopes, so two lines: over-requiring + # would fail a diff that never runs the recipe. `just obs compile` needs + # cargo to regenerate moq.h and pkg-config to locate Qt6 and ffmpeg. Every + # platform: the plugin type-checks against headers, and the dev shell ships + # those even on Darwin, where obs-studio can't build. + scoped '^(cpp/obs/|rs/libmoq/|flake\.nix$)' && tools+=(pkg-config cargo) + # `just obs check` lints with clang-format and gersemi, validates the CMake + # release configuration, and compares the three OBS pins, one of which moves + # on a flake.lock bump alone. + scoped '^(cpp/obs/|flake\.(nix|lock)$)' && tools+=(clang-format gersemi cmake) # Scopes overlap (rs/moq-ffi/ is in five of them), so the same tool can land # in the list twice and be reported missing twice. Splitting on whitespace is @@ -323,7 +329,10 @@ check $BASE="": fi # flake.nix is in scope because `just obs check` is what compares the OBS # version pinned there against buildspec.json, and either side can move. - if echo "$files" | grep -qE '^(cpp/obs/|flake\.nix$)'; then + # flake.lock too, because the third OBS the guard compares is nixpkgs' + # obs-studio, the one `just obs ci` links: it moves on a lock bump alone, + # and that bump is the change that opens the gap. + if echo "$files" | grep -qE '^(cpp/obs/|flake\.(nix|lock)$)'; then just obs check fi # Validates flake eval + dev shell build; it no longer compiles the diff --git a/quest/m0/2868-obs-the-plugin-targets-obs-31-1-1-while-linux-ci-links.md b/quest/m0/2868-obs-the-plugin-targets-obs-31-1-1-while-linux-ci-links.md deleted file mode 100644 index 8f6c3901696..00000000000 --- a/quest/m0/2868-obs-the-plugin-targets-obs-31-1-1-while-linux-ci-links.md +++ /dev/null @@ -1,30 +0,0 @@ -# [S] obs: the plugin targets OBS 31.1.1 while Linux CI links against 32 - -## Goal - -The libobs the released macOS and Windows plugin binaries link against is the -same major as the one the Linux compile gate checks, so a libobs API change -cannot pass CI and surface only in a release build on the platforms with no -gate. - -## Plan - -`cpp/obs/buildspec.json` pins obs-studio 31.1.1, which is what obs-deps hands -a macOS or Windows build. `flake.nix`'s `libobs-headers` pins the same, and -`just obs check` fails if the two drift. Neither is what Linux links: the -`obs.yml` gate uses nixpkgs' obs-studio, a major release ahead, and nothing -compares that third version to the other two. - -- Bump `buildspec.json` and `libobs-headers` together to the current stable, - 32.2.2 as of 2026-08-14: new hashes for the obs-studio, prebuilt obs-deps, - and Qt6 archives (macOS and Windows) plus the nix `fetchzip` hash. -- Check `libobs/obsconfig.h.in` and `frontend/api/obs-frontend-api.h` are - still where `libobs-headers` expects them. -- Extend the drift guard to the nixpkgs version `just obs ci` links against, - so the next gap is a failing check rather than a discovery. -- A real `just obs build` on macOS, since that is the platform the release - ships and PR CI never compiles. - -## Closes - -- [#2868](https://github.com/moq-dev/moq/issues/2868) - close this issue when the quest finishes diff --git a/quest/m0/README.md b/quest/m0/README.md index 90206321ce3..f7f7df0eebb 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -22,4 +22,3 @@ regression test per Root Cause First. - [#2833](/quest/m0/2833-moq-export-ts-a-rewound-timeline-stalls-the-si-table.md) - moq export ts: a rewound timeline stalls SI tables, PCR, and pacing until the media clock catches up - [#2799](/quest/m0/2799-moq-video-capture-negotiates-twice-so-a-window-resize.md) - moq-transcode: the ladder follows a source resolution change instead of keeping the one it started with - [Group charge](/quest/m0/group-charge.md) - charge real per-group cost so MOQ_CACHE_CAPACITY bounds real memory -- [#2868](/quest/m0/2868-obs-the-plugin-targets-obs-31-1-1-while-linux-ci-links.md) - obs: the plugin targets OBS 31.1.1 while Linux CI links against 32 From a8c6b033f0dc17e13660701679ef87b648790dbe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 16:00:03 -0700 Subject: [PATCH 04/29] fix(moq-net): answer for a name the broadcast never served (#3366) Co-authored-by: Claude Opus 5 Co-authored-by: GPT-5 --- doc/bin/gstreamer.md | 4 + js/net/src/broadcast.test.ts | 26 +-- js/net/src/broadcast.ts | 7 +- js/net/src/internal.ts | 2 + js/net/src/track.ts | 6 + ...-nobody-answers-keeps-the-session-alive.md | 47 ----- quest/m0/README.md | 1 - rs/moq-gst/src/source/imp.rs | 69 +++++- rs/moq-net/src/model/broadcast.rs | 196 +++++++++++++++++- rs/moq-net/src/model/track.rs | 60 +++++- 10 files changed, 334 insertions(+), 84 deletions(-) delete mode 100644 quest/m0/3139-moqsrc-a-rendition-nobody-answers-keeps-the-session-alive.md diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index bfb20fb7eb9..50474283771 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -345,6 +345,10 @@ no matter which rendition the catalog announces first. Pads appear once their re shows up in the catalog (sometimes-pads), so an application links them from a `pad-added` handler. +Each pad sends EOS when its rendition ends. A rendition the catalog named but nothing +ever served is given up on when the publisher ends the broadcast, so the element finishes +once the renditions that did arrive have drained rather than waiting to be stopped. + ## Debugging Enable GStreamer debug output: diff --git a/js/net/src/broadcast.test.ts b/js/net/src/broadcast.test.ts index e16a6c15656..a1476fc2ed3 100644 --- a/js/net/src/broadcast.test.ts +++ b/js/net/src/broadcast.test.ts @@ -105,25 +105,19 @@ test("concurrent dynamic producers share a sequence namespace", async () => { broadcast.close(); }); -test("closing a broadcast preserves dequeued request sequences", async () => { +test("closing a broadcast rejects a dequeued request", async () => { const broadcast = new BroadcastProducer(); - const firstSubscriber = broadcast.subscribe("media"); - const firstRequest = await broadcast.requested(); - if (!firstRequest) throw new Error("expected first request"); - const firstProducer = firstRequest.accept(); - expect(firstProducer.appendGroup().sequence).toBe(0); - - const secondSubscriber = broadcast.subscribe("media"); - const secondRequest = await broadcast.requested(); - if (!secondRequest) throw new Error("expected second request"); + const subscriber = broadcast.subscribe("media"); + const request = await broadcast.requested(); + if (!request) throw new Error("expected request"); broadcast.close(); - const secondProducer = secondRequest.accept(); - expect(secondProducer.appendGroup().sequence).toBe(1); + await expect(subscriber.info()).rejects.toThrow("track closed before info was known"); - firstSubscriber.close(); - secondSubscriber.close(); - firstProducer.close(); - secondProducer.close(); + const producer = request.accept(); + expect(() => producer.appendGroup()).toThrow("track is closed"); + await expect(subscriber.info()).rejects.toThrow("track closed before info was known"); + subscriber.close(); + producer.close(); }); test("a request exposes the aggregate subscription options", async () => { diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 01b22f7b434..658de67f05c 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -11,6 +11,7 @@ import * as track from "./track.ts"; /** Reactive backing state shared by broadcast producers and consumers. */ class BroadcastState { requested = new Signal([]); + pending = new Set(); closed = new Once(); tracks = new Map(); sequences = new Map(); @@ -34,8 +35,8 @@ function dequeueRequest(state: BroadcastState): track.Request | undefined { function closeState(state: BroadcastState, abort?: Error) { if (state.closed.peek() !== undefined) return; state.closed.set(abort ?? null); + for (const request of state.pending) request.reject(abort); state.requested.mutate((requests) => { - for (const request of requests) request.reject(abort); requests.length = 0; }); } @@ -76,7 +77,7 @@ function subscribe( } state.requested.mutate((requested) => { - requested.push(hooks.makeRequest({ name, producer, sequences: state.sequences })); + requested.push(hooks.makeRequest({ name, producer, sequences: state.sequences, pending: state.pending })); }); return subscriber; @@ -94,7 +95,7 @@ async function resolveTrackInfo(state: BroadcastState, name: string): Promise { - requested.push(hooks.makeRequest({ name, producer, sequences: state.sequences })); + requested.push(hooks.makeRequest({ name, producer, sequences: state.sequences, pending: state.pending })); }); try { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index b83d936260f..d5a451ea898 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -23,6 +23,8 @@ export interface TrackRequestOptions { producer: Producer; /** Sequence namespaces shared by the broadcast generation. */ sequences: TrackSequences; + /** Requests not yet accepted or rejected by the publisher. */ + pending: Set; } /** Hooks assigned in static blocks by the owning class. */ diff --git a/js/net/src/track.ts b/js/net/src/track.ts index a30da229ad8..d7c82317575 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -154,11 +154,14 @@ export class Request { #producer: Producer; #sequences: TrackSequences; + #pending: Set; private constructor(options: TrackRequestOptions) { this.name = options.name; this.#producer = options.producer; this.#sequences = options.sequences; + this.#pending = options.pending; + this.#pending.add(this); } static { @@ -177,12 +180,14 @@ export class Request { /** Accept the request, committing the track's immutable {@link Info}. */ accept(info: Partial = {}): Producer { + this.#pending.delete(this); bindProducer(this.name, this.#producer, this.#sequences); return this.#producer.accept(info); } /** Reject the request, closing the track optionally with an error. */ reject(err?: Error): void { + this.#pending.delete(this); this.#producer.close(err); } } @@ -376,6 +381,7 @@ export class Producer { /** Commit the immutable publisher properties, resolving {@link info}. Returns `this`. */ accept(info: Partial = {}): this { + if (this.#state.closed.peek() !== undefined) return this; const resolved = infoDefaults(info); this.#state.info.set(resolved); // Propagate to any sink handed out before accept (the on-demand path). diff --git a/quest/m0/3139-moqsrc-a-rendition-nobody-answers-keeps-the-session-alive.md b/quest/m0/3139-moqsrc-a-rendition-nobody-answers-keeps-the-session-alive.md deleted file mode 100644 index 5fba02f7866..00000000000 --- a/quest/m0/3139-moqsrc-a-rendition-nobody-answers-keeps-the-session-alive.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] moqsrc: a rendition nobody answers keeps the session alive after the catalog closes - -## Goal - -A catalog that names a track its publisher never serves does not park a -subscriber: once the catalog closes, the subscription to that track resolves -with an error, `moqsrc` drops that one pump, and the session delivers its -terminal EOS when the served renditions drain. - -## Plan - -`moqsrc`'s pumps (`rs/moq-gst/src/source/imp.rs`) each await their own -subscription, and `follow_catalog` deliberately keeps them alive across -catalog close, because "has not taken a pad yet" also describes a pump spawned -microseconds ago; cancelling on close dropped whole broadcasts, which -`a_closing_catalog_keeps_the_renditions_it_named` guards against. So a pump for -a rendition nobody ever answers keeps the session open until the broadcast -ends or the element stops. - -moq-net already fails fast for one case: `broadcast::Consumer::track` returns -`NotFound` when there is no producer and no `Dynamic`. The parking case is a -request that stays queued forever: a `track::Request` from `reserve_track` -that is never accepted, or a producer whose info never arrives, and over the -network every broadcast is a `Dynamic`, so that is the common shape. -A timeout would stand in for information the publisher has, so the fix is a -promise on the publisher side: closing the catalog means the announced set is -final. - -- moq-net keeps its generic semantics: a dropped `track::Request` or a - producer closed without a reason resolves pending subscribes with - `Error::Dropped`, as `rs/moq-net/src/model/track.rs` documents, and a - `request.reject(reason)` carries that reason. Verify and pin both; do not - fold `NotFound` into the drop path, since a handler lost to a publisher or - transport failure is not an absent track. -- hang publishers (`moqsink` reserves per pad, `moq-cli` and the FFI create - tracks by name): when the catalog finishes, `reject(Error::NotFound)` every - reserved or info-less track the catalog named, so their subscribers resolve - and the catalog meaning stays out of moq-net. -- moqsrc: a pump whose subscribe resolves with an error ends that pump alone, - and the loop exits once the catalog is closed and the served pumps drain. -- Tests: the existing test keeps its media; a new one names a reserved track, - finishes the catalog, and asserts EOS within the served renditions' drain - rather than at element stop. - -## Closes - -- [#3139](https://github.com/moq-dev/moq/issues/3139) - close this issue when the quest finishes diff --git a/quest/m0/README.md b/quest/m0/README.md index f7f7df0eebb..54cf3a74f66 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -17,7 +17,6 @@ regression test per Root Cause First. - [#2405](/quest/m0/2405-js-net-connect-logs-on-every-connection-at-the-wrong.md) - js/net: connect logs print the JWT in the relay URL - [Adapter namespace map](/quest/m0/rs-adapter-namespace-map.md) - moq-net: a duplicate PUBLISH_NAMESPACE on draft-14/15 strands the first request, and the map never shrinks -- [#3139](/quest/m0/3139-moqsrc-a-rendition-nobody-answers-keeps-the-session-alive.md) - moqsrc: a rendition nobody answers keeps the session alive after the catalog closes - [#3080](/quest/m0/3080-fix-watch-audio-ring-truncate-can-race-the-worklet-reader.md) - watch: an audio ring truncate can race the worklet reader for one quantum - [#2833](/quest/m0/2833-moq-export-ts-a-rewound-timeline-stalls-the-si-table.md) - moq export ts: a rewound timeline stalls SI tables, PCR, and pacing until the media clock catches up - [#2799](/quest/m0/2799-moq-video-capture-negotiates-twice-so-a-window-resize.md) - moq-transcode: the ladder follows a source resolution change instead of keeping the one it started with diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 801a44e35db..ed4a9b64106 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -422,9 +422,11 @@ async fn follow_catalog( // That includes a pump still resolving its subscription, which is // indistinguishable here from one that has simply not been polled yet: // a final snapshot naming a track the publisher does serve arrives this - // way, and cancelling on "not live yet" would drop it. A rendition nobody - // ever answers therefore keeps the session alive until the broadcast ends - // or the element is stopped. + // way, and cancelling on "not live yet" would drop it. Whether a + // rendition will ever be served is the publisher's to answer, and + // ending the broadcast is where it does: every name it never served + // resolves with an error, which fails that pump's subscribe and ends + // it here. None => catalog_closed = true, } } @@ -642,9 +644,10 @@ impl Pump { state, mut cancel, } = self; - // Resolves once the publisher answers with the track info. A catalog can name a track its - // publisher never serves, so this can wait forever; racing `cancel` keeps such a pump - // reapable, and holding the wait here rather than in `reconcile` keeps it off every other + // Resolves once the publisher answers, with the track info or with an error (which is + // what ending a broadcast produces for a name nobody served). A publisher that answers + // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and + // holding the wait here rather than in `reconcile` keeps it off every other // rendition. let subscriber = tokio::select! { _ = cancel.changed() => return, @@ -1165,6 +1168,60 @@ mod session_tests { super::RUNTIME.block_on(session).unwrap().unwrap(); } + /// The flip side of the test above: a rendition the publisher reserved and never served is + /// answerable once the publisher ends the broadcast, which resolves every name it never + /// filled. That pump must end on its own so the session terminates on the media it was + /// serving, rather than holding the connection open until the element is stopped. + #[test] + fn a_rendition_nobody_served_ends_with_the_broadcast() { + let _pad_ids = pad_ids(); + let element = element(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap(); + + // One rendition that streams, and one reserved by name that nobody ever accepts. + let mut video = broadcast.create_track("video", hang::container::track_info()).unwrap(); + let _reserved = broadcast.reserve_track("audio").unwrap(); + { + let mut guard = catalog.lock(); + guard.video.renditions = BTreeMap::from([("video".to_string(), video_rendition())]); + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + + let (shutdown, mut shutdown_rx) = watch::channel(false); + let consumer = broadcast.consume(); + let weak = element.downgrade(); + let session = super::RUNTIME.spawn(async move { follow_catalog(consumer, weak, &mut shutdown_rx).await }); + + // Watch for the EOS on the served rendition, before anything can end its track. + let pad = await_pad(&element, "video_"); + let eos = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let seen = eos.clone(); + pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, move |_, info| { + if let Some(gst::PadProbeData::Event(event)) = &info.data + && event.type_() == gst::EventType::Eos + { + seen.store(true, Ordering::Relaxed); + } + gst::PadProbeReturn::Ok + }); + + // End the served rendition and the broadcast, which answers for the reserved one. + catalog.finish().unwrap(); + video.finish().unwrap(); + broadcast.finish(); + + // No shutdown is sent: the session has to end on the media draining alone. + super::RUNTIME + .block_on(async { tokio::time::timeout(Duration::from_secs(10), session).await }) + .expect("the session outlived the renditions it was serving") + .unwrap() + .unwrap(); + assert!(eos.load(Ordering::Relaxed), "the served rendition never emitted EOS"); + drop(shutdown); + } + /// Pipelines link `moqsrc`'s pads by name, so the first video rendition that actually /// arrives has to be `video_0`. A rendition announced but never served must not claim that /// name and leave the real one on `video_1`, where `s.video_0 ! ...` never links. diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index f98992a9ebf..836cd17b9da 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -275,6 +275,22 @@ impl BroadcastState { } } + /// Resolve every name the broadcast never filled, so subscribers waiting on a + /// [`track::Info`] that can no longer arrive fail with `err` instead of parking. + /// + /// Covers a reservation nobody accepted, a request handed to a [`Dynamic`] that + /// never answered it, and one still queued for a handler. A track that carries + /// its info has a publisher and is left alone: an end there is that publisher's + /// call, and its cache stays readable. + fn reject_unserved(&mut self, err: Error) { + for request in self.requests.drain_queued() { + request.reject(err.clone()); + } + for track in self.tracks.iter() { + track.reject(err.clone()); + } + } + /// Live demand: a subscribed spliced track (route-fed broadcast), or a /// pending request / consumed track (ordinary broadcast). See [`Demand`]. fn is_used(&self) -> bool { @@ -441,6 +457,10 @@ impl Producer { /// the producer can't pick the track's properties (e.g. timescale) until it has /// inspected the media, the same shape as a consumer-driven /// [`Dynamic::requested_track`]. + /// + /// Subscribers wait on the name until it is accepted, so a reservation the producer + /// ends up never filling has to be dropped or rejected. Ending the broadcast + /// ([`Self::finish`] or [`Self::abort`]) resolves whatever is left. pub fn reserve_track(&mut self, name: impl Into>) -> Result { let request = track::Request::new(self.info.clone(), name).with_stats(self.stats.clone()); self.state.lock().insert_track(request.weak())?; @@ -577,6 +597,10 @@ impl Producer { /// new tracks are served, whether or not other producer clones are still alive. /// Existing tracks stay readable so consumers can drain what they already have. /// + /// A name that was reserved or requested but never served resolves with + /// [`Error::NotFound`]: nothing can fill it now, so its subscribers fail rather + /// than waiting on a [`track::Info`] that is never coming. + /// /// Borrows rather than consumes, matching [`track::Producer::finish`]. Finishing /// declares the end, so it must not depend on the caller also surrendering the /// handle. @@ -585,6 +609,10 @@ impl Producer { let mut state = self.state.lock(); state.closing = true; state.finished = true; + // A name that was reserved or requested but never served can't arrive now, + // and `Consumer::track` already answers `NotFound` for one asked about after + // this point. Say the same to whoever asked earlier. + state.reject_unserved(Error::NotFound); } // Ending the broadcast is what consumers wait on, so signal it here rather // than leaving it to the last handle drop. @@ -596,7 +624,8 @@ impl Producer { /// Like [`finish`](Self::finish) the end is immediate, whether or not other /// producer clones are still alive, and existing tracks stay readable so /// consumers can drain what they already have (an abort does not cascade into - /// the tracks). Unlike a finish, consumers observe `err` from + /// the tracks), while a name nothing ever served resolves with `err` the same + /// way [`finish`](Self::finish) resolves it. Unlike a finish, consumers observe `err` from /// [`Consumer::closed`], and an origin treats the source as ungracefully lost, /// so the path may linger for a replacement (see /// [`origin::Info::linger`](crate::origin::Info::linger)). @@ -610,7 +639,10 @@ impl Producer { return Err(Error::Closed); } state.closing = true; - state.abort = Some(err); + state.abort = Some(err.clone()); + // Same as a finish: an unserved name is answerable now, with the reason the + // broadcast ended. Published tracks keep their cache (no cascade). + state.reject_unserved(err); } let _ = self.alive.token.close(); Ok(()) @@ -1653,4 +1685,164 @@ mod test { // instead of failing with NotFound. let _fut = subscribe_pending!(consumer, "track1"); } + + /// A reserved name nobody accepts is the parking case a publisher has to be able to + /// end. Ending the broadcast is where it does: `Consumer::track` already answers + /// `NotFound` for a name asked about after this point, so whoever asked earlier gets + /// the same answer instead of waiting on info that can never arrive. + #[tokio::test] + async fn finish_resolves_a_reserved_name() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let _request = producer.reserve_track("track1").unwrap(); + let pending = subscribe_pending!(consumer, "track1"); + + producer.finish(); + assert!(matches!(pending.await, Err(Error::NotFound))); + } + + /// An abort says why the broadcast ended, and an unserved name resolves with that + /// reason rather than a generic failure. + #[tokio::test] + async fn abort_resolves_a_reserved_name_with_its_reason() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let request = producer.reserve_track("track1").unwrap(); + let pending = subscribe_pending!(consumer, "track1"); + + producer.abort(Error::Cancel).unwrap(); + assert!(matches!(pending.await, Err(Error::Cancel))); + + let track = request.accept(None); + let mut subscriber = track.subscribe(None); + assert!(matches!(subscriber.recv_group().await, Err(Error::Cancel))); + } + + /// A request still queued for a handler is the same parking case reached from the + /// consumer side, so it ends the same way. + #[tokio::test] + async fn finish_resolves_a_queued_request() { + let mut producer = Info::new().produce(); + let dynamic = producer.dynamic(); + let consumer = dynamic.consume(); + + let pending = subscribe_pending!(consumer, "track1"); + + producer.finish(); + assert!(matches!(pending.await, Err(Error::NotFound))); + drop(dynamic); + } + + /// A request a handler already took parks the same way if the handler never answers + /// it, so the sweep has to reach that one too. + #[tokio::test] + async fn finish_resolves_a_request_a_handler_never_answered() { + let mut producer = Info::new().produce(); + let mut dynamic = producer.dynamic(); + let consumer = dynamic.consume(); + + let pending = subscribe_pending!(consumer, "track1"); + let _request = dynamic.requested_track().await.unwrap(); + + producer.finish(); + assert!(matches!(pending.await, Err(Error::NotFound))); + drop(dynamic); + } + + /// A reverse fetch can install the track metadata before the live request is + /// accepted, but it does not create a live publisher. Finishing the broadcast + /// must still reject that name so an arrival-order subscriber does not park on + /// backfill that is deliberately absent from its queue. + #[tokio::test] + async fn finish_resolves_an_unaccepted_track_with_fetched_info() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let request = producer.reserve_track("track1").unwrap(); + let dynamic = request.dynamic(); + let track = consumer.track("track1").unwrap(); + let pending_fetch = track.fetch_group(0, None); + let fetch = dynamic.requested_group().await.unwrap(); + let mut group = fetch.accept(None).unwrap(); + group.finish().unwrap(); + pending_fetch.await.unwrap(); + + let mut subscriber = track.subscribe(None).await.unwrap(); + producer.finish(); + assert!(matches!(subscriber.recv_group().await, Err(Error::NotFound))); + + let mut stale = request.accept(None); + assert!(stale.append_group().is_err()); + } + + /// Ending the broadcast doesn't cascade into a track someone is publishing: it keeps + /// its cache and its publisher decides when it ends. + #[tokio::test] + async fn finish_spares_a_served_track() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let mut track = producer.create_track("track1", None).unwrap(); + let mut subscriber = consumer.track("track1").unwrap().subscribe(None).await.unwrap(); + + producer.finish(); + + track.append_group().unwrap(); + subscriber.assert_group(); + track.finish().unwrap(); + } + + /// The publisher may still be holding the `track::Request` for a name the broadcast + /// just gave up on. Accepting it afterwards must not resurrect the track, or a + /// subscriber that was told `NotFound` could be contradicted by a later one. + #[tokio::test] + async fn finish_leaves_a_stale_reservation_inert() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let request = producer.reserve_track("track1").unwrap(); + let pending = subscribe_pending!(consumer, "track1"); + + producer.finish(); + assert!(matches!(pending.await, Err(Error::NotFound))); + + let mut track = request.accept(None); + assert!(track.append_group().is_err()); + let mut subscriber = track.subscribe(None); + assert!(matches!(subscriber.recv_group().await, Err(Error::NotFound))); + assert!(consumer.track("track1").is_err()); + } + + /// Dropping a `track::Request` is not a verdict about the name, so it resolves as + /// `Dropped` (a handler lost to a crashed publisher or a dead transport), never as + /// `NotFound`. Only an explicit rejection may claim the track is absent. + #[tokio::test] + async fn dropping_a_reserved_request_resolves_dropped() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let request = producer.reserve_track("track1").unwrap(); + let pending = subscribe_pending!(consumer, "track1"); + + drop(request); + assert!(matches!(pending.await, Err(Error::Dropped))); + producer.finish(); + } + + /// `track::Request::reject` carries its reason the same way, which is what lets a + /// subscriber tell "no such track" from "the publisher went away". + #[tokio::test] + async fn rejecting_a_reserved_request_carries_the_reason() { + let mut producer = Info::new().produce(); + let consumer = producer.consume(); + + let request = producer.reserve_track("track1").unwrap(); + let pending = subscribe_pending!(consumer, "track1"); + + request.reject(Error::NotFound); + assert!(matches!(pending.await, Err(Error::NotFound))); + producer.finish(); + } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index a3ef056687a..87e0ba89151 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -133,6 +133,9 @@ pub(crate) struct TrackState { // The publisher's properties, once known; always Some for Subscriber/Producer. // Copied by value into each group it creates. info: Option, + // Whether a live Producer was minted. A reverse fetch may install `info` + // before acceptance, so the two states are deliberately separate. + published: bool, // The broadcast this track belongs to. Supplies the cache pool its groups charge // into and the `cache_duration` ceiling clamping `Info::latency_max`. @@ -263,9 +266,23 @@ struct FetchOutcome { } impl TrackState { + fn normalize_info(broadcast: &broadcast::Info, mut info: Info) -> Info { + info.latency_max = info.latency_max.min(broadcast.origin.cache_duration); + info + } + + fn accept(&mut self, info: Info) { + self.published = true; + self.install(info); + } + fn poll_info(&self) -> Poll> { if let Some(info) = &self.info { Poll::Ready(Ok(info.clone())) + } else if let Some(err) = &self.abort { + // Aborted before anyone served it, so the info can never arrive: fail the + // waiting subscribes instead of parking them on a track nobody will fill. + Poll::Ready(Err(err.clone())) } else { Poll::Pending } @@ -558,8 +575,8 @@ impl TrackState { /// group is never retained longer than the origin allows. Every path that binds an /// info to a track funnels through here, covering local publishers and relayed /// (lite / IETF) tracks alike. - fn install(&mut self, mut info: Info) { - info.latency_max = info.latency_max.min(self.broadcast.origin.cache_duration); + fn install(&mut self, info: Info) { + let info = Self::normalize_info(&self.broadcast, info); self.info = Some(info); } @@ -841,6 +858,7 @@ impl TrackState { #[derive(Clone)] pub struct Producer { name: Arc, + info: Info, // The parent broadcast's info, inherited from [`broadcast::Producer::create_track`]. // Top link of the ownership chain; carried for identity and future inheritance. broadcast: Arc, @@ -868,16 +886,14 @@ impl Producer { info: impl Into>, ) -> Self { let name = name.into(); + let info = TrackState::normalize_info(&broadcast, info.into().unwrap_or_default()); let state = TrackState::spawn(broadcast.clone()); - state - .write() - .ok() - .expect("a new track is open") - .install(info.into().unwrap_or_default()); + state.write().ok().expect("a new track is open").accept(info.clone()); let alive = Alive::new(name.clone(), state.clone()); alive.publish(None); Self { name, + info, state, broadcast, prev_subscription: None, @@ -1171,7 +1187,7 @@ impl Producer { // requiring a live producer state. If the track already ended, the returned // subscriber surfaces the close/abort on its first read; the preferences are // simply never registered (nothing aggregates them anymore). - let info = self.state.read().info.clone().expect("producer always has info"); + let info = self.info.clone(); let subscription = kio::Producer::new(preferences); register_subscription(self.state.read(), &subscription); @@ -1566,6 +1582,29 @@ impl TrackWeak { &self.name } + /// Reject a track nothing ever served, resolving its pending subscribes with `err`. + /// + /// A track whose [`Producer`] was minted is left alone and this returns false; + /// so is one that already carries an abort reason. Fetched backfill can install + /// [`Info`] before acceptance, so metadata alone does not prove a publisher exists. + /// + /// Closes the state like [`Producer::abort`], so a [`Request`] still held by the + /// publisher can't `accept` its way back to life afterwards. + pub(crate) fn reject(&self, err: Error) -> bool { + let Some(producer) = self.state.produce() else { + return false; + }; + let Ok(mut state) = producer.write() else { + return false; + }; + if state.published || state.abort.is_some() { + return false; + } + state.abort = Some(err); + state.close(); + true + } + /// Whether anyone is consuming the track right now. A closed track doesn't /// count even if consumers linger to drain its cache: no new work is owed. pub(crate) fn is_used(&self) -> bool { @@ -2685,16 +2724,18 @@ impl Request { /// [`Producer`] is inert: writes fail with the abort error, as if it had been /// aborted immediately after accepting. pub fn accept(self, info: impl Into>) -> Producer { + let info = TrackState::normalize_info(&self.broadcast, info.into().unwrap_or_default()); // A closed state means the track was aborted under us. Mirror `reject` and // tolerate it: the Producer we hand back simply can't write. if let Ok(mut state) = self.state.write() { - state.install(info.into().unwrap_or_default()); + state.accept(info.clone()); } // Accepting the request creates the track producer: count it as one ingress // subscription (closed when the last handle drops). No-op when untagged. self.alive.publish(Some(&self.stats)); Producer { name: self.name, + info, broadcast: self.broadcast, state: self.state, prev_subscription: None, @@ -3424,6 +3465,7 @@ mod test { Duration::from_secs(1), ); assert_eq!(capped.state.read().latency_bound(), Some(Duration::from_secs(1))); + assert_eq!(capped.subscribe(None).info().latency_max, Duration::from_secs(1)); let under = track_producer_capped( "test", From 7039d49840cdbb763af4f44dedabf244314b53f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 16:02:08 -0700 Subject: [PATCH 05/29] chore(claude): add the spawn-quest skill (#3403) Co-authored-by: Claude Opus 5 --- .claude/skills/spawn-quest/SKILL.md | 68 +++++++++++++++++++ .claude/skills/spawn-quest/agents/openai.yaml | 4 ++ 2 files changed, 72 insertions(+) create mode 100644 .claude/skills/spawn-quest/SKILL.md create mode 100644 .claude/skills/spawn-quest/agents/openai.yaml diff --git a/.claude/skills/spawn-quest/SKILL.md b/.claude/skills/spawn-quest/SKILL.md new file mode 100644 index 00000000000..89da4bcf589 --- /dev/null +++ b/.claude/skills/spawn-quest/SKILL.md @@ -0,0 +1,68 @@ +--- +name: spawn-quest +description: Triage every quest in a scope interactively, then spawn background agents to open a PR for each one worth starting now. Use when the user invokes /spawn-quest, optionally with a scope such as /spawn-quest m0, or asks whether a milestone's quests should be worked on now. +--- + +Before you begin, read `quest/AGENTS.md` completely. + +Resolve the first argument to a questline directory under `quest/`, defaulting +to `quest/` itself: `m0` means `quest/m0`. + +Read every quest in the scope, not just the summaries in each `README.md`. Drop +the ones a recent branch or open PR already claims, and say when a branch is +stale (old, no open PR) so it can be reused or deleted. + +Put every quest to the user in priority order - the depth-first walk of the +scope's `Quests` lists - with a recommendation and the one fact behind it. +Batch the questions; do not spend a round per quest. The choices are **start**, +**plan** (use `$plan-quest` when a material decision is unmade), **move**, +**delete**, and **leave**. + +Offer **start** only for a ready quest: `quest/AGENTS.md` executes one only when +it has no `Required` section. For work an agent cannot do - a decision that is +a conversation, a credential only the user can mint, or verification this +machine cannot run, such as a Linux-only build or a benchmark whose exit +criteria is a measured before/after - recommend **leave** and name what would +unblock it. + +## Spawning + +`git fetch origin` once, then settle a base branch **per quest** and pass it +to that quest's agent. Apply the Branch Targeting rules in `CLAUDE.md`: `dev` +only for a semver break in a published API, `main` for everything else. One +scope mixes both, so a single base for the whole wave puts some PR on the +wrong branch. Never let an agent derive its own base: an agent branching from +wherever it stands puts this skill's own tree edits in its PR. + +Spawn one background agent per **start**, each with `isolation: "worktree"`, in +waves of two or three unless the machine is idle - a saturated box makes agents +ship unverified. Never edit a running agent's worktree; message it instead. + +Give each agent the base branch, the quest path, and nothing it could read for +itself. Instruct it to: + +- Read `quest/AGENTS.md`, the quest, `CONTRIBUTING.md`, and the guides for the + areas it touches. +- Cut the quest branch (the quest path without `.md`) from that base and claim + it with an empty placeholder commit whose message contains a freshly + generated UUID, pushed immediately with `git push origin HEAD` after + pointing the upstream at the base. The UUID is what makes the claim a race: + two agents claiming the same quest from the same base in the same second + otherwise produce the same commit object, and the loser's push succeeds as + already-up-to-date instead of being rejected. A rejected push lost the race: + stop and report. Never force the claim push; `--force-with-lease` pinned to + its own claim is fine after a rebase. +- Implement, run `just fix`, `just check`, and `just test` through + `nix develop --command`, exercise the changed surface for real, walk the + Cross-Package Sync table, and open a PR per `CONTRIBUTING.md`, deleting the + quest and every reference to it in that same PR. +- Leave package versions and lockfile version metadata alone. +- Name in the PR any check that did not run, rather than claiming it or leaving + it out. +- Never deploy, apply, or mutate anything outside the repository. +- On a blocker: write what it learned into the quest so the next attempt starts + ahead, push that, then stop and report rather than improvising past it. + +**The scope ends at the open PR.** Neither you nor the agents merge one, and +nobody waits on CI. Report each PR as it lands, raise blockers for the user to +decide, and hand back what is still open. diff --git a/.claude/skills/spawn-quest/agents/openai.yaml b/.claude/skills/spawn-quest/agents/openai.yaml new file mode 100644 index 00000000000..0d7de326bde --- /dev/null +++ b/.claude/skills/spawn-quest/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Spawn Quest" + short_description: "Triage a milestone's quests and spawn agents for the ready ones" + default_prompt: "Use $spawn-quest to triage a scope's quests and spawn agents for the ones worth starting now." From b132fb393d96acd093c197f7c06cb6e1c65fa6e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 16:30:33 -0700 Subject: [PATCH 06/29] docs(quest): require hop-path validation before banning Hop ID 0 (#3404) Co-authored-by: Claude Opus 5 (1M context) --- ...outbound-hop-path-construction-is-never.md | 76 +++++++++++++++++++ ...60-moq-net-ban-hop-id-0-from-hop-chains.md | 7 +- quest/m1/README.md | 1 + 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md diff --git a/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md b/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md new file mode 100644 index 00000000000..2a9606552b1 --- /dev/null +++ b/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md @@ -0,0 +1,76 @@ +# [M] moq-net: outbound HOP_PATH construction is never validated + +## Goal + +A hop chain is validated where it is emitted, not only where it is parsed, so +moq-net cannot construct an advertisement that a conforming receiver must +close the session over. + +Every outbound path enforces the rule once: the entries exactly fill `Length`, +the list is non-empty, and no non-zero Hop ID appears twice. Publisher route +selection skips a route that cannot be legally advertised instead of emitting +it, and the moq-lite ingress cannot introduce a duplicate that later reaches an +IETF cluster peer. + +Boundaries: a route that is invalid to advertise may still be fine to serve +locally, so it is dropped from advertisement rather than from the origin. The +per-path ingress guards added by [#3042](https://github.com/moq-dev/moq/issues/3042) +stay working; this replaces the need for the next route source to remember the +rule again. + +## Plan + +Use the public issue's scope and implementation notes below as the starting +plan. Reconcile paths and assumptions with the current tree before +implementation. + +### Issue context + +`HopPath::validate` runs only on decode, so nothing stops us building and +sending an outbound HOP_PATH that a conforming receiver must reject. + +`ietf::cluster::HopPath::validate` rejects an empty list and a repeated +non-zero Hop ID, but it is only called from `param_decode`. `Advert::forward` +appends our own Hop ID and wraps the result without revalidating, and the +encode path does not check either. A `broadcast::Route` whose `hops` already +carry a duplicate therefore goes out as-is. + +That matters because `draft-lcurley-moq-cluster` requires the receiver to treat +it as fatal: "A receiver MUST close the session with a PROTOCOL_VIOLATION if +the entries do not exactly fill `Length`, if the list is empty, or if a +non-zero Hop ID appears twice." So a malformed chain we construct does not +degrade our own routing, it closes the downstream session. + +Routes reach the model from more than one place, and only the IETF decode path +is checked: + +- The moq-lite ingress builds hop chains itself. `OriginList` caps length but + does not reject duplicates, so a lite chain may legally carry one and, once + shared through the origin, be forwarded to an IETF cluster peer. +- `broadcast::Route` is public with a public `hops` field, so any in-process + producer can attach one. + +[#3042](https://github.com/moq-dev/moq/issues/3042) closed the two reachable +cases it introduced, both at ingress: `ietf::Subscriber::route` discards a chain +already carrying the identity assigned to the peer, and `lite::Subscriber`'s two +responder-append sites do the same. Those are per-path guards. The general gap +is that validity is enforced where a chain is parsed rather than where one is +emitted. + +### Design notes + +Validate in the outbound constructor (`Advert::forward`, or the encode path) and +have publisher route selection skip a route that cannot be legally advertised. +Making the invalid state unrepresentable is preferable to a check callers must +remember: a constructor that can only produce a valid chain means the next route +source gets the rule for free. + +Found by Codex during adversarial review of #3042. + +## Closes + +- [#3049](https://github.com/moq-dev/moq/issues/3049) - close this issue when the quest finishes + +## Related + +- [#3060](/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md) - bans Hop ID 0 from chains outright, and builds on this diff --git a/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md b/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md index 7b2404c1100..6d0358df51c 100644 --- a/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md +++ b/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md @@ -52,7 +52,12 @@ The section's honest summary today, `Declaring 0 therefore trades loop detection Every `== Origin::UNKNOWN` / `!= Origin::UNKNOWN` test that exists to ask "does this chain entry name anybody" is gone, across `lite/subscriber.rs`, `ietf/{subscriber,publisher,cluster}.rs`, and `model/{origin,broadcast}.rs`. The marker survives only where it means "this field is absent". Any survivor in the first category means 0 is still special somewhere and the change is incomplete. -Targets `dev`: it changes published `moq-net` API alongside moq-dev/moq#3049, and lands on top of it. +Targets `dev`: it changes published `moq-net` API alongside +[#3049](/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md), and lands on top of it. + +## Required + +- [#3049](/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md) - validates hop chains where they are emitted; this quest lands on top of it ## Closes diff --git a/quest/m1/README.md b/quest/m1/README.md index 0e51adba618..e3dd9ab333d 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -64,5 +64,6 @@ with the current dev tree before starting. - [#3208](/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md) - Make 2.5 ms Opus frame durations work across bindings - [#2152](/quest/m1/2152-libmoq-c-abi-catch-up-with-the-moq-ffi-surface.md) - libmoq: C ABI catch-up with the moq-ffi surface - [#2933](/quest/m1/2933-moq-ffi-moqroute-round-trip-erases-a-routes-cold-cost.md) - moq-ffi: MoqRoute round-trip erases a route's cold cost +- [#3049](/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md) - moq-net: validate a hop chain where it is emitted, not only where it is parsed - [#3060](/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md) - moq-net: ban Hop ID 0 from hop chains - [#2248](/quest/m1/2248-moq-mux-rebase-fmp4-export-timestamps-for-late-subscribers.md) - moq-mux: rebase fMP4 export timestamps for late subscribers From 3827f0b577f9db989531162bde6e109e8772466e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 16:30:45 -0700 Subject: [PATCH 07/29] docs(quest): plan the HLS identity, peer reconfigure, client goaway, and ACME quests (#3402) Co-authored-by: Claude Fable 5.1 --- ...rs-bypass-auth-api-mode-so-proxy-grants.md | 18 ++++- quest/m2/709-automatic-letsencrypt-support.md | 80 ++++++++++++++++--- quest/m2/README.md | 2 +- quest/m2/drain/README.md | 18 +++-- quest/m2/drain/client-goaway.md | 75 ++++++++++++++++- quest/m2/plan-hls-identity.md | 49 +++++++++--- quest/m2/pop-skipping/README.md | 10 +-- quest/m2/pop-skipping/peer-reconfigure.md | 52 +++++++++++- 8 files changed, 259 insertions(+), 45 deletions(-) diff --git a/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md b/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md index 796d7705b26..af726830b58 100644 --- a/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md +++ b/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md @@ -17,8 +17,20 @@ that, and the request carries no `host`, so host-routed tenants dialing the same path are indistinguishable at the endpoint. - Delete `resolve_mtls`. Build the request through `api_request` with - `mtls: true`, plus `host` in proxy mode as the mode already sends it, and + `mtls` set, plus `host` in proxy mode as the mode already sends it, and resolve through `authorize`, so a grant is read one way for every credential. +- Send the peer's identity beside the flag: `mtls=true` stays exactly as + documented, since deployed endpoints compare it literally, and a new + `mtls_name=` carries the client certificate's first SAN DNS name (the + CN when it has none), so the endpoint can tell peers apart and issue a + per-peer grant. `moq_tokio::tls::PeerIdentity` grows a name accessor beside + `expiry`, parsed with the `x509-parser` it already uses. A certificate with + neither sends the flag alone, and the docs say the name is a value the + endpoint matches, never proof by itself; the CA that signed the cert is. + The name has to reach `verify_mtls` on every transport: QUIC keeps a + `PeerIdentity` on the request, but the HTTPS and WebSocket path reduces the + verified chain to the unit `MtlsPeer` marker in `rs/moq-relay/src/web.rs`, + so that marker carries the identity too. - Token mode: `mtls: true` satisfies "has a credential" without a JWT or a `key`; the certificate is the token. No grant means unrestricted, as today. - Proxy mode: the endpoint returns a grant like anyone else, and no grant is a @@ -33,7 +45,9 @@ same path are indistinguishable at the endpoint. answers "no" still partitions the mesh. - Tests: a proxy-mode mTLS peer refused by an empty reply, scoped by a narrow grant, and admitted unrestricted in token mode; `host` present on the proxy - request. Update the mTLS section of `doc/bin/relay/auth.md`. + request; `mtls=true` unchanged with `mtls_name` carrying the SAN name over + both QUIC and WebSocket, and absent for a nameless cert. + Update the mTLS and auth API sections of `doc/bin/relay/auth.md`. ## Required diff --git a/quest/m2/709-automatic-letsencrypt-support.md b/quest/m2/709-automatic-letsencrypt-support.md index c3e727fed2e..43cd953ae1c 100644 --- a/quest/m2/709-automatic-letsencrypt-support.md +++ b/quest/m2/709-automatic-letsencrypt-support.md @@ -1,23 +1,79 @@ -# [M] Automatic LetsEncrypt support +# [L] Automatic ACME certificates ## Goal -Implement and verify the behavior tracked in [#709](https://github.com/moq-dev/moq/issues/709) -within the issue's stated scope and boundaries. +`moq-relay` provisions and renews its own publicly trusted certificate from +an ACME directory (Let's Encrypt by default) when an `[acme]` section is +configured, so running a relay no longer needs certbot or a hand-copied +certificate. The certificate persists on disk, so a restart serves at once +and only a missing, expiring, or no-longer-matching certificate performs a +challenge. ## Plan -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. +Use [instant-acme](https://crates.io/crates/instant-acme) with default +features off and its aws-lc-rs feature, matching the workspace crypto +provider so no second provider enters the tree. `deny.toml` gates the new +dependency. -### Issue context +- **Config.** A new `[acme]` section on the relay config: `domains` (the + certificate's names), `contact` (an email for the account), `dir` (where + the account key, certificate, and key live; required, no default, since the + relay has no state directory and packaging already grants + `/var/lib/moq-relay`), and `directory` (the ACME directory URL, defaulting + to Let's Encrypt production, with staging documented for rate-limit-safe + trials). Setting it replaces `listen.tls.cert` / `listen.tls.key`, and + supplying both is a config error. It also feeds `web.https` when that + section's cert and key are unset, so one certificate serves the host. +- **Challenge.** HTTP-01 only. The relay's `web.http` router gains + `/.well-known/acme-challenge/{token}`, answered from the in-flight + authorization, and config validation requires `web.http` to be bound while + `[acme]` is set, since the challenge must reach port 80 and `listen.tls` + is UDP. TLS-ALPN-01 is deliberately not offered: it needs a new TCP :443 + acceptor that collides with `web.https`. DNS-01 is not offered either, so a + wildcard in `acme.domains` is a config error rather than a first start that + blocks forever on an issuance HTTP-01 cannot complete. +- **Delivery.** The issuer writes the PEM chain and key under `acme.dir` and + the existing `notify` file watcher swaps them in on the next handshake, so + no new `moq-tokio` surface is needed and per-worker rotation rides + [TLS rotation atomicity](/quest/m1/2924-moq-relay-tls-rotation-is-not-atomic-across-thread-per.md). + Writes are atomic renames into the watched directory, which the watcher + already treats as a reload. The account key and the serving key are created + owner-only (`0600`, the mode `doc/bin/relay/auth.md` already requires of + private keys) before any bytes land, never with the umask default, so a + group-searchable `acme.dir` cannot leak a key to another local user. Only the quinn and noq backends on the tokio + runtime reload certificates today: quiche snapshots its TLS material when + the listener is built, and the io_uring workers read the certificate once at + bind. Config validation refuses `[acme]` with those until they can rotate, + rather than letting a relay serve an expired certificate; lifting the + refusal is part of whatever gives them a reload path. +- **Startup.** With no cached certificate the relay blocks accepting QUIC + until the first issuance succeeds; a self-signed placeholder cannot serve + browsers, so there is nothing honest to serve meanwhile. With a cached + certificate it starts immediately and renews in the background. A cached + certificate whose SANs do not cover every configured domain counts as + missing: an operator who adds or replaces a hostname while reusing + `acme.dir` gets a fresh issuance, not a wait for the renewal threshold. +- **Renewal.** A daily jittered check reads the cached certificate's expiry + with the `x509-parser` already used for peer expiry and renews once a + third of its lifetime remains, Let's Encrypt's own rule for 90-day + certificates. A failed renewal logs and waits for the next check; it never + restarts the relay or discards the serving certificate. The account key is + reused across runs so renewals do not create accounts. +- **Docs.** `doc/bin/relay/config.md` gains the section; `doc/setup/prod.md` + makes ACME the recommended path and keeps the external-certificate path; + `doc/bin/relay/http.md` notes the challenge route; the packaging TOML and + the nix module expose `acme.dir` under the state directory. -One of the biggest hurdles to running `moq-relay` is getting a certificate. We could simplify the setup process by using a Acme library to automatically provision and rotate TLS certificates. - -LetsEncrypt has an API that performs a HTTP/TLS challenge to prove that you own a given domain name. `moq-relay` already requires a public IP address and listening on UDP, optionally listening on TCP too. We could leverage that to perform the challenge in-process without an external `certbot` process. I would use [instant-acme](https://crates.io/crates/instant-acme). - -Bonus points for saving the certificate to disk and only performing the challenge if it's missing or about to expire. This will avoid making LetsEncrypt a startup blocker and potentially avoid being rate limited. +Tests: an in-process ACME test server (or the Pebble container behind a +feature flag) issues on first run, a restart with a valid cached certificate +performs no challenge, a cached certificate missing a configured domain is +reissued at startup, a certificate near expiry renews and the QUIC +listener serves the new chain without restart, a renewal failure leaves the +old certificate serving, both generated keys are mode `0600`, and the config +rejects `[acme]` alongside explicit +paths, without `web.http`, with a wildcard domain, or on a backend that +cannot reload. ## Closes diff --git a/quest/m2/README.md b/quest/m2/README.md index 863b21d40e4..b1d9d50bb24 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -72,7 +72,7 @@ can act on. Each still carries its own plan and regression test. - [#3058](/quest/m2/3058-moq-relay-a-revalidation-re-check-cannot-update-a.md) - moq-relay: a revalidation re-check cannot update a session's tier, and changes its alias only by closing it - [#3137](/quest/m2/3137-moqsrc-bound-the-pending-rendition-subscriptions-a.md) - moqsrc: bound the pending rendition subscriptions a catalog can open - [#3115](/quest/m2/3115-moqsink-the-publication-has-no-generation-so-a-flush.md) - moqsink: a flushing restart after EOS opens a new publication generation -- [#709](/quest/m2/709-automatic-letsencrypt-support.md) - Automatic LetsEncrypt support +- [#709](/quest/m2/709-automatic-letsencrypt-support.md) - the relay provisions and renews its own ACME certificate over HTTP-01, persisted on disk - [Room SDK](/quest/m2/room-sdk.md) - a headless room package: a room is a path prefix, no service, no storage - [LiveKit shim](/quest/m2/livekit-shim.md) - a drop-in livekit-client-compatible package running rooms over MoQ - [Auth verdict](/quest/m2/auth-verdict.md) - the relay hands an opaque credential to its auth API and is told the grant; lands as the proxy mode in #3044 diff --git a/quest/m2/drain/README.md b/quest/m2/drain/README.md index aeb0431c426..060792f0456 100644 --- a/quest/m2/drain/README.md +++ b/quest/m2/drain/README.md @@ -11,9 +11,10 @@ node anyway (a cached resolve, or a pool alias) just gets another GOAWAY. Only after sessions drain or the stop deadline expires does the process exit and the new software boot. -GOAWAY only reaches MoQ sessions, and today nobody acts on it: moq-net (Rust -and JS) decodes and logs the message without closing or migrating the session, -and the wire message is lite04+/IETF only besides. So the stop deadline is the +GOAWAY only reaches MoQ sessions, and only the Rust client acts on it today: +`moq_tokio::Connection` migrates, while `js/net` decodes and logs the message +without closing or migrating the session, and the wire message is +lite04+/IETF only besides. So the stop deadline is the real backstop - for pre-lite04 versions, for client SDKs deployed before client-goaway ships, and for in-process ingest gateways (RTMP/SRT/WHIP/WHEP), which have no GOAWAY equivalent at all: their grace is the DNS-drain window @@ -32,8 +33,9 @@ immediately GOAWAYs any new arrival, so an embedding process can enter drain on SIGTERM after the DNS window and still bound the total stop time. Expose enough phase/session state to prove which bound ended a drain. -**client-goaway.** Rust and JS reconnectors preserve the app-visible session -but discard the pinned address and resolve DNS again before dialing. This is a +**client-goaway.** The JS reconnector migrates like the Rust one, preserving +the app-visible session while resolving DNS again before dialing, and the Rust +path is pinned by a test. This is a scale-down prerequisite, not merely a deploy improvement. RTMP/SRT/WHIP/WHEP cannot receive MoQ GOAWAY, so their contract remains DNS withdrawal followed by the stop deadline and encoder reconnect. @@ -43,9 +45,9 @@ by the stop deadline and encoder reconnect. - [Relay drain api](/quest/m2/drain/relay-drain-api.md) - a drain hook that GOAWAYs every session, including new arrivals, triggered by the embedding process on SIGTERM -- [Client goaway](/quest/m2/drain/client-goaway.md) - native and JavaScript - clients reconnect through a fresh DNS resolve after GOAWAY; today both only - log it +- [Client goaway](/quest/m2/drain/client-goaway.md) - the JavaScript client + migrates on GOAWAY with a handover and the guarded redirect the Rust client + already has, and the Rust drain path gets its regression test ## Related diff --git a/quest/m2/drain/client-goaway.md b/quest/m2/drain/client-goaway.md index 3ca773a8b60..55d80336fdb 100644 --- a/quest/m2/drain/client-goaway.md +++ b/quest/m2/drain/client-goaway.md @@ -2,12 +2,81 @@ ## Goal -Reconnect native and JavaScript clients through fresh DNS after GOAWAY. -Preserve the app-visible session and avoid pinned addresses; today both -clients only decode and log GOAWAY. +The JavaScript client migrates on GOAWAY the way the Rust client already +does: it dials the replacement while the old session keeps serving, follows a +redirect URI under the same guard, and keeps the app-visible handle and its +origins across the swap. The Rust client's fleet-drain path (an empty-URI +GOAWAY redialed through a fresh DNS resolve) is pinned by a regression test. ## Plan moq.pro's (downstream) fleet drain orchestration relies on this behavior: a drained node is already out of DNS when GOAWAY fires, so a re-resolve is what lands clients on a healthy relay. + +Everything below describes `dev`, which is where this quest lands: `main` +still has `moq-native`'s close-only `Reconnect` and no `Connection.Shared`. + +### Rust is done except the proof + +On `dev`, `moq_tokio::Connection` already handles GOAWAY: the session loop returns the +message, `Redirect::resolve` guards the URI (scheme tier never drops, no +widening to a local host, optional same-host mode), the loop redials while a +`Draining` handle keeps the old session serving until it closes or overstays +the handover cap, and `Status::Migrating` is visible to callers. Every dial +resolves DNS again, since `Addrs` holds URLs and each backend resolves at dial +time, so nothing is pinned. The relay always sends an empty URI, and the only +end-to-end coverage is the cluster sibling test with a redirect. + +Add the fleet-drain regression test: a relay GOAWAYs with an empty URI and a +timeout, the client redials the configured URL through a fresh resolve (a +resolver the test can repoint), live tracks hand over at a group boundary, +and the old session closes within the handover cap. The name-based redirect +guard stays [its own quest](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md). + +### JavaScript is greenfield + +On `dev`, `js/net` logs the lite GOAWAY URI, logs the IETF draft-17+ URI, and +does not decode the body at all on the draft-14 to -16 shared control stream. +`Reload` reconnects only on `closed`, tears the old connection down in its +effect cleanup, and `Connection.Shared` (`js/net/src/connection/pool.ts`) +pools by URL href. + +- Surface the peer's GOAWAY on `Established` as a drain signal carrying the + resolved URI and the timeout, decoded on every wire the client speaks, + including the draft-14 to -16 adapter route that currently returns before + reading the body. +- `Reload` mirrors `Draining`: on GOAWAY it dials the target immediately, + swaps the origin wiring (`forwardAnnounced`, `publish`, `subscribe`) once + the replacement is established, and leaves the old session to close on its + own or at a handover cap: the configured cap, lowered to the peer's timeout + when the wire carried a positive one. Lite and IETF drafts 14 to 16 carry no + timeout, and the IETF decoder reads an absent one as zero, so absence means + the cap and never a zero-length handover. Groups in flight finish. A GOAWAY does not go through the backoff delay; a failed + replacement dial does. +- Port the guard: follow by default, refuse a scheme-tier drop or a widening + to a local host, and offer the same-host mode. An empty URI redials the + current URL. A redirect with a certificate pin (`serverCertificateHashes`) + is refused unless the host is unchanged, since the pin cannot verify another + relay; the pool already refuses to share pinned connections. +- `Connection.Shared` re-keys its entry to the redirect target, so a later + caller configured with that URL shares the migrated connection. The app's + handle and shared origin are unchanged; only the pool key moves, and a + caller still asking for the original URL gets a fresh entry. When the + target key already holds a live entry, that entry wins for every new + lookup: the original key is tombstoned so a later caller for either URL + lands on the target's entry, while the migrating entry keeps serving the + handles it already handed out and retires when the last of them releases. + Two connections to one relay for that overlap is the honest cost; entry + removal is identity-guarded so neither cleanup can delete the other's + entry. +- Tests against the in-tree relay: an empty-URI drain migrates without a + dropped group, a GOAWAY without a timeout hands over at the configured cap, + a redirect moves the pool key and origins, a redirect onto an already + pooled key tombstones the original key while existing handles drain, each guard refusal + closes rather than reconnects, and the draft-14 to -16 route decodes the + URI. + +## Related + +- [Redirect guard by name](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md) - the Rust guard classifies local hosts by name; the JS port inherits the same gap until it lands diff --git a/quest/m2/plan-hls-identity.md b/quest/m2/plan-hls-identity.md index eb3d4da533f..b62b6e687df 100644 --- a/quest/m2/plan-hls-identity.md +++ b/quest/m2/plan-hls-identity.md @@ -12,14 +12,43 @@ per-broadcast identity to hang a generation on. ## Plan -Run /plan-quest with the maintainer. The question is narrow: a CDN edge needs -one URL to mean the same bytes on every edge, so the ID has to be minted by -the publisher and travel with the content, not derived from wall-clock or a -relay-local counter. +Deliberately unsettled after a 2026-09 planning pass. The open question is +which layer owns the identity at all: moq-net, hang, or only a managed +deployment (moq.pro) that already keys recordings itself. Until that is +decided this quest stays a plan quest; do not start +[HLS generation](/quest/m2/hls-generation.md) against a guess. -Candidate sources to settle between: a per-broadcast value returned on resolve -(`TRACK_INFO`), a publisher-declared field in the hang catalog, or something -the archive line already keys recordings by. Also settle the mid-broadcast -rendition reconfigure hole that -[HLS generation](/quest/m2/hls-generation.md) records, since a generation that -only moves on restart cannot version a live reconfigure either way. +What the pass settled, so it is not re-derived: + +- The requirement is unchanged: a CDN edge needs one URL to mean the same + bytes on every edge, so the ID has to be minted by the publisher and travel + with the content, never a wall-clock or a relay-local counter. +- The two collision sources stay distinct. A live rendition reconfigure under + the same track name is not banned by any rule (MSF forbids it, hang does + not) and `moq-hls` already tears the pooled rendition down and rebuilds it + when the catalog config changes, so only the URL reuse remains there. A + publisher restart is the harder half: `moq-net` splices a re-attached + source into the existing broadcast by hop ID so consumers never observe + it, and group sequences restart at 0. +- An epoch on ANNOUNCE_START / ANNOUNCE_UPDATE is rejected. Announcements are + capability routes, and a `transcode/*` route followed by a more specific + `transcode/foo` route must keep stitching by hop ID; a per-route epoch would + break that. `main`'s copy of `draft-lcurley-moq-lite.md` still carries the + old Epoch text on those messages and on TRACK, TRACK_INFO, SUBSCRIBE, and + FETCH; `dev` deleted all of it in [moq#3225](https://github.com/moq-dev/moq/pull/3225), + and `dev` is the draft this decision is made against. It was never + implemented on either branch. +- The candidates left, none chosen: an `Epoch` field on TRACK_INFO stamped + from `broadcast::Info` (per-broadcast value, per-track carriage, fits the + existing rule that TRACK_INFO is fixed for a track's lifetime, and a + takeover across epochs would end the subscription instead of splicing); a + hang catalog root `generation` (no wire change, routing-blind, so a + restart's groups can land on the old handle before the new catalog); an + epoch segment in the broadcast path under prefix routes (relays need + nothing, every consumer and token must resolve the newest, random values do + not order); or no protocol identity, with the managed edge minting one per + publisher session downstream. +- If a wire epoch is chosen it folds into lite-06-wip, is minted at random + on `broadcast::Info` with an application override so an archive replays a + recording under its recorded value, and reaches moq-transport as a + SUBSCRIBE_OK parameter. diff --git a/quest/m2/pop-skipping/README.md b/quest/m2/pop-skipping/README.md index 8aa0876ef70..e77ef3ec8d0 100644 --- a/quest/m2/pop-skipping/README.md +++ b/quest/m2/pop-skipping/README.md @@ -137,8 +137,8 @@ replaces the active session while an identical render is a no-op. It also preserves the last-good topology on malformed input, keeps an identical fallback session alive, redacts credentials from parse errors, and tracks overlapping gossip paths so an old unannounce cannot stale a replacement. The remaining -boundary is structured policy that does not live in the URL, especially selected -wire version and the two directional costs of one bidirectional session. +boundary is structured policy that does not live in the URL: the two directional +costs of one bidirectional session, which one `?cost=` cannot split. ## Quests @@ -147,9 +147,9 @@ wire version and the two directional costs of one bidirectional session. draining or idle - [Rank](/quest/m2/pop-skipping/rank.md) - rank warm relay candidates by cold cost and adopt only downhill, with a hold that outlasts cost propagation -- [Peer reconfigure](/quest/m2/pop-skipping/peer-reconfigure.md) - reconfigure - peer policy when negotiated versions or directional costs change, proving - both sides of an asymmetric link +- [Peer reconfigure](/quest/m2/pop-skipping/peer-reconfigure.md) - structured + peer entries carry the charged cost, the declared cost, and the credential, + and a change redials, proving both sides of an asymmetric link ## Related diff --git a/quest/m2/pop-skipping/peer-reconfigure.md b/quest/m2/pop-skipping/peer-reconfigure.md index ce074c2b73b..9609c8edee4 100644 --- a/quest/m2/pop-skipping/peer-reconfigure.md +++ b/quest/m2/pop-skipping/peer-reconfigure.md @@ -2,7 +2,51 @@ ## Goal -Reconfigure structured peer policy when negotiated versions or directional -costs change. Extend [moq#2874](https://github.com/moq-dev/moq/pull/2874) and -prove both directions of an asymmetric link without regressing the landed -cost, credential, or no-op behavior. +A cluster peer entry can carry structured policy that has no home in a URL: +the price this relay charges to pull from the peer, the price it declares to +the peer for the reverse direction, and the credential. Changing any of it at +runtime replaces the live session the same way a `?cost=` change does today, +and an identical render stays a no-op. + +## Plan + +[moq#2874](https://github.com/moq-dev/moq/pull/2874) landed the URL half: +`?cost=` and an inline `?jwt=` are dial configuration, the query-less URL is +the identity, and a changed render replaces the session while preserving +inactive fallbacks and the last-good topology. What it cannot express is an +asymmetric link from one side. One `?cost=N` is both what this relay charges +locally to pull from the peer and what it declares in SETUP as its own egress +price, so pricing the two directions differently needs the peer to list us +with its own `?cost=`. + +Decisions: + +- Peer entries in `cluster.connect` and `connect_api` accept an object beside + the bare URL string, deserialized untagged into the existing `DialTarget`: + `url` (required, still the canonical identity), `cost` (what this relay + charges to pull from the peer, the routing input), `egress` (what it declares + in SETUP as its own price toward the peer, defaulting to `cost`), and + `token` (replaces an inline `?jwt=`). Unknown fields reject the whole list, + so a typo keeps the last-good topology exactly as a malformed URL does. +- Gossip and mDNS keep advertising URLs only. Their allowlist admits `?cost=` + alone, and the draft already makes a declared price an assertion the + receiver may override, so a peer never needs to push structured policy at us. +- Any field change replaces the session, the rule [moq#2874](https://github.com/moq-dev/moq/pull/2874) + set for `?cost=`. A URL entry and an object entry that normalize to the same + `DialTarget` are the same entry. Reconciling cost in place without a redial + is deliberately not done: it would be a second code path for one field. +- No per-peer wire version. ALPN negotiation picks the best common version per + session and the global `--version` list is the only pin; a per-peer + override is a fleet cutover concern that stays downstream. + +The split lands in `moq_tokio::Client` as separate charged and declared costs +(today `with_cost` sets both), and in the relay's session setup so the routing +side reads the charged value while SETUP carries the declared one. + +Tests: the asymmetric link in both directions, two relays each pricing the +other differently, with routes ranked per side from the charged value and the +declared value visible on the far side; a `connect_api` update that changes +only `egress` redials that peer and no other; an identical object render is a +no-op; an object and URL form of one peer coexisting is a conflict; an unknown +field keeps the previous list. Update `doc/bin/relay/cluster.md` and +`doc/bin/relay/config.md` with the object form. From 9b90ea5a485901bfd3f488b7461bf147f5bc9b6b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 16:34:41 -0700 Subject: [PATCH 08/29] fix(claude): fetch and resolve stale branches before spawn-quest triages (#3405) Co-authored-by: Claude Opus 5 --- .claude/skills/spawn-quest/SKILL.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.claude/skills/spawn-quest/SKILL.md b/.claude/skills/spawn-quest/SKILL.md index 89da4bcf589..f7a52c735ef 100644 --- a/.claude/skills/spawn-quest/SKILL.md +++ b/.claude/skills/spawn-quest/SKILL.md @@ -8,9 +8,20 @@ Before you begin, read `quest/AGENTS.md` completely. Resolve the first argument to a questline directory under `quest/`, defaulting to `quest/` itself: `m0` means `quest/m0`. +`git fetch origin` **before** looking at claims. The remote branch is the +coordination mechanism, so stale remote-tracking refs mean offering a quest +another agent already claimed, and the spawned agent finds out only when its +push is rejected. + Read every quest in the scope, not just the summaries in each `README.md`. Drop -the ones a recent branch or open PR already claims, and say when a branch is -stale (old, no open PR) so it can be reused or deleted. +the ones a recent branch or open PR already claims. + +Resolve a stale branch (old, no open PR) before keeping its quest in the start +pool, rather than only mentioning it. Left in place it makes the agent's claim +push fail as non-fast-forward, which the agent then reports as a lost race that +never happened. Inspect its tip: reuse the work, or delete the ref only when it +is confirmed stale, is not checked out, and has no unmerged commits. Otherwise +treat the quest as claimed and take it out of the pool. Put every quest to the user in priority order - the depth-first walk of the scope's `Quests` lists - with a recommendation and the one fact behind it. @@ -27,8 +38,7 @@ unblock it. ## Spawning -`git fetch origin` once, then settle a base branch **per quest** and pass it -to that quest's agent. Apply the Branch Targeting rules in `CLAUDE.md`: `dev` +Settle a base branch **per quest** and pass it to that quest's agent. Apply the Branch Targeting rules in `CLAUDE.md`: `dev` only for a semver break in a published API, `main` for everything else. One scope mixes both, so a single base for the whole wave puts some PR on the wrong branch. Never let an agent derive its own base: an agent branching from From ecd2fdf3cf2e7332cf919063114d2ae34b2ab6f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 17:18:52 -0700 Subject: [PATCH 09/29] docs(quest): finish the reviewed planning contracts (#3411) Co-authored-by: Claude Fable 5.1 Co-authored-by: GPT-5 --- ...rs-bypass-auth-api-mode-so-proxy-grants.md | 3 +- quest/m2/709-automatic-letsencrypt-support.md | 98 ++++++++++++++----- quest/m2/drain/README.md | 7 +- quest/m2/drain/client-goaway.md | 43 ++++---- quest/m2/plan-hls-identity.md | 8 +- quest/m2/pop-skipping/peer-reconfigure.md | 26 +++-- 6 files changed, 124 insertions(+), 61 deletions(-) diff --git a/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md b/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md index af726830b58..b1979817e59 100644 --- a/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md +++ b/quest/m2/3087-relay-mtls-peers-bypass-auth-api-mode-so-proxy-grants.md @@ -46,7 +46,8 @@ same path are indistinguishable at the endpoint. - Tests: a proxy-mode mTLS peer refused by an empty reply, scoped by a narrow grant, and admitted unrestricted in token mode; `host` present on the proxy request; `mtls=true` unchanged with `mtls_name` carrying the SAN name over - both QUIC and WebSocket, and absent for a nameless cert. + both QUIC and WebSocket, the CN for a certificate with no DNS SAN, and + absent for a nameless cert. Update the mTLS and auth API sections of `doc/bin/relay/auth.md`. ## Required diff --git a/quest/m2/709-automatic-letsencrypt-support.md b/quest/m2/709-automatic-letsencrypt-support.md index 43cd953ae1c..a79ae41df68 100644 --- a/quest/m2/709-automatic-letsencrypt-support.md +++ b/quest/m2/709-automatic-letsencrypt-support.md @@ -24,56 +24,100 @@ dependency. to Let's Encrypt production, with staging documented for rate-limit-safe trials). Setting it replaces `listen.tls.cert` / `listen.tls.key`, and supplying both is a config error. It also feeds `web.https` when that - section's cert and key are unset, so one certificate serves the host. + section's cert and key are unset, so one certificate serves the host; + explicit `web.https` paths win, and the docs say so, since an operator may + deliberately front HTTPS with a different certificate. - **Challenge.** HTTP-01 only. The relay's `web.http` router gains `/.well-known/acme-challenge/{token}`, answered from the in-flight authorization, and config validation requires `web.http` to be bound while `[acme]` is set, since the challenge must reach port 80 and `listen.tls` - is UDP. TLS-ALPN-01 is deliberately not offered: it needs a new TCP :443 + is UDP. The relay cannot prove external reachability, so the contract is + documented instead: the ACME server connects to port 80 of every configured + domain, and the operator either binds `web.http` there or forwards 80 to + it. A challenge that never arrives surfaces as the bounded first-issuance + failure below, naming the domain and the port. TLS-ALPN-01 is deliberately not offered: it needs a new TCP :443 acceptor that collides with `web.https`. DNS-01 is not offered either, so a wildcard in `acme.domains` is a config error rather than a first start that blocks forever on an issuance HTTP-01 cannot complete. -- **Delivery.** The issuer writes the PEM chain and key under `acme.dir` and - the existing `notify` file watcher swaps them in on the next handshake, so - no new `moq-tokio` surface is needed and per-worker rotation rides +- **Delivery.** The issuer writes the key and the PEM chain as one file + under `acme.dir`, and the existing `notify` file watcher swaps it in on the + next handshake, so no new `moq-tokio` surface is needed and per-worker + rotation rides [TLS rotation atomicity](/quest/m1/2924-moq-relay-tls-rotation-is-not-atomic-across-thread-per.md). - Writes are atomic renames into the watched directory, which the watcher - already treats as a reload. The account key and the serving key are created + One file is what makes the rotation atomic: the watcher reloads both paths + on every event and refuses a mismatched pair, so separate key and chain + files would open a window where a new chain meets the old key, and a + restart inside that window could not build TLS at all. The TLS loader + therefore accepts a combined PEM for both `cert` and `key`, and the ACME + output is a single atomic rename into the watched directory. A reload that + fails keeps serving the last complete pair. With `[acme]` set, failing to + establish the file watcher is a startup error, since a relay that cannot + reload would silently serve its certificate to expiry. The account key and the serving key are created owner-only (`0600`, the mode `doc/bin/relay/auth.md` already requires of - private keys) before any bytes land, never with the umask default, so a - group-searchable `acme.dir` cannot leak a key to another local user. Only the quinn and noq backends on the tokio + private keys) before any bytes land, never with the umask default, and a + missing `acme.dir` is created `0700`, so a group-searchable parent cannot + leak a key to another local user. Only the quinn and noq backends on the tokio runtime reload certificates today: quiche snapshots its TLS material when the listener is built, and the io_uring workers read the certificate once at bind. Config validation refuses `[acme]` with those until they can rotate, rather than letting a relay serve an expired certificate; lifting the refusal is part of whatever gives them a reload path. -- **Startup.** With no cached certificate the relay blocks accepting QUIC - until the first issuance succeeds; a self-signed placeholder cannot serve - browsers, so there is nothing honest to serve meanwhile. With a cached - certificate it starts immediately and renews in the background. A cached - certificate whose SANs do not cover every configured domain counts as - missing: an operator who adds or replaces a hostname while reusing - `acme.dir` gets a fresh issuance, not a wait for the renewal threshold. +- **Startup.** With no usable cached certificate, bind `web.http` first and + serve only the challenge route while issuance is pending. Reuse that bound + listener for the normal router after issuance, so there is no + close-and-rebind race. QUIC, HTTPS, the other HTTP routes, health, and the + readiness notification stay unavailable until the certificate and key are + persisted; a self-signed placeholder cannot serve browsers, so there is + nothing honest to serve meanwhile. The attempt is bounded per the + repository's retry rule: capped exponential backoff with jitter inside a + startup budget (a few minutes, configurable), after which the relay exits + with the last real ACME error, the domain, and the port it expected the + challenge on. With a usable cached certificate it starts immediately and + renews in the background. Usable means not yet expired: an expired cache + serves exactly as badly as none, so it blocks the same way and goes through + the same bounded first issuance. A cached certificate whose SANs do not + cover every configured domain counts as missing: an operator who adds or + replaces a hostname while reusing `acme.dir` gets a fresh issuance, not a + wait for the renewal threshold. The directory URL that issued the cached + certificate is persisted beside it, and a changed `acme.directory` (staging + to production, say) also counts as missing, so a staging certificate is + never served as if trusted. - **Renewal.** A daily jittered check reads the cached certificate's expiry with the `x509-parser` already used for peer expiry and renews once a third of its lifetime remains, Let's Encrypt's own rule for 90-day - certificates. A failed renewal logs and waits for the next check; it never - restarts the relay or discards the serving certificate. The account key is - reused across runs so renewals do not create accounts. + certificates. Each renewal attempt owns its own retry sequence per the + repository's retry rule: capped exponential backoff with jitter inside a + bounded budget, stopping early on an explicit non-retryable ACME response + (a rejected order, a rate limit with a stated wait), so a transient + directory blip does not burn a day of the renewal window. A renewal that + exhausts its budget logs and waits for the next check; it never restarts + the relay or discards the serving certificate. The account key is reused + across runs so renewals do not create accounts. - **Docs.** `doc/bin/relay/config.md` gains the section; `doc/setup/prod.md` makes ACME the recommended path and keeps the external-certificate path; `doc/bin/relay/http.md` notes the challenge route; the packaging TOML and the nix module expose `acme.dir` under the state directory. Tests: an in-process ACME test server (or the Pebble container behind a -feature flag) issues on first run, a restart with a valid cached certificate -performs no challenge, a cached certificate missing a configured domain is -reissued at startup, a certificate near expiry renews and the QUIC -listener serves the new chain without restart, a renewal failure leaves the -old certificate serving, both generated keys are mode `0600`, and the config -rejects `[acme]` alongside explicit -paths, without `web.http`, with a wildcard domain, or on a backend that -cannot reload. +feature flag) issues on first run while only the HTTP-01 route answers, then +the same listener serves the normal router without a rebind; a restart with a +valid cached certificate performs no challenge, a cached certificate missing +a configured domain or issued by a different directory is reissued at +startup, an unreachable directory fails the first start inside the budget +with an actionable error, a restart with an expired cached certificate blocks +and reissues rather than serving it, a transient renewal error retries within +the attempt and a non-retryable response stops it early, a certificate near +expiry renews and both the quinn and noq listeners serve the new chain through +a fresh handshake without restart (the `web.https` reload is a separate test), +a renewal failure leaves the old certificate serving, a reload of a +half-written pair keeps the old one, both generated keys are mode `0600` after +issuance and after renewal, and the config rejects `[acme]` alongside explicit +paths, without `web.http`, with a wildcard domain, or on a backend that cannot +reload. + +## Required + +- [TLS rotation atomicity](/quest/m1/2924-moq-relay-tls-rotation-is-not-atomic-across-thread-per.md) - every supported worker must share one reloadable identity before ACME can promise renewal ## Closes diff --git a/quest/m2/drain/README.md b/quest/m2/drain/README.md index 060792f0456..61c1ff07178 100644 --- a/quest/m2/drain/README.md +++ b/quest/m2/drain/README.md @@ -13,8 +13,9 @@ the new software boot. GOAWAY only reaches MoQ sessions, and only the Rust client acts on it today: `moq_tokio::Connection` migrates, while `js/net` decodes and logs the message -without closing or migrating the session, and the wire message is -lite04+/IETF only besides. So the stop deadline is the +and at most closes the session afterwards (the IETF path does, the lite path +does not), leaving any reconnect to the ordinary close-triggered backoff +rather than migrating. The wire message is lite04+/IETF only besides. So the stop deadline is the real backstop - for pre-lite04 versions, for client SDKs deployed before client-goaway ships, and for in-process ingest gateways (RTMP/SRT/WHIP/WHEP), which have no GOAWAY equivalent at all: their grace is the DNS-drain window @@ -35,7 +36,7 @@ enough phase/session state to prove which bound ended a drain. **client-goaway.** The JS reconnector migrates like the Rust one, preserving the app-visible session while resolving DNS again before dialing, and the Rust -path is pinned by a test. This is a +path gains the regression test it lacks. This is a scale-down prerequisite, not merely a deploy improvement. RTMP/SRT/WHIP/WHEP cannot receive MoQ GOAWAY, so their contract remains DNS withdrawal followed by the stop deadline and encoder reconnect. diff --git a/quest/m2/drain/client-goaway.md b/quest/m2/drain/client-goaway.md index 55d80336fdb..9cec9cca5f1 100644 --- a/quest/m2/drain/client-goaway.md +++ b/quest/m2/drain/client-goaway.md @@ -6,7 +6,8 @@ The JavaScript client migrates on GOAWAY the way the Rust client already does: it dials the replacement while the old session keeps serving, follows a redirect URI under the same guard, and keeps the app-visible handle and its origins across the swap. The Rust client's fleet-drain path (an empty-URI -GOAWAY redialed through a fresh DNS resolve) is pinned by a regression test. +GOAWAY redialed through a fresh DNS resolve) gains the regression test it +lacks today. ## Plan @@ -36,16 +37,18 @@ guard stays [its own quest](/quest/m1/2624-moq-native-goaway-redirect-guard-clas ### JavaScript is greenfield -On `dev`, `js/net` logs the lite GOAWAY URI, logs the IETF draft-17+ URI, and -does not decode the body at all on the draft-14 to -16 shared control stream. -`Reload` reconnects only on `closed`, tears the old connection down in its -effect cleanup, and `Connection.Shared` (`js/net/src/connection/pool.ts`) -pools by URL href. +On `dev`, `js/net` logs the lite GOAWAY URI and keeps that session open, +logs the IETF draft-17+ URI and then closes the session when its control +loop ends, and on the draft-14 to -16 shared control stream reads the message +body but returns without decoding it. Nothing migrates: `Reload` reconnects +only after `closed` fires, through its backoff, tears the old connection +down in its effect cleanup, and `Connection.Shared` +(`js/net/src/connection/pool.ts`) pools by URL href. - Surface the peer's GOAWAY on `Established` as a drain signal carrying the resolved URI and the timeout, decoded on every wire the client speaks, - including the draft-14 to -16 adapter route that currently returns before - reading the body. + including the draft-14 to -16 adapter route that currently returns without + decoding the body it has already read. - `Reload` mirrors `Draining`: on GOAWAY it dials the target immediately, swaps the origin wiring (`forwardAnnounced`, `publish`, `subscribe`) once the replacement is established, and leaves the old session to close on its @@ -63,20 +66,20 @@ pools by URL href. caller configured with that URL shares the migrated connection. The app's handle and shared origin are unchanged; only the pool key moves, and a caller still asking for the original URL gets a fresh entry. When the - target key already holds a live entry, that entry wins for every new - lookup: the original key is tombstoned so a later caller for either URL - lands on the target's entry, while the migrating entry keeps serving the - handles it already handed out and retires when the last of them releases. - Two connections to one relay for that overlap is the honest cost; entry - removal is identity-guarded so neither cleanup can delete the other's - entry. + target key already holds a live entry, that entry wins: the migrating entry + is removed from the pool without replacing the target entry. Existing + handles keep its migrated connection, but a new lookup for the original URL + dials fresh and a target lookup joins the target entry. The migrated entry + retires when its last existing handle releases it. Two connections to one + relay for that overlap is the honest cost; entry removal is identity-guarded + so neither cleanup can delete another entry. - Tests against the in-tree relay: an empty-URI drain migrates without a dropped group, a GOAWAY without a timeout hands over at the configured cap, a redirect moves the pool key and origins, a redirect onto an already - pooled key tombstones the original key while existing handles drain, each guard refusal - closes rather than reconnects, and the draft-14 to -16 route decodes the - URI. + pooled key keeps existing handles on both entries but makes a new caller for + the original URL dial fresh, each guard refusal closes rather than + reconnects, and the draft-14 to -16 route decodes the URI. -## Related +## Required -- [Redirect guard by name](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md) - the Rust guard classifies local hosts by name; the JS port inherits the same gap until it lands +- [Redirect guard by name](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md) - pin validated DNS results before enabling cross-host redirects by default diff --git a/quest/m2/plan-hls-identity.md b/quest/m2/plan-hls-identity.md index b62b6e687df..9d2cfcc63cb 100644 --- a/quest/m2/plan-hls-identity.md +++ b/quest/m2/plan-hls-identity.md @@ -18,7 +18,7 @@ deployment (moq.pro) that already keys recordings itself. Until that is decided this quest stays a plan quest; do not start [HLS generation](/quest/m2/hls-generation.md) against a guess. -What the pass settled, so it is not re-derived: +The pass settled the following points, so they do not need to be re-derived: - The requirement is unchanged: a CDN edge needs one URL to mean the same bytes on every edge, so the ID has to be minted by the publisher and travel @@ -50,5 +50,7 @@ What the pass settled, so it is not re-derived: publisher session downstream. - If a wire epoch is chosen it folds into lite-06-wip, is minted at random on `broadcast::Info` with an application override so an archive replays a - recording under its recorded value, and reaches moq-transport as a - SUBSCRIBE_OK parameter. + recording under its recorded value. On moq-lite it rides TRACK_INFO (lite's + SUBSCRIBE_OK carries only the resolved group); on moq-transport, which has + no TRACK_INFO, it rides a SUBSCRIBE_OK parameter, and the adapter maps + both onto the same `track::Info` field. diff --git a/quest/m2/pop-skipping/peer-reconfigure.md b/quest/m2/pop-skipping/peer-reconfigure.md index 9609c8edee4..01b4fe30319 100644 --- a/quest/m2/pop-skipping/peer-reconfigure.md +++ b/quest/m2/pop-skipping/peer-reconfigure.md @@ -26,15 +26,25 @@ Decisions: `url` (required, still the canonical identity), `cost` (what this relay charges to pull from the peer, the routing input), `egress` (what it declares in SETUP as its own price toward the peer, defaulting to `cost`), and - `token` (replaces an inline `?jwt=`). Unknown fields reject the whole list, - so a typo keeps the last-good topology exactly as a malformed URL does. + `token` (replaces an inline `?jwt=`). `DialTarget` grows `egress` and a + normalized credential, with an inline `?jwt=` and an object `token` parsed + to the same representation, and its equality covers every field, so an + `egress`-only or `token`-only update is a change and the token is never + dropped. Unknown fields reject the whole list, so a typo keeps the + last-good topology exactly as a malformed URL does, and so does an object + whose `url` still carries `?cost=` or `?jwt=`: policy has one home per + form, and a mixed entry is rejected rather than given a precedence a + migration could silently get wrong. - Gossip and mDNS keep advertising URLs only. Their allowlist admits `?cost=` alone, and the draft already makes a declared price an assertion the receiver may override, so a peer never needs to push structured policy at us. - Any field change replaces the session, the rule [moq#2874](https://github.com/moq-dev/moq/pull/2874) set for `?cost=`. A URL entry and an object entry that normalize to the same - `DialTarget` are the same entry. Reconciling cost in place without a redial - is deliberately not done: it would be a second code path for one field. + `DialTarget` are one entry, deduplicated as `parse_peer_list` already does; + two entries for one identity with differing policy are the conflict that + rejects the list, whatever their forms. Reconciling cost in place without a + redial is deliberately not done: it would be a second code path for one + field. - No per-peer wire version. ALPN negotiation picks the best common version per session and the global `--version` list is the only pin; a per-peer override is a fleet cutover concern that stays downstream. @@ -46,7 +56,9 @@ side reads the charged value while SETUP carries the declared one. Tests: the asymmetric link in both directions, two relays each pricing the other differently, with routes ranked per side from the charged value and the declared value visible on the far side; a `connect_api` update that changes -only `egress` redials that peer and no other; an identical object render is a -no-op; an object and URL form of one peer coexisting is a conflict; an unknown -field keeps the previous list. Update `doc/bin/relay/cluster.md` and +only `egress` or only `token` redials that peer and no other; an identical +object render is a no-op; equivalent URL and object forms of one peer +deduplicate while differing policies for one identity conflict; an unknown +field, or an object whose `url` carries `?cost=` or `?jwt=`, keeps the +previous list. Update `doc/bin/relay/cluster.md` and `doc/bin/relay/config.md` with the object form. From 7bdc170dd2e76e668ebe6177c1e126db23cf4136 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 17:36:36 -0700 Subject: [PATCH 10/29] ci(nightly): compile the Apple-only code paths (#3413) Co-authored-by: Claude Opus 5 --- .github/workflows/nightly.yml | 43 +++++++++++++++++++++++++++++++---- CLAUDE.md | 4 ++-- doc/bin/obs.md | 2 +- rs/CLAUDE.md | 9 ++++---- rs/justfile | 16 ++++++------- 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3a297fec09a..9426bab0c93 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -10,10 +10,10 @@ name: Nightly # that a PR gate can't justify for how rarely it catches something. `obs` is # diff-independent because its PR-time counterpart can't be: obs.yml has to # name the paths that can break an external link to libmoq.a, and no list of -# paths covers every one of them (see the comment there). `windows` is -# diff-independent because what it catches is a whole target's worth of -# `#[cfg(target_os = "windows")]` code that no Linux job compiles at all, and a -# Windows runner is too slow to gate every PR on. +# paths covers every one of them (see the comment there). `windows` and `macos` +# are diff-independent because what they catch is a whole target's worth of +# `#[cfg(target_os = ...)]` code that no Linux job compiles at all, and those +# runners are throttled too hard to gate every PR on. # # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. @@ -134,3 +134,38 @@ jobs: - name: Check shell: bash run: just rs windows + + # `just rs macos` is the only thing that compiles moq-video's VideoToolbox + # encode/decode and its ScreenCaptureKit / AVFoundation capture, plus + # moq-audio's ScreenCaptureKit system audio and its TCC permission pre-check. + # The Linux gate skips all of it. moq-video has a release-time backstop, since + # libmoq's `libmoq-v*` tag build compiles it on Apple Silicon, but moq-audio + # has none: libmoq and moq-ffi take it codecs-only, so no release build ever + # compiles its capture backend. + # + # Same shape as `windows` and for the same reasons: a separate job because it + # needs an Apple host and the runner-native toolchain, and no Swatinem cache + # because a once-a-day job shouldn't evict the Linux `target/` that every pull + # request restores. swift.yml already compiles moq-ffi here, so this stays + # scoped to the two crates holding the Apple-gated code. + macos: + name: macOS + runs-on: macos-latest + timeout-minutes: 60 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + # Pinned: `--locked` fixes just's own dependencies, not which version of + # just cargo selects. + - name: Install just + run: cargo install --locked just@1.52.0 + + - name: Check + run: just rs macos diff --git a/CLAUDE.md b/CLAUDE.md index 3b12020ce95..6bf7748f5f1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,7 +44,7 @@ One tool stays unrequired: `swift` exists only on macOS, so swift.yml is its gat Two path-filtered workflows run recipes of their own alongside `check.yml`, each for something the Linux `just check` can't reach: swift.yml (`swift/scripts/check.sh`, on a Mac) and obs.yml (`just obs ci`, which links the OBS plugin against nixpkgs' libobs/Qt6 -- Linux is the only platform where that needs no obs-deps download). -Four gates live outside the PR path, in `.github/workflows/nightly.yml`: `just rs audit` (cargo-deny) because an advisory lands without this repo changing, `just rs features` (the `--all-features` and `--no-default-features` compiles) because each is a full extra workspace compile that shares almost nothing with the default one, `just obs ci` because obs.yml's path filter can't be complete (a build script can change what linking `libmoq.a` needs without touching a manifest), and `just rs windows` on a Windows runner because the Linux jobs never compile the `#[cfg(target_os = "windows")]` code at all. A break there lands on `main` rather than being caught in review, which is the accepted trade; anything that must block a merge belongs in `check`. +Five gates live outside the PR path, in `.github/workflows/nightly.yml`: `just rs audit` (cargo-deny) because an advisory lands without this repo changing, `just rs features` (the `--all-features` and `--no-default-features` compiles) because each is a full extra workspace compile that shares almost nothing with the default one, `just obs ci` because obs.yml's path filter can't be complete (a build script can change what linking `libmoq.a` needs without touching a manifest), and `just rs windows` / `just rs macos` on their own runners because the Linux jobs never compile the `#[cfg(target_os = ...)]` code at all. A break there lands on `main` rather than being caught in review, which is the accepted trade; anything that must block a merge belongs in `check`. ## Architecture @@ -72,7 +72,7 @@ Top-level layout only. Per-crate and per-package detail lives in the nested guid - `/rs/` - Rust crates: core networking (`moq-net`), native helpers, the relay, CLIs, media muxing/codecs, and the FFI/C bindings. See `rs/CLAUDE.md`. - `/js/` - TypeScript/JavaScript packages for the browser, published as `@moq/*`. See `js/CLAUDE.md`. - `/py/`, `/swift/`, `/kt/`, `/go/`, `/dart/` - language wrappers over `rs/moq-ffi` (see [Language Bindings](#language-bindings)). `/py/` has `py/CLAUDE.md`; the others defer to their `README.md`. -- `/cpp/` - C/C++ consumers of `libmoq`. `cpp/obs/` is the OBS Studio plugin (CMake; links `libmoq` via `MOQ_LOCAL`), licensed GPL-2.0-or-later because it links `libobs`. `just check` type-checks it via `just obs compile`, which needs headers rather than an obs-deps download, and obs.yml links it on Linux for `cpp/obs/` and `rs/libmoq/` PRs. Still manual, like `just rs macos`: `just obs build` (a loadable plugin, and the only path on macOS/Windows) and `just obs test` (`cpp/obs/test/` against stubbed libobs/libmoq under ThreadSanitizer). See `doc/bin/obs.md`. +- `/cpp/` - C/C++ consumers of `libmoq`. `cpp/obs/` is the OBS Studio plugin (CMake; links `libmoq` via `MOQ_LOCAL`), licensed GPL-2.0-or-later because it links `libobs`. `just check` type-checks it via `just obs compile`, which needs headers rather than an obs-deps download, and obs.yml links it on Linux for `cpp/obs/` and `rs/libmoq/` PRs. Still manual: `just obs build` (a loadable plugin, and the only path on macOS/Windows) and `just obs test` (`cpp/obs/test/` against stubbed libobs/libmoq under ThreadSanitizer). See `doc/bin/obs.md`. - `/demo/` - demos and test media: relay configs, the web demo, MoQ Boy, media hosting, and a network throttle script. - `/test/` - test harnesses that span more than one language or need a server. `test/smoke/` is the cross-language interop matrix (`just test smoke[-full]`); `test/wasm/` runs the `@moq/wasm` bindings in headless Chromium against a real relay (`just test wasm`), which is the only behavioral coverage `rs/moq-wasm` has. - `/doc/` - documentation site (VitePress, deployed via Cloudflare). The `/draft/` section is generated from `drafts/` by `doc/.vitepress/drafts.ts`. diff --git a/doc/bin/obs.md b/doc/bin/obs.md index a134b4982a2..025223c3d93 100644 --- a/doc/bin/obs.md +++ b/doc/bin/obs.md @@ -101,7 +101,7 @@ Run them after touching `cpp/obs/src/`. just obs test ``` -`just obs ci` runs the same tests without the sanitizer, and that is the copy which gates a merge: `obs.yml` invokes that recipe, while `just obs test` is manual like `just rs macos`. ThreadSanitizer adds the interleavings on top, needs its own build, and needs a Clang or GCC whose runtime *runs* on the host, so `just obs test` fails rather than skipping when one isn't available. On Windows run it from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. +`just obs ci` runs the same tests without the sanitizer, and that is the copy which gates a merge: `obs.yml` invokes that recipe, while `just obs test` is manual. ThreadSanitizer adds the interleavings on top, needs its own build, and needs a Clang or GCC whose runtime *runs* on the host, so `just obs test` fails rather than skipping when one isn't available. On Windows run it from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. Both find the `libobs` headers the same way `just obs compile` does, and regenerate `moq.h` the same way; set `OBS_INCLUDE_DIR` to point somewhere else. That shared step asks cargo where the header landed and reads the answer with `jq`, so outside the dev shell (running from WSL, say) `jq` has to be installed alongside cargo and the compiler. diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 73954a531f7..f3ef492f66e 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -176,7 +176,7 @@ Then `Config::load()?` (initializes tracing), build clients/servers via `.init() - Config-merge regressions belong next to the config (`moq-relay/src/config.rs::tests`); they serialize env mutation with a lock since clap reads env. -- **Local checks only compile the host's platform and target, and PR CI is Linux-only.** `#[cfg(target_os = "...")]` code for other platforms is invisible to them, and `cargo fmt` skips those modules too. Windows and Mac runners cost too much for a per-PR gate, so those platforms are manual: +- **Local checks only compile the host's platform and target, and PR CI is Linux-only apart from swift.yml.** `#[cfg(target_os = "...")]` code for other platforms is invisible to them, and `cargo fmt` skips those modules too. Windows and Mac runners are throttled too hard for a per-PR gate, so those platforms run nightly instead of blocking a merge: - Windows (moq-video's Media Foundation and D3D11 backends): `just rs windows`, which must run ON Windows. You can't reproduce it elsewhere, since cross-compiling dies in openh264-sys2's vendored C++. It names `moq-cli/play` explicitly, since that feature is what pulls in moq-video's wgpu renderer and moq-audio's cpal output; a default-feature build compiles neither. - macOS (moq-video's VideoToolbox and ScreenCaptureKit, moq-audio's system audio): `just rs macos`, which must run ON macOS. Scoped to moq-video + moq-audio, and needs `--all-features` because moq-audio's capture backend is off by default. @@ -185,11 +185,12 @@ Then `Config::load()?` (initializes tracing), build clients/servers via `.init() What still compiles these automatically, and when: - - moq-video's platform backends are gated on `target_os` alone, and libmoq depends on moq-video, so a `libmoq-v*` tag builds them on `windows-latest` and Apple Silicon. That's a release-time backstop, not a PR one: a break lands on `main` and surfaces at the tag. - - **moq-audio's macOS capture has no automated backstop at all.** ScreenCaptureKit system audio and the TCC pre-check sit behind the off-by-default `capture` feature, and every consumer leaves it off (libmoq and moq-ffi don't enable it; moq-cli's own `capture` feature is off in release builds). `just rs macos` is the only thing that compiles it, ever. + - `.github/workflows/nightly.yml` runs `just rs windows` and `just rs macos` on their own runners once a day. That is the only automated coverage either recipe has, and it is deliberately not a required check: a break lands on `main` and surfaces the next morning rather than in review. + - moq-video's platform backends are gated on `target_os` alone, and libmoq depends on moq-video, so a `libmoq-v*` tag builds them on `windows-latest` and Apple Silicon. That's a release-time backstop on top of the nightly one. + - moq-audio's macOS capture has no release-time backstop: ScreenCaptureKit system audio and the TCC pre-check sit behind the off-by-default `capture` feature, and every consumer leaves it off (libmoq and moq-ffi don't enable it; moq-cli's own `capture` feature is off in release builds). Nightly's `just rs macos` is the only thing that compiles it. - `.github/workflows/swift.yml` still runs on a Mac for `swift/**` and `rs/moq-ffi/**` PRs, so moq-ffi and the Swift wrapper keep a PR-time gate. - Run the matching recipe by hand when you touch this code, and if you can't (no such host), say plainly in the PR that it's uncompiled rather than implying CI covered it. + Nightly catching it a day later is the fallback, not the plan: run the matching recipe by hand when you touch this code, and if you can't (no such host), say plainly in the PR that it's uncompiled rather than implying CI covered it. - **`just rs loom` model-checks concurrent handoffs in kio and moq-net.** It stays outside `check` and `test`: `--cfg loom` swaps kio's Mutex/atomics for loom's instrumented ones, which rebuilds the whole dependency tree and can't share artifacts with a normal `cargo test`. Use it when developing or diagnosing concurrent handoffs. Budget about a minute of model checking on top of that build. The search is exhaustive on purpose, so don't reach for `preemption_bound` to speed it up; the recipe already buys the speed back with `--release`, which matters here because a model check reruns the body once per interleaving. diff --git a/rs/justfile b/rs/justfile index bb95147786f..d3a0692c007 100644 --- a/rs/justfile +++ b/rs/justfile @@ -374,9 +374,9 @@ fix-changed $FILES: # builds vendored C++ that needs an MSVC toolchain and openh264 is a # non-optional dependency. # -# Windows runners cost too much for a per-PR gate, so nightly.yml runs this -# once a day instead. A break there lands on main rather than being caught in -# review, which is the same trade the other nightly gates make. +# Windows runners are throttled too hard for a per-PR gate, so nightly.yml runs +# this once a day instead. A break there lands on main rather than being caught +# in review, which is the same trade the other nightly gates make. # # Default features rather than `--all-features`: jemalloc and the quiche # backend don't build on MSVC, while moq-video's Linux-only `nvidia` default @@ -395,11 +395,11 @@ windows *args: # VideoToolbox encode/decode and its ScreenCaptureKit / AVFoundation capture, # plus moq-audio's ScreenCaptureKit system audio and TCC permission pre-check. # -# Nothing in CI runs this either; Mac runners cost too much for a per-PR gate. -# The moq-video half has a release-time backstop, since libmoq's `libmoq-v*` tag -# build compiles it on Apple Silicon. The moq-audio half has none: libmoq and -# moq-ffi take it codecs-only, so no release build compiles its capture -# backend and this recipe is the only thing that ever does. +# Same trade as `windows`: nightly.yml runs this once a day rather than gating a +# merge on a Mac runner. The moq-video half also has a release-time backstop, +# since libmoq's `libmoq-v*` tag build compiles it on Apple Silicon. The +# moq-audio half has none: libmoq and moq-ffi take it codecs-only, so no release +# build compiles its capture backend and this recipe is the only thing that does. # # Scoped to those two crates instead of the workspace, because they hold all # the Apple-gated code the Linux gate misses. moq-ffi (and through it the From 63b212df46c421c4e3a30982c18a2a21826143b6 Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Sat, 5 Sep 2026 02:49:32 +0200 Subject: [PATCH 11/29] fix(moq-video): stop openh264 leaking its picture pool, and survive a lost picture (#3357) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Luke Curley --- rs/moq-transcode/src/feed.rs | 6 + rs/moq-transcode/src/rung.rs | 63 ++- .../src/decode/backend/mediafoundation.rs | 130 ++++- rs/moq-video/src/decode/backend/mod.rs | 34 +- rs/moq-video/src/decode/backend/nvdec.rs | 91 ++- rs/moq-video/src/decode/backend/openh264.rs | 517 +++++++++++++++++- rs/moq-video/src/decode/backend/probe.rs | 105 +++- rs/moq-video/src/decode/backend/v4l2.rs | 107 +++- .../src/decode/backend/videotoolbox.rs | 153 +++++- rs/moq-video/src/decode/consumer.rs | 192 ++++++- rs/moq-video/src/decode/decoder.rs | 17 +- rs/moq-video/src/decode/sink.rs | 30 +- .../test_data/bframes_64x64_pattern_30f.h264 | Bin 0 -> 9131 bytes rs/moq-video/src/v4l2.rs | 41 +- 14 files changed, 1330 insertions(+), 156 deletions(-) create mode 100644 rs/moq-video/src/decode/test_data/bframes_64x64_pattern_30f.h264 diff --git a/rs/moq-transcode/src/feed.rs b/rs/moq-transcode/src/feed.rs index 5cf2c071788..c74d7a6f0cf 100644 --- a/rs/moq-transcode/src/feed.rs +++ b/rs/moq-transcode/src/feed.rs @@ -198,6 +198,12 @@ async fn decode(inner: &Inner, sender: &broadcast::Sender) -> Result<(), E } } } + // A buffered decoder still owns this group's tail. Drain it before the + // boundary so those frames stay in this group rather than appearing after + // the next group's keyframe. + for decoded in decoder.flush().await? { + let _ = sender.send(Item::Frame(Arc::new(decoded))); + } let _ = sender.send(Item::End); } Ok(()) diff --git a/rs/moq-transcode/src/rung.rs b/rs/moq-transcode/src/rung.rs index 6928f9054cd..bd70e7ba1e6 100644 --- a/rs/moq-transcode/src/rung.rs +++ b/rs/moq-transcode/src/rung.rs @@ -340,8 +340,8 @@ async fn fetch(rung: Rung, request: moq_net::track::GroupRequest) -> Result<(), } /// Transcode one fetched source group to completion into one output group, -/// draining the encoder at the end. (The live path rides the shared feed -/// instead; see [`live`].) +/// draining the decoder and encoder at the end. (The live path rides the shared +/// feed instead; see [`live`].) async fn transcode_group( pipeline: Pipeline, container: &moq_mux::catalog::hang::Container, @@ -392,8 +392,8 @@ async fn transcode_group_inner( } } - // One-shot group: drain whatever the encoder still buffers. Each packet keeps - // the timestamp of the frame it was encoded from, so the tail stays in step. + // One-shot group: drain both codec stages. Each packet keeps the timestamp of + // the frame it was encoded from, so the tail stays in step. write(output, active, pipeline.finish().await?)?; Ok(()) } @@ -486,35 +486,44 @@ impl Pipeline { let mut encoded = Vec::new(); for raw in self.decoder.decode(payload, timestamp, keyframe).await? { - // Already at the rung size (the decoder scaled): feed the frame through - // as-is, keeping a GPU frame on the GPU. - let raw = match raw.size() == self.size { - true => raw, - false => raw.resize_with(self.size, &self.rung.resize)?, - }; - if self.encoder.is_none() { - let mut opened = self.rung.encode(raw.surface.color()).await?; - if std::mem::take(&mut self.pending_keyframe) { - opened.keyframe(); - } - self.encoder = Some(opened); - } - let encoder = self.encoder.as_mut().expect("just opened"); - encoded.extend(encoder.encode(raw).await?); + encoded.extend(self.encode_frame(raw).await?); } Ok(encoded) } - /// Drain the encoder, keeping each buffered packet's own timestamp. + /// Resize and encode one decoded frame. + async fn encode_frame(&mut self, raw: moq_video::Frame) -> Result, Error> { + // Already at the rung size (the decoder scaled): feed the frame through + // as-is, keeping a GPU frame on the GPU. + let raw = match raw.size() == self.size { + true => raw, + false => raw.resize_with(self.size, &self.rung.resize)?, + }; + if self.encoder.is_none() { + let mut opened = self.rung.encode(raw.surface.color()).await?; + if std::mem::take(&mut self.pending_keyframe) { + opened.keyframe(); + } + self.encoder = Some(opened); + } + let encoder = self.encoder.as_mut().expect("just opened"); + Ok(encoder.encode(raw).await?) + } + + /// Drain the decoder and then the encoder, keeping every frame's timestamp. /// - /// Consumes the pipeline, since flushing the encoder consumes it: a one-shot - /// group's pipeline is done once drained. - async fn finish(self) -> Result, Error> { - // No encoder means no frame ever decoded, so there is nothing buffered. - match self.encoder { - Some(encoder) => encoder.finish().await.map_err(Into::into), - None => Ok(Vec::new()), + /// Consumes the pipeline, since finishing the encoder consumes it: a one-shot + /// group's pipeline is done once both codec stages are drained. + async fn finish(mut self) -> Result, Error> { + let mut encoded = Vec::new(); + for raw in self.decoder.flush().await? { + encoded.extend(self.encode_frame(raw).await?); } + + if let Some(encoder) = self.encoder { + encoded.extend(encoder.finish().await?); + } + Ok(encoded) } } diff --git a/rs/moq-video/src/decode/backend/mediafoundation.rs b/rs/moq-video/src/decode/backend/mediafoundation.rs index 5a3306dacec..ea619bf6c14 100644 --- a/rs/moq-video/src/decode/backend/mediafoundation.rs +++ b/rs/moq-video/src/decode/backend/mediafoundation.rs @@ -28,6 +28,7 @@ //! Used only from the one decode task (the consumer's `read` loop), so the COM //! handles are wrapped in a thread-confined `Send` type. +use std::collections::VecDeque; use std::ffi::c_void; use std::mem::ManuallyDrop; use std::ptr; @@ -39,10 +40,11 @@ use windows::Win32::Media::MediaFoundation::{ IMF2DBuffer, IMFDXGIBuffer, IMFDXGIDeviceManager, IMFMediaType, IMFSample, IMFTransform, MF_E_NO_MORE_TYPES, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_STREAM_CHANGE, MF_LOW_LATENCY, MF_MT_FRAME_SIZE, MF_MT_MAJOR_TYPE, MF_MT_MINIMUM_DISPLAY_APERTURE, MF_MT_SUBTYPE, MFCreateMediaType, MFCreateMemoryBuffer, - MFCreateSample, MFMediaType_Video, MFOffset, MFT_CATEGORY_VIDEO_DECODER, MFT_ENUM_FLAG_SORTANDFILTER, - MFT_ENUM_FLAG_SYNCMFT, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, - MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, - MFTEnumEx, MFVideoArea, MFVideoFormat_H264, MFVideoFormat_HEVC, MFVideoFormat_NV12, + MFCreateSample, MFMediaType_Video, MFOffset, MFSampleExtension_Discontinuity, MFT_CATEGORY_VIDEO_DECODER, + MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_SYNCMFT, MFT_MESSAGE_COMMAND_DRAIN, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, + MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, + MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFT_REGISTER_TYPE_INFO, MFTEnumEx, MFVideoArea, MFVideoFormat_H264, + MFVideoFormat_HEVC, MFVideoFormat_NV12, }; use windows::core::{GUID, Interface}; @@ -60,6 +62,10 @@ pub(crate) const NAME: &str = "mediafoundation"; /// 30fps tick. const HNS_PER_SEC: i64 = 10_000_000; const NOMINAL_FPS: i64 = 30; +/// Maximum timestamps retained while accepted access units produce no picture. +/// H.264 and H.265 decoded-picture buffers are smaller than this for the +/// supported profiles, so anything older can no longer be a valid reorder. +const MAX_PENDING: usize = 32; pub(crate) struct MediaFoundation { transform: IMFTransform, @@ -81,6 +87,12 @@ pub(crate) struct MediaFoundation { /// Output buffer size to allocate when `provides_samples` is false. output_size: u32, sample_index: i64, + /// Container timestamps paired with the synthetic sample times fed to the MFT. + pending: VecDeque<(i64, Timestamp)>, + /// Last output timestamp, used only if the MFT emits an unmatched sample. + last_timestamp: Option, + /// Mark the first accepted sample after a drain as a new stream epoch. + discontinuity: bool, /// Kept alive for the MFT's lifetime; the MFT holds its own ref but we own the /// pairing. Drops before `_com`. _manager: IMFDXGIDeviceManager, @@ -116,10 +128,8 @@ impl MediaFoundation { .map_err(|e| mf_err("set D3D manager", e))?; } - // Disable the decoder's output-reorder buffer so each access unit produces - // its frame right away. Our streams carry no B-frames, so this adds no - // latency and avoids holding frames until an end-of-stream drain (which the - // per-access-unit `Backend` interface never signals). + // Ask the decoder to minimize its reorder delay. This is advisory, so stream + // boundaries still perform the explicit Media Foundation drain sequence. let attrs = unsafe { transform.GetAttributes().map_err(|e| mf_err("MFT GetAttributes", e))? }; unsafe { let _ = attrs.SetUINT32(&MF_LOW_LATENCY, 1); @@ -135,6 +145,9 @@ impl MediaFoundation { provides_samples: false, output_size: 0, sample_index: 0, + pending: VecDeque::new(), + last_timestamp: None, + discontinuity: false, _manager: manager, _com: com, }; @@ -276,7 +289,7 @@ impl MediaFoundation { sample.AddBuffer(&buffer).map_err(|e| mf_err("AddBuffer", e))?; let tick = HNS_PER_SEC / NOMINAL_FPS; sample - .SetSampleTime(self.sample_index * tick) + .SetSampleTime(self.sample_time()) .map_err(|e| mf_err("SetSampleTime", e))?; sample .SetSampleDuration(tick) @@ -285,6 +298,11 @@ impl MediaFoundation { Ok(sample) } + /// Synthetic Media Foundation timestamp for the next input sample. + fn sample_time(&self) -> i64 { + self.sample_index * (HNS_PER_SEC / NOMINAL_FPS) + } + /// Pull every frame the MFT has ready, stopping when it asks for more input. /// No-op until the output type is configured (the decoder rejects /// `ProcessOutput` before then). @@ -292,7 +310,7 @@ impl MediaFoundation { /// Reports whether the decoder got anywhere, which is not the same as whether /// it handed back a picture: renegotiating the output type is progress too, and /// it produces no frame. - fn drain_output(&mut self, out: &mut Vec) -> Result { + fn drain_output(&mut self, out: &mut Vec) -> Result { if !self.output_configured { return Ok(false); } @@ -306,7 +324,7 @@ impl MediaFoundation { /// One `ProcessOutput`. Returns `true` if it produced a frame or handled a /// stream change (so the caller keeps draining), `false` on /// `MF_E_TRANSFORM_NEED_MORE_INPUT` (the decoder wants the next access unit). - fn process_output(&mut self, out: &mut Vec) -> Result { + fn process_output(&mut self, out: &mut Vec) -> Result { // In DXVA mode the MFT provides its own texture-backed sample, so we pass a // null slot; otherwise we hand it a system-memory buffer to fill. let provided = if self.provides_samples { @@ -333,7 +351,9 @@ impl MediaFoundation { match result { Ok(()) => { if let Some(sample) = sample { - out.push(self.sample_to_surface(&sample)?); + let sample_time = unsafe { sample.GetSampleTime() }.ok(); + let surface = self.sample_to_surface(&sample)?; + out.push(Frame::new(surface, self.take_timestamp(sample_time))); } Ok(true) } @@ -346,6 +366,32 @@ impl MediaFoundation { } } + /// Pair an output sample back to the container timestamp of its input. + fn take_timestamp(&mut self, sample_time: Option) -> Timestamp { + let found = sample_time.and_then(|time| self.pending.iter().position(|(fed, _)| *fed == time)); + let matched = match found { + Some(index) => self.pending.remove(index), + None => { + let oldest = self.pending.pop_front(); + if oldest.is_some() { + tracing::debug!(?sample_time, "decoder output did not match a fed sample time"); + } + oldest + } + }; + + match matched { + Some((_, timestamp)) => { + self.last_timestamp = Some(timestamp); + timestamp + } + None => { + tracing::warn!("decoder produced output with no access unit outstanding"); + self.last_timestamp.unwrap_or(Timestamp::ZERO) + } + } + } + /// Wrap a decoded output sample as a [`Surface`]. The DXVA path keeps the /// picture on the GPU, blitting it out of the decoder's pool into a texture of /// ours; a system-memory fallback deinterleaves the contiguous NV12 to packed @@ -382,6 +428,10 @@ impl Backend for MediaFoundation { let mut out = Vec::new(); let sample = self.build_sample(&access_unit)?; + if self.discontinuity { + unsafe { sample.SetUINT32(&MFSampleExtension_Discontinuity, 1) } + .map_err(|e| mf_err("mark discontinuity", e))?; + } // The decoder accepts one access unit at a time. If it's still holding // output it returns NOTACCEPTING; drain that, then the input goes in. loop { @@ -405,6 +455,8 @@ impl Backend for MediaFoundation { Err(e) => return Err(mf_err("ProcessInput", e)), } } + self.discontinuity = false; + remember_timestamp(&mut self.pending, self.sample_time(), timestamp); self.sample_index += 1; // The decoder only reports the picture size once it has parsed the first @@ -415,9 +467,28 @@ impl Backend for MediaFoundation { } self.drain_output(&mut out)?; - // The synchronous MFT drains within the call, so every output frame - // belongs to the access unit just submitted. - Ok(out.into_iter().map(|surface| Frame::new(surface, timestamp)).collect()) + Ok(out) + } + + fn flush(&mut self) -> Result, Error> { + let mut out = Vec::new(); + if self.output_configured { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0) + .map_err(|e| mf_err("drain", e))?; + } + self.drain_output(&mut out)?; + // A synchronous MFT may accept input immediately after NEED_MORE_INPUT. + // START_OF_STREAM is still useful to implementations that track the + // notification, but it is optional for this MFT category. + unsafe { + let _ = self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + } + } + self.pending.clear(); + self.discontinuity = true; + Ok(out) } fn name(&self) -> &str { @@ -425,6 +496,15 @@ impl Backend for MediaFoundation { } } +/// Remember which container timestamp belongs to an accepted sample while +/// bounding streams whose pictures are repeatedly dropped. +fn remember_timestamp(pending: &mut VecDeque<(i64, Timestamp)>, sample_time: i64, timestamp: Timestamp) { + pending.push_back((sample_time, timestamp)); + while pending.len() > MAX_PENDING { + pending.pop_front(); + } +} + /// The picture region of a decoder output type, from its minimum display /// aperture: the part of the coded frame that is actually the image. /// @@ -497,3 +577,23 @@ fn enumerate_decoder(subtype: GUID) -> Result { transform.ok_or_else(|| Error::Codec(anyhow::anyhow!("failed to activate decoder MFT"))) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pending_timestamps_are_bounded() { + let mut pending = VecDeque::new(); + for index in 0..MAX_PENDING + 5 { + remember_timestamp( + &mut pending, + index as i64, + Timestamp::from_micros(index as u64).unwrap(), + ); + } + + assert_eq!(pending.len(), MAX_PENDING); + assert_eq!(pending.front().map(|(sample, _)| *sample), Some(5)); + } +} diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index ec6e65ecee8..e5cb5a5ed50 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -67,6 +67,13 @@ pub(crate) trait Backend: Send { /// reordering. fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error>; + /// Return the pictures the codec still holds once the stream has ended. + /// + /// Backends configured for zero delay return no frames. A backend that + /// reorders pictures overrides this so the end of a track does not drop its + /// buffered tail. + fn flush(&mut self) -> Result, Error>; + /// The decoder name in use, e.g. `"videotoolbox"` (for logging). fn name(&self) -> &str; } @@ -123,11 +130,24 @@ const SOFTWARE: Candidate = Candidate { /// `Hardware` / `Software` can never select one: they exist to be asked for by /// name. #[cfg(test)] -const NAMED_ONLY: &[Candidate] = &[Candidate { - name: probe::NAME, - supports: |c| matches!(c, Codec::H264), - open: probe::Probe::open, -}]; +const NAMED_ONLY: &[Candidate] = &[ + Candidate { + name: probe::NAME, + supports: |c| matches!(c, Codec::H264), + open: probe::Probe::open, + }, + Candidate { + name: probe::BUFFERED_NAME, + supports: |c| matches!(c, Codec::H264), + open: probe::Buffered::open, + }, + #[cfg(not(target_os = "macos"))] + Candidate { + name: probe::BLOCKING_FLUSH_NAME, + supports: |c| matches!(c, Codec::H264), + open: probe::BlockingFlush::open, + }, +]; #[cfg(not(test))] const NAMED_ONLY: &[Candidate] = &[]; @@ -249,6 +269,10 @@ mod tests { Ok(Vec::new()) } + fn flush(&mut self) -> Result, Error> { + Ok(Vec::new()) + } + fn name(&self) -> &str { "stub" } diff --git a/rs/moq-video/src/decode/backend/nvdec.rs b/rs/moq-video/src/decode/backend/nvdec.rs index 549f907a548..90a477e65e7 100644 --- a/rs/moq-video/src/decode/backend/nvdec.rs +++ b/rs/moq-video/src/decode/backend/nvdec.rs @@ -14,12 +14,12 @@ //! rides the decoder itself: [`Config::resize`] maps to cuvid's target size, so //! the hardware emits frames already at the output resolution. //! -//! The cuvid parser is driven synchronously: each access unit is pushed with -//! `CUVID_PKT_ENDOFPICTURE` and zero display delay, so its callbacks (sequence / -//! decode / display) all fire inside `cuvidParseVideoData` on the calling -//! thread, and the display queue is drained before `decode` returns. Timestamps -//! are threaded through the parser (`ulClockRate` is set to microseconds), so -//! output frames keep correct presentation times even across reordering. +//! The cuvid parser is driven synchronously: callbacks (sequence / decode / +//! display) fire inside `cuvidParseVideoData` on the calling thread. Each access +//! unit uses `CUVID_PKT_ENDOFPICTURE`, and an explicit end-of-stream packet drains +//! any pictures the display queue still holds. Timestamps are threaded through +//! the parser (`ulClockRate` is set to microseconds), so output frames keep +//! correct presentation times even across reordering. use core::ffi::{c_int, c_uint, c_ulong, c_ulonglong, c_void}; use std::ptr; @@ -52,6 +52,8 @@ pub(crate) struct Nvdec { /// Boxed so its address is stable: the parser holds a raw pointer to it for /// the lifetime of the parser (callbacks dereference it during parse). state: Box, + /// Mark the first access unit after a drain as a new parser epoch. + discontinuity: bool, } // Used from one thread at a time (the decode loop); the CUDA context is rebound @@ -157,44 +159,75 @@ impl Nvdec { } tracing::info!(decoder = NAME, codec = ?codec, resize = ?config.resize, "opened video decoder"); - Ok(Box::new(Self { parser, state })) + Ok(Box::new(Self { + parser, + state, + discontinuity: false, + })) } -} -impl Backend for Nvdec { - fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - // The parser callbacks (decoder create, decode) and the map/copy below - // all need the CUDA context current on this thread. + /// Submit one parser packet and copy every picture its display callback made + /// ready before returning. + fn parse(&mut self, packet: &mut CUVIDSOURCEDATAPACKET) -> Result, Error> { + // Parser callbacks and the map/copy below all need the CUDA context current + // on this thread. self.state .ctx .bind_to_thread() .map_err(|e| codec_err(format!("CUDA bind: {e:?}")))?; - let mut packet = CUVIDSOURCEDATAPACKET { - // ENDOFPICTURE: each payload is one complete access unit, so the - // parser emits it immediately instead of waiting for the next AU to - // detect the picture boundary (one-in one-out latency). - flags: (CUvideopacketflags::CUVID_PKT_TIMESTAMP as c_ulong) - | (CUvideopacketflags::CUVID_PKT_ENDOFPICTURE as c_ulong), - payload_size: access_unit.len() as c_ulong, - payload: access_unit.as_ptr(), - // cuvid's clock rate is microseconds (set at parser creation). - timestamp: timestamp.as_micros() as i64, - }; - - // SAFETY: parser and packet are valid; the payload outlives the call - // (the parser copies what it needs before returning). - let result = unsafe { (self.state.api.parse_video_data)(self.parser, &mut packet) }; + // SAFETY: parser and packet are valid. A non-null payload is owned by the + // caller for the duration of this synchronous call. + let result = unsafe { (self.state.api.parse_video_data)(self.parser, packet) }; if let Some(error) = self.state.error.take() { + self.state.ready.clear(); return Err(codec_err(error)); } if result != cudarc::driver::sys::CUresult::CUDA_SUCCESS { + self.state.ready.clear(); return Err(codec_err(format!("cuvidParseVideoData: {result:?}"))); } let ready = std::mem::take(&mut self.state.ready); ready.iter().map(|disp| self.state.map_frame(disp)).collect() } +} + +impl Backend for Nvdec { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + let mut flags = (CUvideopacketflags::CUVID_PKT_TIMESTAMP as c_ulong) + | (CUvideopacketflags::CUVID_PKT_ENDOFPICTURE as c_ulong); + if self.discontinuity { + flags |= CUvideopacketflags::CUVID_PKT_DISCONTINUITY as c_ulong; + } + let mut packet = CUVIDSOURCEDATAPACKET { + // ENDOFPICTURE: each payload is one complete access unit, so the + // parser sees the picture boundary without waiting for the next AU. + flags, + payload_size: access_unit.len() as c_ulong, + payload: access_unit.as_ptr(), + // cuvid's clock rate is microseconds (set at parser creation). + timestamp: timestamp.as_micros() as i64, + }; + + let frames = self.parse(&mut packet)?; + self.discontinuity = false; + Ok(frames) + } + + fn flush(&mut self) -> Result, Error> { + let mut packet = CUVIDSOURCEDATAPACKET { + // NVIDIA requires an explicit EOS packet to release every picture still + // waiting in display order, even with zero display delay. + flags: CUvideopacketflags::CUVID_PKT_ENDOFSTREAM as c_ulong, + payload_size: 0, + payload: ptr::null(), + timestamp: 0, + }; + let frames = self.parse(&mut packet)?; + self.discontinuity = true; + Ok(frames) + } fn name(&self) -> &str { NAME @@ -527,6 +560,10 @@ mod tests { } } } + for decoded in decoder.flush().unwrap() { + let i420 = decoded.surface.to_i420().unwrap().into_owned(); + out.push((decoded.timestamp.as_micros() as u64, i420)); + } assert!(!out.is_empty(), "NVDEC produced no frames"); out } diff --git a/rs/moq-video/src/decode/backend/openh264.rs b/rs/moq-video/src/decode/backend/openh264.rs index 52ace239326..2350ffac780 100644 --- a/rs/moq-video/src/decode/backend/openh264.rs +++ b/rs/moq-video/src/decode/backend/openh264.rs @@ -4,11 +4,18 @@ //! H.264 access units (SPS/PPS inline ahead of each keyframe) and returns packed //! I420. +use std::cmp::Reverse; +use std::collections::BinaryHeap; + use bytes::Bytes; use moq_net::Timestamp; use openh264::OpenH264API; -use openh264::decoder::{Decoder, DecoderConfig}; +use openh264::decoder::{Decoder, DecoderConfig, Flush}; use openh264::formats::YUVSource; +use openh264_sys2::{ + DECODING_STATE, dsBitstreamError, dsDataErrorConcealed, dsDepLayerLost, dsErrorFree, dsNoParamSets, dsOutOfMemory, + dsRefListNullPtrs, dsRefLost, +}; use super::{Backend, Codec, Config}; use crate::frame::{I420, Surface}; @@ -16,63 +23,513 @@ use crate::{Error, Frame}; pub(crate) const NAME: &str = "openh264"; +/// The decoding states that describe one picture rather than the decoder. +/// +/// Every one of them is ordinary in a live stream. A skipped group loses the +/// reference chain (`dsRefLost`), a subscriber that joins mid-sequence has no +/// parameter sets yet (`dsNoParamSets`), a truncated access unit is a bitstream +/// error, and openh264 reports `dsOutOfMemory` when its picture pool runs dry, +/// having already reinitialised itself before returning. In all of them the +/// decoder is still usable and the next keyframe restores the picture, so they +/// cost a picture rather than the stream. +/// +/// `dsInvalidArgument` and `dsInitialOptExpected` are absent on purpose: they +/// say the decoder was driven wrongly, which no amount of further bitstream +/// fixes. So is `dsDstBufNeedExpan`, which says the picture does not fit the +/// buffer handed in. `dsFramePending` is absent because it is not a lost +/// picture at all: `codec_app_def.h` defines it as needing more throughput +/// before a picture comes out, and openh264 only ever sets it while parsing +/// without decoding, which this backend never asks for. +const PICTURE_LOST: DECODING_STATE = dsRefLost + | dsBitstreamError + | dsDepLayerLost + | dsNoParamSets + | dsDataErrorConcealed + | dsRefListNullPtrs + | dsOutOfMemory; + +/// The subset of [`PICTURE_LOST`] after which openh264 has emptied its +/// reordering buffer. +/// +/// This is the distinction the timestamp queue turns on. `DecodeFrame2WithCtx` +/// calls `ResetDecoder` for exactly these two before returning +/// (`welsDecoderExt.cpp`), and `ResetDecoder` runs +/// `ResetReorderingPictureBuffers`, so every picture the decoder was holding is +/// gone and the timestamps waiting for those pictures are stale. +/// +/// The rest fall through to `ReorderPicturesInDisplay` with the buffer intact. +/// Their pictures are still coming, so discarding the queue there would hand +/// each of them the timestamp of a picture several places later: on a stream +/// that reorders, one skipped group would leave the whole rest of the session +/// stamped a reordering depth early, spacing intact, and nothing would look +/// wrong except that the audio never lines up again. +const DECODER_RESET: DECODING_STATE = dsOutOfMemory | dsRefListNullPtrs; + +/// How many access unit timestamps may be outstanding before the oldest is +/// dropped. +/// +/// H.264 caps the decoded picture buffer at 16 frames, so a conforming stream +/// never has more than that many pictures in flight and this is only reached by +/// one that codes pictures the decoder never hands back. The cap keeps that from +/// growing without bound. +const MAX_PENDING: usize = 32; + pub(crate) struct Openh264 { decoder: Decoder, + /// The timestamps of access units whose picture has not come out yet. + /// + /// Access units arrive in decode order and pictures come back in display + /// order, which differ as soon as the stream codes B slices, so the + /// timestamp handed to the call a picture falls out of is not that picture's. + /// A conforming decoder never releases a picture before every picture that + /// displays ahead of it has been fed in, so the oldest timestamp still + /// outstanding always belongs to the next picture out. + pending: BinaryHeap>, + /// Access units the decoder has refused since the last picture came out, so + /// a run of them logs once rather than once per picture. + lost: u64, +} + +/// Builds an openh264 decoder configured the way this backend needs one. +/// +/// Shared with [`Backend::flush`], which replaces the decoder rather than +/// reusing it: draining goes through `FlushFrame`, whose picture release is the +/// one that does not work single threaded, so every picture drained leaks a +/// slot from a pool of `num_ref_frames + 2`. At the end of a track that costs +/// nothing, because the decoder is about to be dropped. `Decoder::flush` is +/// public and documented as leaving the decoder usable, though, so a caller +/// draining at every seek would exhaust the pool in two or three of them and +/// then lose a group to the reset that follows. +fn new_decoder() -> Result { + let config = DecoderConfig::new().flush_after_decode(Flush::NoFlush); + Decoder::with_api_config(OpenH264API::from_source(), config) + .map_err(|e| Error::Codec(anyhow::anyhow!("openh264 decoder init: {e}"))) } impl Openh264 { /// openh264 decodes H.264 only; `config` is accepted for signature parity /// (no hardware scaler; callers scale the CPU frames themselves). - pub(crate) fn open(codec: Codec, _config: &Config) -> Result, Error> { + pub(crate) fn open(codec: Codec, config: &Config) -> Result, Error> { + Ok(Box::new(Self::new(codec, config)?)) + } + + /// The decoder itself, for a caller that wants the concrete type: the tests + /// drain it through a method the [`Backend`] trait does not carry. + fn new(codec: Codec, _config: &Config) -> Result { if codec != Codec::H264 { return Err(Error::Codec(anyhow::anyhow!( "openh264 cannot decode {}", codec.label() ))); } - let decoder = Decoder::with_api_config(OpenH264API::from_source(), DecoderConfig::new()) - .map_err(|e| Error::Codec(anyhow::anyhow!("openh264 decoder init: {e}")))?; + // `Flush::NoFlush`, against the crate's default. The default calls + // `FlushFrame` after any decode that produced no picture, and + // `FlushFrame` releases the reordering slot through a picture buffer + // pointer openh264 only sets on its threaded path: single threaded that + // pointer is null, so the picture is handed out and its reference count + // never comes back down. The pool holds `num_ref_frames + 2` pictures, so + // a stream that reorders (any stream with B slices) exhausts it within a + // second, and every access unit after that fails with `dsOutOfMemory` + // until the next keyframe. Letting openh264 release pictures on its own + // schedule costs one picture of delay at the start of a reordering + // sequence and nothing at all on a baseline one. + let decoder = new_decoder()?; tracing::info!(decoder = NAME, "opened H.264 decoder"); - Ok(Box::new(Self { decoder })) + Ok(Self { + decoder, + pending: BinaryHeap::new(), + lost: 0, + }) + } + + /// Records an access unit the decoder refused. + /// + /// Returns no frames for a state that describes the picture, so playback + /// carries on and recovers at the next keyframe, and the error itself for a + /// state that describes the decoder. + /// + /// Only a state in [`DECODER_RESET`] takes the outstanding timestamps with + /// it. After the others the decoder still holds the pictures those + /// timestamps belong to. + fn picture_lost(&mut self, timestamp: Timestamp, err: &openh264::Error) -> Result, Error> { + let state = i32::try_from(err.native_code()).unwrap_or(dsErrorFree); + if state == dsErrorFree || state & !PICTURE_LOST != 0 { + return Err(Error::Codec(anyhow::anyhow!("openh264 decode: {err}"))); + } + + if state & DECODER_RESET != 0 { + self.pending.clear(); + } else { + // The decoder kept every older reordered picture, but refused this + // access unit. Remove its timestamp by value: it need not be the oldest + // timestamp in the heap when access units arrive in decode order. + let mut removed = false; + self.pending.retain(|pending| { + if !removed && pending.0 == timestamp { + removed = true; + false + } else { + true + } + }); + } + self.lost += 1; + if self.lost == 1 { + tracing::warn!( + decoder = NAME, + state = format_args!("{state:#06x}"), + "picture lost, waiting for the next keyframe" + ); + } else { + tracing::trace!(decoder = NAME, state = format_args!("{state:#06x}"), "picture lost"); + } + Ok(Vec::new()) + } + + /// Takes the timestamp the next picture out belongs to, reporting a recovery + /// on the way if pictures had been going missing. + fn picture_out(&mut self) -> Option { + if self.lost > 0 { + tracing::info!(decoder = NAME, lost = self.lost, "picture recovered"); + self.lost = 0; + } + self.pending.pop().map(|Reverse(timestamp)| timestamp) + } + + /// Remembers `timestamp` as belonging to a picture still to come. + fn picture_in(&mut self, timestamp: Timestamp) { + self.pending.push(Reverse(timestamp)); + if self.pending.len() > MAX_PENDING { + // Preserve the oldest timestamps, which are the pictures the decoder + // could still hand out next, and discard the newest. Only a stream that + // never hands its pictures back gets here, so rebuilding the small heap + // does not matter. + let mut kept: Vec<_> = self.pending.drain().collect(); + kept.sort_unstable_by_key(|pending| pending.0); + kept.truncate(MAX_PENDING); + self.pending = kept.into_iter().collect(); + } } } +/// Copies one decoded picture out of openh264's own buffers. +fn picture(yuv: &impl YUVSource, timestamp: Timestamp) -> Result { + let (width, height) = yuv.dimensions(); + if width % 2 != 0 || height % 2 != 0 { + return Err(Error::Codec(anyhow::anyhow!( + "decoded frame has odd dimensions {width}x{height}, expected 4:2:0" + ))); + } + let (y_stride, uv_stride, _) = yuv.strides(); + + let frame = I420::from_planes( + yuv.y(), + yuv.u(), + yuv.v(), + y_stride, + uv_stride, + width as u32, + height as u32, + ); + Ok(Frame::new(Surface::I420(frame), timestamp)) +} + impl Backend for Openh264 { fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { - let decoded = self - .decoder - .decode(&access_unit) - .map_err(|e| Error::Codec(anyhow::anyhow!("openh264 decode: {e}")))?; + self.picture_in(timestamp); - // `None` means the decoder buffered the access unit but has no picture - // yet (e.g. parameter sets only, or it needs more data). - let Some(yuv) = decoded else { - return Ok(Vec::new()); + // `None` means the decoder took the access unit but has no picture yet: + // parameter sets only, or a picture its reordering buffer is holding. + // The timestamp is corrected below, once the borrow of the decoder ends. + let decoded = match self.decoder.decode(&access_unit) { + Ok(Some(yuv)) => Some(picture(&yuv, timestamp)?), + Ok(None) => None, + Err(err) => return self.picture_lost(timestamp, &err), }; - let (width, height) = yuv.dimensions(); - if width % 2 != 0 || height % 2 != 0 { - return Err(Error::Codec(anyhow::anyhow!( - "decoded frame has odd dimensions {width}x{height}, expected 4:2:0" - ))); + match decoded { + Some(mut frame) => { + if let Some(timestamp) = self.picture_out() { + frame.timestamp = timestamp; + } + Ok(vec![frame]) + } + None => Ok(Vec::new()), } - let (y_stride, uv_stride, _) = yuv.strides(); + } + + /// Return the reordered pictures still buffered at the end of the stream. + fn flush(&mut self) -> Result, Error> { + let tail = self + .decoder + .flush_remaining() + .map_err(|e| Error::Codec(anyhow::anyhow!("openh264 flush: {e}")))?; + // `tail` borrows the decoder, so copy the pictures before taking their + // timestamps. The placeholder is replaced before any frame reaches a + // caller; an unstamped picture is dropped below. + let decoded: Vec = tail + .iter() + .map(|yuv| picture(yuv, Timestamp::ZERO)) + .collect::>()?; + drop(tail); - let frame = I420::from_planes( - yuv.y(), - yuv.u(), - yuv.v(), - y_stride, - uv_stride, - width as u32, - height as u32, - ); - // openh264 is one-in one-out, so the input timestamp is the output's. - Ok(vec![Frame::new(Surface::I420(frame), timestamp)]) + let mut frames = Vec::with_capacity(decoded.len()); + let mut unstamped = 0usize; + for mut frame in decoded { + match self.picture_out() { + Some(timestamp) => { + frame.timestamp = timestamp; + frames.push(frame); + } + None => unstamped += 1, + } + } + if unstamped > 0 { + tracing::warn!( + decoder = NAME, + dropped = unstamped, + "flushed pictures had no timestamp waiting for them" + ); + } + self.pending.clear(); + self.lost = 0; + // `FlushFrame` leaks a single-threaded decoder's picture-pool slots, so + // replace the drained decoder before it is reused for another stream. + self.decoder = new_decoder()?; + Ok(frames) } fn name(&self) -> &str { NAME } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use moq_mux::codec::annexb; + + use super::*; + + /// libx264 at High profile with three B frames, 30 pictures at 64x64 and a + /// keyframe every 15. Generated with: + /// + /// ```text + /// ffmpeg -f lavfi -i "testsrc2=s=64x64:r=30" -frames:v 30 -c:v libx264 \ + /// -profile:v high -bf 3 -g 15 -pix_fmt yuv420p -f h264 \ + /// bframes_64x64_pattern_30f.h264 + /// ``` + const BFRAMES: &[u8] = include_bytes!("../test_data/bframes_64x64_pattern_30f.h264"); + + /// How many pictures the fixture codes. + const PICTURES: usize = 30; + + /// The interval between fixture pictures: 30fps, the rate it was generated at. + const FRAME_MICROS: u64 = 33_333; + + fn nal_type(nal: &Bytes) -> u8 { + nal.first().map_or(0, |b| b & 0x1f) + } + + /// Splits an Annex-B stream into one access unit per coded picture. + /// + /// A slice opens a new access unit, and so does a parameter set, SEI, or + /// delimiter that follows one: libx264 writes the parameter sets immediately + /// ahead of each IDR, so they belong to the picture after them. + fn access_units() -> Vec<(Bytes, bool)> { + let mut buf = Bytes::from_static(BFRAMES); + let mut iter = annexb::NalIterator::new(&mut buf); + let mut nals: Vec = iter + .by_ref() + .map(|nal| nal.expect("fixture is valid Annex-B")) + .collect(); + nals.extend(iter.flush().expect("fixture is valid Annex-B")); + + let is_slice = |nal: &Bytes| matches!(nal_type(nal), 1 | 5); + let mut units = Vec::new(); + let mut current: Vec = Vec::new(); + for nal in nals { + if current.iter().any(is_slice) && matches!(nal_type(&nal), 1 | 5 | 6 | 7 | 8 | 9) { + units.push(( + annexb::build_prefix(current.iter()), + current.iter().any(|nal| nal_type(nal) == 5), + )); + current.clear(); + } + current.push(nal); + } + if current.iter().any(is_slice) { + units.push(( + annexb::build_prefix(current.iter()), + current.iter().any(|nal| nal_type(nal) == 5), + )); + } + assert_eq!(units.len(), PICTURES, "the fixture split wrongly"); + units + } + + fn at(index: usize) -> Timestamp { + Timestamp::from_micros(index as u64 * FRAME_MICROS).expect("fixture timestamp") + } + + fn open() -> Openh264 { + Openh264::new(Codec::H264, &Config::new()).expect("the software decoder always opens") + } + + fn decode_fixture(broken: Option) -> Vec { + let mut decoder = open(); + let mut frames = Vec::new(); + for (index, (payload, keyframe)) in access_units().into_iter().enumerate() { + let payload = if broken == Some(index) { + payload.slice(..payload.len() / 3) + } else { + payload + }; + frames.extend( + decoder + .decode(payload, at(index), keyframe) + .unwrap_or_else(|e| panic!("access unit {index} ended the stream: {e}")), + ); + } + frames.extend(decoder.flush().expect("the drain works")); + frames + } + + fn pixels(frame: &Frame) -> Vec { + frame.surface.to_i420().expect("software output is I420").data.clone() + } + + /// Draining goes through the release path that does not work single + /// threaded, so the decoder is replaced rather than reused. Flushing + /// repeatedly on a reordering stream would otherwise exhaust the pool. + #[test] + fn repeated_flushes_keep_decoding() { + let mut decoder = open(); + for round in 0..6 { + let mut decoded = 0; + for (index, (payload, keyframe)) in access_units().into_iter().enumerate() { + decoded += decoder + .decode(payload, at(index), keyframe) + .unwrap_or_else(|e| panic!("round {round} access unit {index}: {e}")) + .len(); + } + decoded += decoder.flush().expect("the drain works").len(); + assert_eq!(decoded, PICTURES, "round {round} lost pictures"); + } + } + + /// Regression: a stream that reorders used to exhaust openh264's picture pool + /// within a second, because the crate's flush-after-decode released the + /// reordering slot without releasing the picture. Every access unit after + /// that failed with `dsOutOfMemory` until the next keyframe, which a player + /// sees as a picture that appears for a third of a second and then freezes. + #[test] + fn a_stream_with_b_frames_decodes_all_the_way_through() { + let mut decoder = open(); + let mut decoded = 0; + for (index, (payload, keyframe)) in access_units().into_iter().enumerate() { + decoded += decoder + .decode(payload, at(index), keyframe) + .unwrap_or_else(|e| panic!("access unit {index} failed: {e}")) + .len(); + } + decoded += decoder.flush().expect("the drain works").len(); + + assert_eq!(decoded, PICTURES, "the stream lost pictures"); + } + + /// A reordered picture carries the timestamp of the access unit it was coded + /// in, not of the call it fell out of. The two differ by several frames once + /// B slices are in play, so stamping a picture with the call's timestamp + /// scrambles playout. + #[test] + fn reordered_pictures_keep_their_own_timestamps() { + let mut decoder = open(); + let mut timestamps = Vec::new(); + for (index, (payload, keyframe)) in access_units().into_iter().enumerate() { + for frame in decoder.decode(payload, at(index), keyframe).expect("decodes") { + timestamps.push(frame.timestamp.as_micros()); + } + } + for frame in decoder.flush().expect("the drain works") { + timestamps.push(frame.timestamp.as_micros()); + } + + let expected: Vec = (0..PICTURES).map(|i| at(i).as_micros()).collect(); + assert_eq!(timestamps, expected, "pictures came out mis-stamped"); + } + + /// A truncated access unit costs the picture, not the stream: the decoder + /// keeps taking access units and the next keyframe restores the picture. + #[test] + fn a_truncated_access_unit_costs_pictures_not_the_stream() { + let mut decoder = open(); + let units = access_units(); + let broken = 5; + let mut decoded = 0; + let mut recovered = 0; + for (index, (payload, keyframe)) in units.into_iter().enumerate() { + let payload = if index == broken { + payload.slice(..payload.len() / 3) + } else { + payload + }; + let frames = decoder + .decode(payload, at(index), keyframe) + .unwrap_or_else(|e| panic!("access unit {index} ended the stream: {e}")) + .len(); + decoded += frames; + // The fixture's second keyframe is at 15, so anything after it is the + // decoder having carried on rather than pictures buffered before the + // break. + if index > broken { + recovered += frames; + } + } + assert!(decoded > 0, "nothing decoded at all"); + assert!(recovered > 0, "the decoder never recovered from the truncated unit"); + } + + /// A refused access unit is removed from the pending timestamps by value. In + /// decode order it may be newer than reordered pictures still in the decoder, + /// so popping the oldest would shift every later picture's presentation time. + #[test] + fn a_truncated_access_unit_does_not_shift_later_timestamps() { + let clean = decode_fixture(None); + let expected: HashMap, Timestamp> = + clean.iter().map(|frame| (pixels(frame), frame.timestamp)).collect(); + assert_eq!(expected.len(), clean.len(), "fixture pictures are not unique"); + + let broken = 5; + let mut recovered = 0; + for frame in decode_fixture(Some(broken)) { + let Some(timestamp) = expected.get(&pixels(&frame)) else { + continue; + }; + assert_eq!( + frame.timestamp, *timestamp, + "a surviving picture was stamped as a lost one" + ); + if *timestamp >= at(15) { + recovered += 1; + } + } + assert!(recovered > 0, "the second GOP never recovered clean pictures"); + } + + /// A malformed stream can code pictures OpenH264 never returns. The bound on + /// pending timestamps keeps the oldest candidates and drops the newest one. + #[test] + fn pending_limit_preserves_the_next_picture() { + let mut decoder = open(); + for index in 0..=MAX_PENDING { + decoder.picture_in(at(index)); + } + + let timestamps: Vec<_> = std::iter::from_fn(|| decoder.picture_out()).collect(); + let expected: Vec<_> = (0..MAX_PENDING).map(at).collect(); + assert_eq!(timestamps, expected); + } +} diff --git a/rs/moq-video/src/decode/backend/probe.rs b/rs/moq-video/src/decode/backend/probe.rs index 6f16b3451ea..3304bfc86a6 100644 --- a/rs/moq-video/src/decode/backend/probe.rs +++ b/rs/moq-video/src/decode/backend/probe.rs @@ -9,6 +9,8 @@ //! `Hardware` / `Software`, so it can't be picked by accident. use std::sync::Mutex; +#[cfg(not(target_os = "macos"))] +use std::sync::atomic::{AtomicBool, Ordering}; use std::thread::ThreadId; use bytes::Bytes; @@ -19,6 +21,13 @@ use crate::{Error, Frame, I420, Size, Surface}; pub(crate) const NAME: &str = "probe"; +/// A test decoder that holds one picture until the next call or a flush. +pub(crate) const BUFFERED_NAME: &str = "probe-buffered"; + +/// A test decoder whose flush waits until the test releases it. +#[cfg(not(target_os = "macos"))] +pub(crate) const BLOCKING_FLUSH_NAME: &str = "probe-blocking-flush"; + /// What happened to the codec, and where. `open` and `drop` are the pair that /// matters: the Windows backend opens a COM apartment in one and closes it in /// the other, so they have to land on the same thread. @@ -26,6 +35,12 @@ pub(crate) type Event = (&'static str, ThreadId); static LOG: Mutex> = Mutex::new(Vec::new()); +#[cfg(not(target_os = "macos"))] +static FLUSH_ENTERED: AtomicBool = AtomicBool::new(false); + +#[cfg(not(target_os = "macos"))] +static FLUSH_RELEASED: AtomicBool = AtomicBool::new(false); + /// Serializes the tests that read [`LOG`], which is process-wide. nextest gives /// each test its own process, but `cargo test` does not. #[cfg(not(target_os = "macos"))] @@ -45,6 +60,25 @@ pub(crate) fn take() -> Vec { std::mem::take(&mut LOG.lock().unwrap()) } +/// Prepare the blocking flush probe for one cancellation test. +#[cfg(not(target_os = "macos"))] +pub(crate) fn prepare_blocking_flush() { + FLUSH_ENTERED.store(false, Ordering::SeqCst); + FLUSH_RELEASED.store(false, Ordering::SeqCst); +} + +/// Whether the blocking flush has started on the codec thread. +#[cfg(not(target_os = "macos"))] +pub(crate) fn flush_entered() -> bool { + FLUSH_ENTERED.load(Ordering::SeqCst) +} + +/// Let the blocking flush finish so the codec thread can be joined. +#[cfg(not(target_os = "macos"))] +pub(crate) fn release_flush() { + FLUSH_RELEASED.store(true, Ordering::SeqCst); +} + fn record(what: &'static str) { LOG.lock().unwrap().push((what, std::thread::current().id())); } @@ -58,6 +92,11 @@ pub(crate) const SIZE: Size = Size { pub(crate) struct Probe; +pub(crate) struct Buffered(Option); + +#[cfg(not(target_os = "macos"))] +pub(crate) struct BlockingFlush; + impl Probe { pub(crate) fn open(_codec: Codec, _config: &Config) -> Result, Error> { record("open"); @@ -65,17 +104,38 @@ impl Probe { } } +impl Buffered { + pub(crate) fn open(_codec: Codec, _config: &Config) -> Result, Error> { + Ok(Box::new(Self(None))) + } +} + +#[cfg(not(target_os = "macos"))] +impl BlockingFlush { + pub(crate) fn open(_codec: Codec, _config: &Config) -> Result, Error> { + Ok(Box::new(Self)) + } +} + +fn frame(timestamp: Timestamp) -> Result { + let i420 = I420::new( + SIZE.width, + SIZE.height, + vec![0x80u8; I420::len(SIZE.width, SIZE.height)], + )?; + Ok(Frame::new(Surface::I420(i420), timestamp)) +} + impl Backend for Probe { /// One mid-gray frame per access unit, carrying the timestamp it came in /// with, so a test can tell which payload a frame came from. fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { record("decode"); - let i420 = I420::new( - SIZE.width, - SIZE.height, - vec![0x80u8; I420::len(SIZE.width, SIZE.height)], - )?; - Ok(vec![Frame::new(Surface::I420(i420), timestamp)]) + Ok(vec![frame(timestamp)?]) + } + + fn flush(&mut self) -> Result, Error> { + Ok(Vec::new()) } fn name(&self) -> &str { @@ -83,6 +143,39 @@ impl Backend for Probe { } } +#[cfg(not(target_os = "macos"))] +impl Backend for BlockingFlush { + fn decode(&mut self, _access_unit: Bytes, _timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + Ok(Vec::new()) + } + + fn flush(&mut self) -> Result, Error> { + FLUSH_ENTERED.store(true, Ordering::SeqCst); + while !FLUSH_RELEASED.load(Ordering::SeqCst) { + std::thread::yield_now(); + } + Ok(Vec::new()) + } + + fn name(&self) -> &str { + BLOCKING_FLUSH_NAME + } +} + +impl Backend for Buffered { + fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + Ok(self.0.replace(frame(timestamp)?).into_iter().collect()) + } + + fn flush(&mut self) -> Result, Error> { + Ok(self.0.take().into_iter().collect()) + } + + fn name(&self) -> &str { + BUFFERED_NAME + } +} + impl Drop for Probe { fn drop(&mut self) { record("drop"); diff --git a/rs/moq-video/src/decode/backend/v4l2.rs b/rs/moq-video/src/decode/backend/v4l2.rs index d6c01b48a82..d9bc0223c78 100644 --- a/rs/moq-video/src/decode/backend/v4l2.rs +++ b/rs/moq-video/src/decode/backend/v4l2.rs @@ -37,7 +37,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; use moq_net::Timestamp; -use v4l::v4l_sys::V4L2_CID_MIN_BUFFERS_FOR_CAPTURE; +use v4l::v4l_sys::{V4L2_CID_MIN_BUFFERS_FOR_CAPTURE, V4L2_DEC_CMD_START, V4L2_DEC_CMD_STOP}; use super::{Backend, Codec, Config}; use crate::v4l2::{self, Dequeue, Device, Dir, Format, Planes, Queue, Rect, Request, Role}; @@ -178,16 +178,7 @@ impl V4l2 { fn submit(&mut self, access_unit: &Bytes, timestamp: Timestamp) -> Result<(), Error> { let deadline = Instant::now() + BUFFER_TIMEOUT; let index = loop { - while let Some(buffer) = self.coded.dequeue(&self.device)?.buffer() { - if buffer.failed() { - tracing::warn!( - decoder = NAME, - buffer = buffer.index, - "V4L2 decoder could not decode an access unit" - ); - } - self.coded.reclaim(buffer.index); - } + self.reclaim_coded()?; if let Some(index) = self.coded.take_free() { break index; } @@ -218,6 +209,23 @@ impl V4l2 { Ok(()) } + /// Reclaim every access-unit buffer the driver has finished reading. + fn reclaim_coded(&mut self) -> Result { + let mut reclaimed = 0; + while let Some(buffer) = self.coded.dequeue(&self.device)?.buffer() { + if buffer.failed() { + tracing::warn!( + decoder = NAME, + buffer = buffer.index, + "V4L2 decoder could not decode an access unit" + ); + } + self.coded.reclaim(buffer.index); + reclaimed += 1; + } + Ok(reclaimed) + } + /// Negotiate the CAPTURE queue against the size the driver has just reported. fn negotiate(&mut self) -> Result<(), Error> { if let Some(pictures) = self.pictures.take() { @@ -387,6 +395,44 @@ impl V4l2 { self.device.wait(POLL_INTERVAL); } } + + /// Drive an explicit drain until CAPTURE reaches LAST and OUTPUT is reclaimed. + /// + /// Returns whether a source change arrived during the drain. That event stops + /// the decoder at the old format, so the caller must renegotiate before it can + /// continue through any access units queued for the new format. + fn drain_sequence(&mut self) -> Result<(Vec, bool), Error> { + let mut frames = Vec::new(); + let mut ended = false; + let mut changed = false; + let mut deadline = Instant::now() + DRAIN_TIMEOUT; + loop { + let before = frames.len(); + let reclaimed = self.reclaim_coded()?; + changed |= self.device.take_source_change(); + if !ended { + ended = self.drain(&mut frames)?; + } + + // A source change stops the old sequence at LAST while access units for + // the new format may remain on OUTPUT. Renegotiation is what lets those + // continue, so do not wait here for buffers the stopped decoder cannot + // return yet. + if ended && (changed || self.coded.outstanding() == 0) { + return Ok((frames, changed)); + } + + if reclaimed > 0 || frames.len() > before { + deadline = Instant::now() + DRAIN_TIMEOUT; + } else if Instant::now() >= deadline { + return Err(Error::Codec(anyhow::anyhow!( + "V4L2 decoder did not finish its drain within {DRAIN_TIMEOUT:?}, holding {} access unit(s)", + self.coded.outstanding() + ))); + } + self.device.wait(POLL_INTERVAL); + } + } } /// The `struct timeval` an access unit rides the driver under, which the @@ -459,6 +505,45 @@ impl Backend for V4l2 { Ok(frames) } + fn flush(&mut self) -> Result, Error> { + // A drain only begins with both queues streaming. Before the driver has + // reported a format there is no CAPTURE queue and therefore no picture it + // can hand back. + if self.pictures.is_none() { + // OUTPUT may still hold undecodable access units. Restarting it triggers + // the stateful decoder's seek/reset sequence and returns those buffers, + // so none can surface after the next stream's keyframe. + self.coded.restart(&self.device)?; + self.submitted.clear(); + self.since = None; + return Ok(Vec::new()); + } + + let mut frames = Vec::new(); + loop { + self.device.decoder_cmd(V4L2_DEC_CMD_STOP)?; + let (tail, changed) = self.drain_sequence()?; + frames.extend(tail); + + if changed { + self.negotiate()?; + } + self.device.decoder_cmd(V4L2_DEC_CMD_START)?; + + // A source change can leave work for the new format inside the decoder + // even after every OUTPUT buffer was returned. Resume it, then always run + // a fresh drain so the flush still covers everything submitted before it. + if changed { + continue; + } + break; + } + + self.submitted.clear(); + self.since = None; + Ok(frames) + } + fn name(&self) -> &str { NAME } diff --git a/rs/moq-video/src/decode/backend/videotoolbox.rs b/rs/moq-video/src/decode/backend/videotoolbox.rs index f4043573ff8..915995eb6f1 100644 --- a/rs/moq-video/src/decode/backend/videotoolbox.rs +++ b/rs/moq-video/src/decode/backend/videotoolbox.rs @@ -14,10 +14,11 @@ //! asks, via the same path the capture surfaces use. //! //! Hand-written on the raw `objc2-video-toolbox` bindings; there's no -//! higher-level crate we trust. Decoding is synchronous (no async flag), so the -//! output callback fires from within `decode_frame` before it returns, which is -//! what lets the `!Send` CoreFoundation handles stay thread-confined. +//! higher-level crate we trust. Decoding is synchronous (no async flag), so +//! callbacks run on the decode task. VideoToolbox can still retain reordered +//! pictures until a later decode or an explicit drain. +use std::collections::VecDeque; use std::ffi::c_void; use std::ptr::{self, NonNull}; @@ -26,8 +27,9 @@ use moq_mux::codec::annexb::NalIterator; use moq_net::Timestamp; use objc2_core_foundation::{CFDictionary, CFNumber, CFNumberType, CFRetained, CFString}; use objc2_core_media::{ - CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMTime, CMVideoFormatDescriptionCreateFromH264ParameterSets, - CMVideoFormatDescriptionCreateFromHEVCParameterSets, kCMBlockBufferAssureMemoryNowFlag, + CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMSampleTimingInfo, CMTime, CMTimeFlags, + CMVideoFormatDescriptionCreateFromH264ParameterSets, CMVideoFormatDescriptionCreateFromHEVCParameterSets, + kCMBlockBufferAssureMemoryNowFlag, kCMTimeInvalid, }; use objc2_core_video::{ CVImageBuffer, CVPixelBuffer, CVPixelBufferGetHeight, CVPixelBufferGetWidth, kCVPixelBufferPixelFormatTypeKey, @@ -43,6 +45,11 @@ use crate::{Error, Frame}; pub(crate) const NAME: &str = "videotoolbox"; +/// Maximum access-unit timestamps retained while the decoder produces no +/// picture. H.264 and H.265 decoded-picture buffers are smaller than this for +/// the supported profiles, so anything older can no longer be a valid reorder. +const MAX_PENDING: usize = 32; + /// A parameter-set NAL we pull out of the stream to (re)build the format /// description; `Slice` is everything else (the coded picture data we decode). enum NalKind { @@ -56,7 +63,7 @@ enum NalKind { /// `decode_frame`. Boxed so its address is a stable refcon for the session. #[derive(Default)] struct Sink { - frames: Vec, + frames: Vec<(PixelBuffer, i64)>, error: Option, } @@ -76,6 +83,11 @@ pub(crate) struct VideoToolbox { sps: Option, pps: Option, built_from: Option>, + /// Synthetic presentation time assigned to the next input sample. + sample_index: i64, + /// Original container timestamps keyed by the synthetic presentation times + /// VideoToolbox returns through the callback. + pending: VecDeque<(i64, Timestamp)>, sink: Box, } @@ -101,6 +113,8 @@ impl VideoToolbox { sps: None, pps: None, built_from: None, + sample_index: 0, + pending: VecDeque::new(), sink: Box::new(Sink::default()), })) } @@ -169,6 +183,28 @@ impl VideoToolbox { self.built_from = Some(params); Ok(true) } + + /// Pair one callback time back to the timestamp of its submitted access unit. + fn take_timestamp(&mut self, sample_time: i64) -> Result { + let found = self.pending.iter().position(|(fed, _)| *fed == sample_time); + let Some(index) = found else { + return Err(Error::Codec(anyhow::anyhow!( + "decoder output did not match fed sample time {sample_time}" + ))); + }; + Ok(self.pending.remove(index).expect("index found above").1) + } + + /// Convert every callback result collected by the last decode or drain. + fn take_frames(&mut self) -> Result, Error> { + std::mem::take(&mut self.sink.frames) + .into_iter() + .map(|(surface, sample_time)| { + let timestamp = self.take_timestamp(sample_time)?; + Ok(Frame::new(Surface::PixelBuffer(surface), timestamp)) + }) + .collect() + } } impl Backend for VideoToolbox { @@ -212,8 +248,9 @@ impl Backend for VideoToolbox { return Ok(Vec::new()); } + let sample_time = self.sample_index; let format = self.format.as_ref().expect("format ensured above"); - let sample = make_sample_buffer(&avcc, format)?; + let sample = make_sample_buffer(&avcc, format, sample_time)?; let session = self.session.as_ref().expect("session ensured above"); self.sink.frames.clear(); @@ -231,12 +268,42 @@ impl Backend for VideoToolbox { "VideoToolbox decode callback failed: {error}" ))); } - // The decode callback fires synchronously inside `decode_frame`, so - // every output frame belongs to the access unit just submitted. - Ok(std::mem::take(&mut self.sink.frames) - .into_iter() - .map(|surface| Frame::new(Surface::PixelBuffer(surface), timestamp)) - .collect()) + remember_timestamp(&mut self.pending, sample_time, timestamp); + self.sample_index += 1; + self.take_frames() + } + + fn flush(&mut self) -> Result, Error> { + self.sink.frames.clear(); + self.sink.error = None; + + let status = if let Some(session) = self.session.take() { + // This finishes delayed pictures and waits for every callback before the + // session is invalidated, so no callback can outlive `sink`. + let status = unsafe { session.wait_for_asynchronous_frames() }; + unsafe { session.invalidate() }; + status + } else { + 0 + }; + self.format = None; + self.built_from = None; + + let result = if status != 0 { + Err(Error::Codec(anyhow::anyhow!( + "VTDecompressionSessionWaitForAsynchronousFrames failed: {status}" + ))) + } else if let Some(error) = self.sink.error.take() { + Err(Error::Codec(anyhow::anyhow!( + "VideoToolbox drain callback failed: {error}" + ))) + } else { + self.take_frames() + }; + + self.sink.frames.clear(); + self.pending.clear(); + result } fn name(&self) -> &str { @@ -244,15 +311,24 @@ impl Backend for VideoToolbox { } } -/// C callback VideoToolbox invokes (synchronously, from `decode_frame`) for each -/// decoded frame. Retains the NV12 pixel buffer so the picture stays on the GPU. +/// Remember which container timestamp belongs to a submitted sample while +/// bounding streams whose pictures are repeatedly dropped. +fn remember_timestamp(pending: &mut VecDeque<(i64, Timestamp)>, sample_time: i64, timestamp: Timestamp) { + pending.push_back((sample_time, timestamp)); + while pending.len() > MAX_PENDING { + pending.pop_front(); + } +} + +/// C callback VideoToolbox invokes from decode or drain for each decoded frame. +/// Retains the NV12 pixel buffer so the picture stays on the GPU. unsafe extern "C-unwind" fn output_callback( refcon: *mut c_void, _source_frame_refcon: *mut c_void, status: i32, _flags: VTDecodeInfoFlags, image_buffer: *mut CVImageBuffer, - _pts: CMTime, + pts: CMTime, _duration: CMTime, ) { let sink = unsafe { &mut *(refcon as *mut Sink) }; @@ -263,6 +339,13 @@ unsafe extern "C-unwind" fn output_callback( let Some(image) = NonNull::new(image_buffer) else { return; // dropped frame }; + let flags = pts.flags; + let timescale = pts.timescale; + let sample_time = pts.value; + if !flags.contains(CMTimeFlags::Valid) || timescale != 1 || sample_time < 0 { + sink.error = Some("invalid presentation timestamp".to_string()); + return; + } // The decoded image buffer is a CVPixelBuffer; retain it (the callback only // borrows) and keep it as-is rather than downloading here. The retain is also @@ -273,7 +356,8 @@ unsafe extern "C-unwind" fn output_callback( let width = CVPixelBufferGetWidth(&pixel_buffer) as u32; let height = CVPixelBufferGetHeight(&pixel_buffer) as u32; - sink.frames.push(PixelBuffer::new(pixel_buffer, width, height)); + sink.frames + .push((PixelBuffer::new(pixel_buffer, width, height), sample_time)); } /// Build a `CMVideoFormatDescription` from the ordered parameter-set NAL units @@ -332,7 +416,11 @@ fn create_format_description(codec: Codec, params: &[Bytes]) -> Result Result, Error> { +fn make_sample_buffer( + avcc: &[u8], + format: &CMFormatDescription, + sample_time: i64, +) -> Result, Error> { let mut block_ptr: *mut CMBlockBuffer = ptr::null_mut(); let status = unsafe { CMBlockBuffer::create_with_memory_block( @@ -367,6 +455,11 @@ fn make_sample_buffer(avcc: &[u8], format: &CMFormatDescription) -> Result Result NalKind { Codec::Av1 => NalKind::Slice, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pending_timestamps_are_bounded() { + let mut pending = VecDeque::new(); + for index in 0..MAX_PENDING + 5 { + remember_timestamp( + &mut pending, + index as i64, + Timestamp::from_micros(index as u64).unwrap(), + ); + } + + assert_eq!(pending.len(), MAX_PENDING); + assert_eq!(pending.front().map(|(sample, _)| *sample), Some(5)); + } +} diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index bd753754752..0bbfeafc055 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -24,6 +24,10 @@ pub struct Consumer { /// One AU yields one frame in the low-delay path, but a backend may hand back /// more, so we buffer to keep `read` one-frame-per-call. pending: VecDeque, + /// Whether the ended track's decoder has already been drained. + drained: bool, + /// Last container discontinuity observed. A change starts a fresh codec epoch. + discontinuity: u64, } impl Consumer { @@ -55,6 +59,8 @@ impl Consumer { decoder, track, pending: VecDeque::new(), + drained: false, + discontinuity: 0, }) } @@ -63,15 +69,32 @@ impl Consumer { self.decoder.name() } - /// Read the next decoded I420 frame, or `None` when the track ends. + /// Read the next decoded I420 frame, or `None` after the track ends and the + /// decoder's buffered tail has been drained. pub async fn read(&mut self) -> Result, Error> { loop { if let Some(frame) = self.pending.pop_front() { return Ok(Some(frame)); } - - let Some(mux_frame) = self.track.read().await? else { + if self.drained { return Ok(None); + } + + let mux_frame = self.track.read().await?; + let discontinuity = self.track.discontinuity(); + if discontinuity != self.discontinuity { + // The tail belongs to the abandoned codec epoch. Draining resets the + // backend for reuse, but none of those pictures may cross the seam. + self.decoder.flush().await?; + self.pending.clear(); + self.discontinuity = discontinuity; + } + + let Some(mux_frame) = mux_frame else { + let tail = self.decoder.flush().await?; + self.pending.extend(tail); + self.drained = true; + continue; }; self.pending.extend( @@ -85,8 +108,12 @@ impl Consumer { #[cfg(test)] mod tests { + use bytes::Bytes; + use moq_net::Timestamp; + use super::*; use crate::decode::Kind; + use crate::decode::backend::probe; use crate::encode::{Config as EncodeConfig, Encoder, Kind as EncodeKind, Producer as EncodeProducer}; #[tokio::test] @@ -150,4 +177,163 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert_eq!(frame.size(), crate::Size::new(320, 240)); } + + /// A track ends before a decoder that reorders pictures does. The consumer + /// drains the backend once and returns its tail before reporting the end. + #[tokio::test] + async fn track_end_drains_buffered_decoder() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video", hang::container::track_info()).unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new(track, moq_mux::catalog::hang::Container::Legacy); + for index in 0..2u64 { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(index * 33_333).unwrap(), + duration: None, + payload: Bytes::from_static(b"access unit"), + keyframe: index == 0, + }) + .unwrap(); + } + producer.finish().unwrap(); + + let catalog = VideoConfig::new(hang::catalog::H264 { + inline: true, + profile: 0x42, + constraints: 0, + level: 30, + }); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "video", + Config { + kind: Kind::Named(probe::BUFFERED_NAME.into()), + ..Config::new() + }, + ) + .await + .unwrap(); + + let mut timestamps = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + timestamps.push(frame.timestamp.as_micros()); + } + assert_eq!(timestamps, vec![0, 33_333]); + assert!( + consumer.read().await.unwrap().is_none(), + "the decoder was drained twice" + ); + } + + /// A declared discontinuity abandons the previous codec epoch. A delayed + /// picture from before the seam is drained and discarded before the first new + /// keyframe is decoded. + #[tokio::test] + async fn discontinuity_discards_buffered_tail() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video", hang::container::track_info()).unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new(track, moq_mux::catalog::hang::Container::Legacy); + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(100_000).unwrap(), + duration: None, + payload: Bytes::from_static(b"old access unit"), + keyframe: true, + }) + .unwrap(); + producer.discontinuity().unwrap(); + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::ZERO, + duration: None, + payload: Bytes::from_static(b"new access unit"), + keyframe: true, + }) + .unwrap(); + producer.finish().unwrap(); + + let catalog = VideoConfig::new(hang::catalog::H264 { + inline: true, + profile: 0x42, + constraints: 0, + level: 30, + }); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "video", + Config { + kind: Kind::Named(probe::BUFFERED_NAME.into()), + ..Config::new() + }, + ) + .await + .unwrap(); + + let mut timestamps = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + timestamps.push(frame.timestamp.as_micros()); + } + assert_eq!(timestamps, vec![0]); + } + + /// Cancellation while a threaded flush is in flight leaves the sink poisoned. + /// The next read surfaces that error rather than reporting a clean end and + /// silently discarding the tail. + #[cfg(not(target_os = "macos"))] + #[tokio::test] + async fn cancelled_track_end_flush_is_not_reported_as_drained() { + probe::prepare_blocking_flush(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video", hang::container::track_info()).unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new(track, moq_mux::catalog::hang::Container::Legacy); + producer.finish().unwrap(); + + let catalog = VideoConfig::new(hang::catalog::H264 { + inline: true, + profile: 0x42, + constraints: 0, + level: 30, + }); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "video", + Config { + kind: Kind::Named(probe::BLOCKING_FLUSH_NAME.into()), + ..Config::new() + }, + ) + .await + .unwrap(); + + let mut read = Box::pin(consumer.read()); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); + loop { + tokio::select! { + _result = &mut read => panic!("flush returned before cancellation"), + _ = tokio::time::sleep(std::time::Duration::from_millis(1)) => { + if probe::flush_entered() { + break; + } + if tokio::time::Instant::now() >= deadline { + probe::release_flush(); + panic!("flush never reached the codec thread"); + } + } + } + } + drop(read); + probe::release_flush(); + + let err = match consumer.read().await { + Err(err) => err, + Ok(_) => panic!("cancelled flush must poison the sink"), + }; + assert!(err.to_string().contains("cancelled call"), "unexpected error: {err}"); + } } diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 5a10d426984..1832fe3a21f 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -181,6 +181,16 @@ impl Decoder { self.backend.decode(access_unit, timestamp, keyframe) } + + /// Return the frames the backend still holds once the stream has ended. + /// + /// Call this after the last access unit and before dropping the decoder. The + /// decoder remains reusable and waits for a keyframe before accepting the + /// next stream. + pub fn flush(&mut self) -> Result, Error> { + self.got_keyframe = false; + self.backend.flush() + } } fn is_supported_av1(av1: &AV1) -> bool { @@ -246,15 +256,16 @@ mod tests { decoded.extend(decoder.decode(encoded.payload, encoded.timestamp, keyframe).unwrap()); } } + decoded.extend(decoder.flush().unwrap()); assert!(!decoded.is_empty(), "decoder produced no frames"); for out in &decoded { assert_gray(&out.surface.to_i420().unwrap(), 320, 240); } - // The timestamp rides through the codec and comes back on each picture. These - // backends don't reorder, so it returns in feed order: strictly increasing and - // drawn from the values we fed. + // The timestamp rides through the codec and comes back on each picture, + // including any tail released by the drain. It returns in presentation order: + // strictly increasing and drawn from the values we fed. let micros: Vec = decoded.iter().map(|d| d.timestamp.as_micros()).collect(); assert!( micros.windows(2).all(|w| w[0] < w[1]), diff --git a/rs/moq-video/src/decode/sink.rs b/rs/moq-video/src/decode/sink.rs index 84cd30f6717..e51ebe466e7 100644 --- a/rs/moq-video/src/decode/sink.rs +++ b/rs/moq-video/src/decode/sink.rs @@ -39,11 +39,11 @@ use threaded::Inner; /// /// # Cancellation /// -/// Not cancel-safe, and it says so rather than letting it slide. A queued decode +/// Not cancel-safe, and it says so rather than letting it slide. A queued request /// runs whether or not anyone is still waiting, so dropping the future (racing it -/// in a `select!`, giving it a timeout) leaves the decoder a step ahead of the -/// stream, holding frames nobody received. Rather than let the next call carry on -/// against a decoder that has moved, the sink refuses every call after a +/// in a `select!`, giving it a timeout) can leave the decoder a step ahead of the +/// caller. Rather than let the next call carry on against a decoder that has moved, +/// the sink refuses every call after a /// cancelled one. Drop it and open another. /// /// macOS never refuses, because there is no thread to run ahead: the decoder runs @@ -73,6 +73,14 @@ impl Sink { pub async fn decode(&mut self, payload: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error> { self.0.decode(payload, timestamp, keyframe).await } + + /// Drain the frames the decoder still holds once the stream has ended. + /// + /// Otherwise [`Decoder::flush`](super::Decoder::flush), confined to the codec + /// thread like [`decode`](Self::decode). + pub async fn flush(&mut self) -> Result, Error> { + self.0.flush().await + } } #[cfg(not(target_os = "macos"))] @@ -96,6 +104,9 @@ mod threaded { keyframe: bool, resp: oneshot::Sender, Error>>, }, + Flush { + resp: oneshot::Sender, Error>>, + }, } /// Build a decoder and serve requests until the channel closes. Runs entirely @@ -122,6 +133,9 @@ mod threaded { } => { let _ = resp.send(decoder.decode(&payload, timestamp, keyframe)); } + Request::Flush { resp } => { + let _ = resp.send(decoder.flush()); + } } } // `decoder` drops here, on this thread, balancing the COM apartment. @@ -160,6 +174,10 @@ mod threaded { }) .await } + + pub async fn flush(&mut self) -> Result, Error> { + self.0.request(|resp| Request::Flush { resp }).await + } } } @@ -194,6 +212,10 @@ mod inline { ) -> Result, Error> { self.0.decode(&payload, timestamp, keyframe) } + + pub async fn flush(&mut self) -> Result, Error> { + self.0.flush() + } } } diff --git a/rs/moq-video/src/decode/test_data/bframes_64x64_pattern_30f.h264 b/rs/moq-video/src/decode/test_data/bframes_64x64_pattern_30f.h264 new file mode 100644 index 0000000000000000000000000000000000000000..26fae67b8a7f9ba570ebe72e05af032ae2bbe3ae GIT binary patch literal 9131 zcma)gbwJcl&^HZ&q=bZUbjQ(kl!TO$(s956M{{(SlyobNlr#bY0@7WAba!`4-gEwZ zp7-zfpS{`HnVp%>%zpQFkdTm&tt^nR%jaa7+GUZDP?6C74yWR8Qgx9KYt{!FLv-y3 z0s|cm|GFr*_j7yhO+zZjEMSiQ_{5V}kRQMaFn5GO06;+j0E~~9mlt5hCm_fp07fjl z;X(*-tE$NC|61X)cD1)d$T&K=LLD7kL;>btGq5=b2!KH>L3{uUh?$+Exh)7N z$|K4H06T!~yj&n49#1||9#0^U4`2@gSwlPlE^cNBN(kWO;)PH}j3zJ(5Ri)pp@bL! z_E1lVg~{I?fd~x~7}&uI0s;yG%&lRL_Fxl)DiGibgV@&D}pe0wg>Sb zTmV~mJ32r>yg&{h5MT**aW!#rv4uMQmH4-Tvy+LVrKJnR6~xI4aJ7aZG+aQu06RxV zTd*}^V)B1HAi%{AYL4*b{}?;~2iSi$%%S#R*S~&19b6$WJ1{~B!8fyWgMqzF%pL8W zz^;h7IRYYA7#Qk+*a9I41OJt=gn{iLE(oy9Oq{$BG}HoN+sp)P0e1Qug_(&N6zuZX z2owhSFD?%V)XLh`46)|u1aUC2a&$th{$n~JRBa(%h`m8Tf&ZmU?4b?_Sr>DN1H|0T z6~xc;*GU-oFQ71pi#0+SW^VHTg8i!qGY6T&ECKdr2vq+Dg_wYN`M7uh&VNGz@o)(u zNTV!xUq8tzzMA!m5BeMTDmq>w#a)m^EKvJ0B_yna; zu4FZmb*-&AlOKyHGp&rU++z3V8&kIN=jjr;Brf-=nXglaig!mvZ z5K@Dd)(Oi=o*lb_8jxn5M33{UFpm#GK_zc2XdJ>6B4oSX2Kmxz_`M zTb+%PeHn;~xtnNO0{0zi0s1Hm-+@#!hw)Tsk0T=1P&W9%vT_sdKlGRcUTuDDhl=9p zLW;g;skL-*1_?b69xM`V^9Z`bYA?T>q^~Uxykvdz+H(QVv24LKWhX_8_veqm>G0}d zHB;*iK~9*C$Jg4HkB8Bm8Ge(ITYoTQTog`=nqEw+>#n}Yf`Rsgkc8V(Y{W{XA2(__ ziBY?j@=vGhS;H<4-5~Ypuw=NjkkrBH=C*i8DF>vwX0noUWjHldXZ(P^6A9bz3#pL2 zY;cVI>eS;5P5iYl$-A;MDbk3dKYDQQ-`Lm+Sqln@4%P>@VNVwuRYa($W`Dn(IR1DcZg~8{`goi#!C&CL z?1$M0GtzdWxk{~8!ib*WIOi+Ttdt}}2+3~8C=8~+Zt{}gs#5q=F6b=TY z+q8 zQ|tkV@1hsxv-^r%?WOOf%f{mVi5JAU=}Qj|mAUqKjmkY{Tgmc*i?}BPer#8Iu8jLl zMDKLubFWoQ6hIB9)MFHBUY^hP&wE3T15_w1_ZhUQHlNimB=`>ls;tFZgTr){Q|eS1Gz9cFek9;8M~(6B%D#kotF07}`f=!4U%t{{ zz>q4{mVjWklHU{TjSBD4i-fXdy}95Cqb1uLD-U?(5YQlyTq zWta19c=+)Rw!EF^YQSn`=~-AG1BL68pOyHr`gh?;;*=9w++mls-_C9J3cPd)($)cm zg{H&I4f&z`40lwx^*)@y@vBP|{g9tl9ZjzeqsOI zN({GKL`P{QN#bj>ITF%<}Wu?*deLgM*c6Eg841`jj@R z6=-;O5^B<2xkxwmQ?dH%tNnx-^MuzazbgekY;;4@X)@`#`XxNF}%&86|2YY`u}b$z!l{_Nc1=yp1HEx}uifvevq<5OT0C5>PdIYFqm% ziL7&fpniKQC;jf#F00)8zF2{tS}1<++m;uHZm)JU={2i1GHYsgvSQN+(+aE zG6RHjGwAUNR|-__u1E>FIPQy>>dF~RZqkc#O+X5kE-4(e*Z5u>6VIL zrL8}!5TBVePxXZ#?W?5c%r|5WRP;Q47p&!Z0YM_g5Ms$92kBHe90+cA_HxB`!q@C8 z1`BC|lMW>)_0EsMa`s#VKoej|*MLBAyh{go*cN2Ul9^c-V!{8HT7LFvyFK4P6|ZC#GAk;Ea1`iGjT zFVBoIU37jP<`;NqC@9YJ##9}|ZG{u*md@QezHD8!eNj$y{l*wrqME@oeScZrrxe^G z*5wSA<`s4$$`xGc(RR0^U#ZTDdm_7=PX2Nwg^c+xO`(W1oerM8c0YRucF)*&#iY(4nKaTu;usx3NM#^yT?hCBo^>=ENP4bV^HRy7LlnGi|;+&bQH8gyGDvaVk zr)MtQw_waDmUHsKjqWm~pJ>)!!hhukgk~sV4pvXfbTSvKLtg&xCn!>J4O;`kL)X?k z`Pv{lS+oZv?}!v98tLCF!Fojb2jxxcGIzC-MH*vEYe3Jvv~BUchaj4N(CZ2D{!E51 zB+;ctMz_l*3ew8Wcoo-}noeQtndwNY8LvzmLuuoa!c5x(Qs3U^bQ!lj$%fa|-zLjy zK~A(2xqtfR4IPv0VrRw?qiuo zPm<7Tb>A=3%N54fH6)=3b&u`)b9tkvLh_^EeEKu=C`Jg%C6z&r&qQhC?6HY+HNsa? zxXmny(kUT#G=^7H=HbU(o=d=5Ghm0NtT}Z+DyCVg_Cw#V667e3F?{FOO3=0VQW8F( zxf?uYrV-UoBndz8nvPR!HpGfb(UA2;(To|;>NBumEc{})&q;f;E5=CQ8u<+8deywt~pUouOVSb_aIw#;>%~ z{m%2CE!a-9K{wg(>$c|yLUBnS#B);52(t-yGFjS7iq2EgOi;j`hRPj0F*^O+#^8T- z)&E~zou0(w6>y6co)pBD`uY!`gh;gqR`TU!SBhPjGfVW^^iVw>``wa8bXH93eT5ZY z*B^g3kq8gC7>WptDadzfZqqhLp|`$YRS@P(6{^BN^c}}6Lmm$8k3dpsByugd3}~To zo9GB@{|}G!;eW>-t7`1@LSTTA`E7|W^BsWnjG$$)uR0{TAmBu}g7Q^Ni^CYv#{efG zJ+$xgHcYxg{qp@^&-1IHJ8Z~3-`T)B&Eiw<>97eq3H>kH6vVYQAYlRuDW7SgfCnFv zAOTFV#yV#Gchc-nInI0PBMYN#`E$Ri1C1F*rXS|45}sMRmorwgLN)H*j>AnJ{)YQc_rqo3%^x?Bpnt%_Uj9U@W z>G{1nQ~JRM#_2t82mQT+CLl1}IsD~nH-2+G*-)id(9Prq>&&Rus7fJ{h(&NDYZb(= z#Vt-aHtUUi7pyZiu-O=O`YR>N>!(IOv_=_BEZuXpp!TOWtv8hydymyp5J=?rUv9GZ zn7r-xQ z%y_9@1(YoRfs%oTC|EnhJ*C139~mzJjpS=21PXALdG#sKMf`5frd%nTu&q;C)GMTV zznQHsb`NW1IZbXT;u$y4zoh9V0^tJY2KBw;)=q% z`h@zCme>8=$UEyVf=n7i5+evX88BKScjx*a)+KRHmvUMuu@$wOLg;T5YLeI_#2G}gV zhtsrsmCZf~?eu#^x+s`FkCRMznfXxA)w*4~#_+VrE#8;q_)d)P3&-XuW3_@^7oW*d zt|yxdhbY-w(v60^zjpt>dP$IoU?h`C9Xo{?k9?&hn-X#xb!}Sfk$fWNR zsKdCNQ2;MSr^_+@-1yYgOUTfg7k2=H6tmL%A}i%vo*vu+LzgBD>Du8ArNn+``7A?} z6WAsgN}`IY|2Fp#<*|+6C1Q<1*9a1m_-xzfTf5ILeSmNWgeL{<{R0A69 z6v}v2;@Yl0>eFi5{(PNRDPAh9-jV*LF6jt@5mC-~zPSGovUi2*FlR-Y`(E=3G#JnO zME4lfSUofeAMm-BVu?qSJ%w{_bf^2tHlgE{S4yF6bcVH8nV7Z{Xi2*@|51no5-UkJ zVcM2kX#98z5435niJ{LXJKA6vSpVoAj?*<*YE4*7u@r@ei5COe_G+eP6Viw!G=GsX zh`VKB@e@@Udberx$@s$;sb1_j-bj0(KAcsRwBG%qw8&2{k;fz zCjWCV>I@;FO~}qr3W2c!LNaKCe%<)sJOTFIcg7)px1xT*X3@;43Pa`$*@&Zy3_@ z1T^im8Y{o}k@)&J8DasMD=9m*&A7K0EC(A5cV717?PK!QuF9YiwddFxD6zIidJI7K z3)=qGlw3*-rk8r4Av$`w>ZCa5B60WgZ)4%v=g@`Hz%1KBtX%5$^NJjMHM!$MnaIn! z@GN#ZYz-{IPCT*TtdF~A&+lVc{eAm>Lw*W_6Q5o!aNe#|o7>^`){b8lc=JfeY)$+i z8-G(nC&3#ljGuc-M5E)(8apd;%SRoxWL4^DsVp{T@K?(f->V2&vFjrQ7KB*UU6 z(AO@Gi*`TrledJNQww#YwpGrxxxc)-M8C6U zz^T>kh^|$5d~2zz{x$;YZtntlxInv5QKzxyjc)sb3m%H;Fpv>F%9uscYq0uCtPr9p z5F3erI)+48s=CKz)blQCWD7o{ocRw& ze$Ud*$_R*~NxwW&)-o4#^3I3rPIapkQW?ek!XLyLWM3=|WyRmp)W${)h3p;BKxL$9 zD zo3g=iZ!FDzHotfe%s|{QY$ZwHcaBBV55hyWEXd_2meYfFJCi|vJKhxJ=R}%&jC0$Z zjT?gcesmAth!ji3>t>g)H!P&%Qjy64)r~3GNTW99#(D(Lsam5oFQQhjudZ=oWk=)R zSc^c?PX!sTSsPWP)_o_4AN*u8z_K*jCYuL4gmY_SJFRREZH8eS0M{+vG5ke1y=I5f zRlRWu4gGWM(Voi;{i3ZkU02=%Q|V2IdG=UJdlFm+CaOsBn7On6E5k>l# zM$&Kt$>Vhx>4i~8x*@P0rb?GnYGqSR=fSvNj*3-UMlKe)D8uy;?N2m8%2~e1<2@S9 z4F7^${H%TMDF8k0a%>OlrZYwte&{Vb$5nbM)yqrm>(4?isnq3^Z1rhvP)KC=|SAhHg&@s+#SG8yWwM7hfhi+^te-6oa(D2 z84HFk2Wk&It#=J5jyis*oQKkLpQMHTOvM?FsgAt|sojbRaNu0np81s>bf_*QF|QY?aVH=A;}Z+>`_X6sa5Tonpj9Wee%^kd9x;*toNc!f1wFgkEMtbUpNcRL?B1b zKxQ{7ZO>QlxumzyG%v%Nw}BK_8)S>ZY>pC3_CrKyn89JjE^q|6q*_{;M|f4}tB4<@ z8O-+IKZ*a^6eUyk6In2woZ<^A#}hujDpxfhjB3ussH>j@^)orYsb0??9o}XUXVE0s zUxau_)6A&o`;-N~@#5?rSwoUHxfj$i8P}M;YeRj{!6xN@& z_=h6#Je5}q(YDrJyPKXhKK_ZUqqSq!J2C0M83mFv&MQ1FN(LK z`IX_4t2CMHIHi92ulB9~_m%Rijb)%{sOouliT4eM&Kh!Q(8$~F-eA+g`J4)593*k- z-a$%l>a>aN%Wb6eptS3gR{A)dk`1ru30?i%wpW*5RI=V;u(VV%`*`|Tn#UGdCB%-2(iM`?9TEhQYqa3U8ox-i3vUY(PRS?x}Mk|@7VEK7#Xr+^} zYC^Pd_Vub#Xs1y!w?S;R(OpvI%;B`4i;$;3k=U9(u1{Dzw(XlbKV9= z@{nD@GSgh_H#~l!>G18MweX67Tae(-{4Q=x?u;kbr|>t#lKb%WAS%zxNd40p>bS3G znyrFcWtYO7L;5c>xSXv;9hS?J^y@#Tpg`Qpe6Blre-eMml5l@~G+o}Vj{tqxW_Mm% z0mg2Y6yA6|sb?Ga_W76*>_Pt(Vy6sy*;I}gEq@BVE#=a2qXF$TT++U;QOwZySWi4a^H~FkEfM3UeZ|%># zJsU^4WZn;Qda=BMwqb3_1ChM_8Stn4qXci$7*k>TXxCg~oBr#C!bsdj%0G@v&vnqQ z%0!+OC3hk{uxhs5yaeRXJCXUz{d2Gwaa!t8EP>y^3JHpEs+$ybuGIkk$ds5Ss({bS zJfk^t@$@E<<6|7HHk8vM&*m=h%uhDK0j0KvEly$FaPHUx;{+AxN0YL-P~v4h9|pV{ z|6dsuyKm10*Lgi9m^LEAeqQDetT^sA$+$@WJ1Qmr-bYx!+9*1sxED|7PJUcEkPI?A zQ*s(@SCltcQzG~NGz^XNEfbiHg47TiE-kc?2oTi@thI~lm&9_X2tpo=+@xwU&ieuH zdrFT8zkUYYv_-02Tbk877cB|h+)-jCL+e%KD2L@IySc4nX`QhOcoYP;O>9GK)7DwL z^H%46>zu&-PO_?teOIkIw$xbP!5Jl`&b!D(247O!l`g)3(xRrey^dco0fO3I_Rwtd zaN@bSOi`6Gif~)p0b|#7#Xr_@eB(GV7p;}_Tz$mW6h;1M!L>3#pRqxHzPbC|mV51F zBaJ;!4VD#RhLMLvRNw!_t$)fYLFn`5+j(lVNdf`850bvWG@eOE@?|WMt3M^om~{NZ zXXsxeObA&3T{(ziwv62y4V2DTY(7?{{DvIEp`Joz0R!>%A56ev{Q}@nf z_5geVJMy(T7rv2!2lG+X!>YbrwIylHsa(S6eAULK$osini;64J%VsQ&pgr)+kWfGn z_fQvn3;pt2pe}SLu6~K}T)3{@qh#inzF@?BLxVWqTwE4#9T>#Fj zv7P)((HCWuG#9sK4Y*j!A*#`tbtk1v*jhA=#Y0Gkuahl-Zx(p>3t2W#nURytV=P+K=KUgB9zW&lzp+ijqDFK30H83Gpd0u4}Z5=im$l||#WZpkWfxNsQL0Hf4LLRk-<*JC0|@mgft8vm>hIFD&`~ zDlpvJ}|Aj4R7?2kW|Lvx8*bqfh%B~REglA z%6Pv(QN31sJB1LjmAENMb0R^xJon%OmlN*+K@?GWW_-Kz@&9l~^laAfnKV&I;ii{G zxPEsUb@)wg&NE2A9B3X}h&%dex^lrb<4<0J3IfJ`mElTgs6lYwh^zahnR^x%_5rDn zkt&KTl-W^&@k`Vvi7r-aM>-=kH>uo~>*^avoP4537Ks)~xecpAIJ>nE0vCQSMQ`Tn z@CP66c|!533b-P;Fux!F36_NK`+xh~xZ(dKre{*A%15unkH^o89#8|FEm=>d80Y!J zHc;q5AYhKk`svi@lMXUVcBbZgXk>~PM}?(t5D``4{-;Z7R+%*}Eep&~n&V+T^}b?h S*8`94hf!acRGdZjPW}hBuR6H^ literal 0 HcmV?d00001 diff --git a/rs/moq-video/src/v4l2.rs b/rs/moq-video/src/v4l2.rs index dbf0bf3f29f..27044c035f2 100644 --- a/rs/moq-video/src/v4l2.rs +++ b/rs/moq-video/src/v4l2.rs @@ -33,8 +33,8 @@ use v4l::v4l_sys::{ V4L2_CAP_VIDEO_M2M_MPLANE, V4L2_EVENT_SOURCE_CHANGE, V4L2_SEL_TGT_COMPOSE, timeval, v4l2_buf_type_V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, v4l2_buf_type_V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, v4l2_buffer, v4l2_capability, v4l2_colorspace_V4L2_COLORSPACE_REC709, v4l2_colorspace_V4L2_COLORSPACE_SMPTE170M, v4l2_control, - v4l2_encoder_cmd, v4l2_event, v4l2_event_subscription, v4l2_field_V4L2_FIELD_NONE, v4l2_fmtdesc, v4l2_format, - v4l2_memory_V4L2_MEMORY_MMAP, v4l2_plane, v4l2_quantization_V4L2_QUANTIZATION_FULL_RANGE, + v4l2_decoder_cmd, v4l2_encoder_cmd, v4l2_event, v4l2_event_subscription, v4l2_field_V4L2_FIELD_NONE, v4l2_fmtdesc, + v4l2_format, v4l2_memory_V4L2_MEMORY_MMAP, v4l2_plane, v4l2_quantization_V4L2_QUANTIZATION_FULL_RANGE, v4l2_quantization_V4L2_QUANTIZATION_LIM_RANGE, v4l2_requestbuffers, v4l2_selection, v4l2_streamparm, v4l2_xfer_func_V4L2_XFER_FUNC_709, v4l2_ycbcr_encoding_V4L2_YCBCR_ENC_601, v4l2_ycbcr_encoding_V4L2_YCBCR_ENC_709, }; @@ -72,11 +72,11 @@ pub(crate) const RAW: &[u32] = &[NV12, NV12M, YUV420, YUV420M]; /// Planes a format may use here: Y, U, and V. const MAX_PLANES: usize = 3; -/// The three requests the `v4l` crate does not export, built the way +/// Requests the `v4l` crate does not export, built the way /// `linux/ioctl.h` builds them. The shifts are `asm-generic`'s, which is what /// `v4l`'s own table already assumes. mod request { - use v4l::v4l_sys::{v4l2_event, v4l2_event_subscription, v4l2_selection}; + use v4l::v4l_sys::{v4l2_decoder_cmd, v4l2_event, v4l2_event_subscription, v4l2_selection}; use v4l::v4l2::vidioc::_IOC_TYPE; const READ: u32 = 2; @@ -89,6 +89,7 @@ mod request { pub(super) const DQEVENT: _IOC_TYPE = code(READ, 89, size_of::()); pub(super) const SUBSCRIBE_EVENT: _IOC_TYPE = code(WRITE, 90, size_of::()); pub(super) const G_SELECTION: _IOC_TYPE = code(READ | WRITE, 94, size_of::()); + pub(super) const DECODER_CMD: _IOC_TYPE = code(READ | WRITE, 96, size_of::()); #[cfg(test)] mod tests { @@ -97,7 +98,7 @@ mod request { use super::*; - /// The three codes above have to come out the way `linux/ioctl.h` builds + /// The codes above have to come out the way `linux/ioctl.h` builds /// the rest, and nothing else here would notice if they did not: a wrong /// code is `ENOTTY` at runtime on whichever board reaches it first. /// @@ -109,6 +110,9 @@ mod request { assert_eq!(code(READ, 0, size_of::()), vidioc::VIDIOC_QUERYCAP); assert_eq!(code(WRITE, 18, size_of::()), vidioc::VIDIOC_STREAMON); assert_eq!(code(READ | WRITE, 5, size_of::()), vidioc::VIDIOC_S_FMT); + // `v4l` 0.14 omits this newer request. This is the value from + // `videodev2.h`, and also checks the generated command struct's ABI size. + assert_eq!(DECODER_CMD, 0xc048_5660); } } } @@ -132,6 +136,7 @@ unsafe trait Arg: Sized { // same, with no reference, no enum, and no niche. unsafe impl Arg for v4l2_buffer {} unsafe impl Arg for v4l2_capability {} +unsafe impl Arg for v4l2_decoder_cmd {} unsafe impl Arg for v4l2_encoder_cmd {} unsafe impl Arg for v4l2_event {} unsafe impl Arg for v4l2_event_subscription {} @@ -567,6 +572,17 @@ impl Device { .map_err(|err| self.err(format_args!("ENCODER_CMD {cmd}"), err)) } + /// Issue a `VIDIOC_DECODER_CMD`, which drains or resumes a stateful decoder. + pub(crate) fn decoder_cmd(&self, cmd: u32) -> Result<(), Error> { + // The stateful drain sequence wants `flags` and `pts` zero, which is where + // every request starts. + let mut command = v4l2_decoder_cmd::zeroed(); + command.cmd = cmd; + // SAFETY: `VIDIOC_DECODER_CMD` takes a `v4l2_decoder_cmd`. + unsafe { self.ioctl(request::DECODER_CMD, &mut command) } + .map_err(|err| self.err(format_args!("DECODER_CMD {cmd}"), err)) + } + /// Set a codec control, failing if the driver rejects it. pub(crate) fn set_control(&self, id: u32, value: i32) -> Result<(), Error> { let mut control = v4l2_control { id, value }; @@ -874,11 +890,26 @@ impl Queue { Ok(()) } + /// Stop and restart a queue, taking every buffer back from the driver. + pub(crate) fn restart(&mut self, device: &Device) -> Result<(), Error> { + if self.streaming { + device.stream(self.dir, false)?; + self.streaming = false; + } + self.free = (0..self.buffers.len() as u32).collect(); + self.stream_on(device) + } + /// Whether the queue has been started. pub(crate) fn streaming(&self) -> bool { self.streaming } + /// Buffers currently owned by the driver. + pub(crate) fn outstanding(&self) -> usize { + self.buffers.len() - self.free.len() + } + /// Stop the queue and hand its buffers back to the driver, which is how a /// decoder re-negotiates CAPTURE when the stream changes size. /// From f5e4ae1e615a178940ddb4be222b599405c2fdfb Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Sat, 5 Sep 2026 03:00:00 +0200 Subject: [PATCH 12/29] feat(moq-audio)!: let publish_capture carry a catalog extension (#3356) --- rs/moq-audio/src/encode/capture.rs | 46 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index e8ff569761d..27d526200ab 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -7,6 +7,8 @@ use std::time::Duration; use rand::RngExt; +use moq_mux::catalog::hang::CatalogExt; + use super::producer::Reserved; use super::{Input, Options, Producer}; use crate::capture; @@ -191,20 +193,20 @@ impl Publication { /// rendition on the first success. Until then the broadcast advertises no /// audio, which [`Status::Starting`] reports. A source that never appears is /// retried with capped backoff rather than blocking the controls. - pub fn new( + pub fn new( broadcast: moq_net::broadcast::Producer, - catalog: moq_mux::catalog::Producer, + catalog: moq_mux::catalog::Producer, options: PublicationOptions, - ) -> Result<(Self, Driver), Error> { + ) -> Result<(Self, Driver), Error> { Self::build(broadcast, catalog, options, Supervisor::default()) } - fn build( + fn build( mut broadcast: moq_net::broadcast::Producer, - catalog: moq_mux::catalog::Producer, + catalog: moq_mux::catalog::Producer, options: PublicationOptions, supervisor: Supervisor, - ) -> Result<(Self, Driver), Error> { + ) -> Result<(Self, Driver), Error> { let reserved = Reserved::new(&mut broadcast, catalog, &options.encode)?; let track_name: Arc = reserved.name().into(); let desired = Desired { @@ -330,9 +332,9 @@ impl Publication { /// The driver owns the broadcast producer so its identity remains alive through /// stop, failure, replacement, and restart. Dropping the final [`Publication`] /// ends the driver and releases that identity. -pub struct Driver { +pub struct Driver { _broadcast: moq_net::broadcast::Producer, - track: Option, + track: Option>, /// The codec settings the rendition is built from once the layout is known. encode: Options, clock: moq_mux::Clock, @@ -346,13 +348,13 @@ pub struct Driver { park_on_failure: bool, } -impl fmt::Debug for Driver { +impl fmt::Debug for Driver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Driver").finish_non_exhaustive() } } -impl Driver { +impl Driver { /// Run capture until the final control handle drops or the MoQ track ends. pub async fn run(self) -> Result<(), Error> { self.run_with(DeviceSource).await @@ -562,7 +564,7 @@ impl Driver { } } - fn producer_mut(&mut self) -> &mut Producer { + fn producer_mut(&mut self) -> &mut Producer { self.track .as_mut() .and_then(Track::producer) @@ -602,15 +604,15 @@ enum DriveEvent { /// One per publication, moved twice in its life, so the size difference between /// the variants never costs a copy worth an allocation to avoid. #[allow(clippy::large_enum_variant)] -enum Track { +enum Track { /// Registered in the broadcast but not the catalog, because the rendition /// describes a PCM layout no probe has returned yet. - Reserved(Reserved), + Reserved(Reserved), /// Probed, so the rendition is registered and samples can be encoded. - Encoding(Producer), + Encoding(Producer), } -impl Track { +impl Track { fn track(&self) -> &moq_net::track::Producer { match self { Self::Reserved(reserved) => reserved.track(), @@ -618,7 +620,7 @@ impl Track { } } - fn producer(&mut self) -> Option<&mut Producer> { + fn producer(&mut self) -> Option<&mut Producer> { match self { Self::Reserved(_) => None, Self::Encoding(producer) => Some(producer), @@ -692,9 +694,9 @@ fn publish_state( /// /// This convenience function runs a controllable [`Publication`] with its /// initial source. Use [`Publication::new`] directly to retain controls. -pub async fn publish_capture( +pub async fn publish_capture( broadcast: moq_net::broadcast::Producer, - catalog: moq_mux::catalog::Producer, + catalog: moq_mux::catalog::Producer, capture: capture::Config, encode: Options, clock: moq_mux::Clock, @@ -793,8 +795,8 @@ trait Output { fn write(&mut self, samples: capture::Samples, timestamp_us: u64) -> Result<(), Error>; } -struct EncoderOutput<'a> { - producer: &'a mut Producer, +struct EncoderOutput<'a, E: CatalogExt> { + producer: &'a mut Producer, clock: &'a moq_mux::Clock, /// The source the supervisor was started with, so a published state can /// never name an input this stream isn't actually reading. @@ -804,7 +806,7 @@ struct EncoderOutput<'a> { device: Option, } -impl Output for EncoderOutput<'_> { +impl Output for EncoderOutput<'_, E> { fn waiting(&mut self) { self.device = None; self.silence(); @@ -847,7 +849,7 @@ impl Output for EncoderOutput<'_> { } } -impl EncoderOutput<'_> { +impl EncoderOutput<'_, E> { /// Levels ride their own channel: they change every buffer, so folding them /// into [`State`] would wake every [`Publication::changed`] waiter at the /// capture rate and rebuild the whole snapshot to do it. From 03db0e05a217497da028e72437a407e7366025c9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:16:18 -0700 Subject: [PATCH 13/29] fix(claude): let spawn-quest reclaim an abandoned quest branch (#3409) Co-authored-by: Claude Opus 5 --- .claude/skills/spawn-quest/SKILL.md | 32 +++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/.claude/skills/spawn-quest/SKILL.md b/.claude/skills/spawn-quest/SKILL.md index f7a52c735ef..42fc670b85b 100644 --- a/.claude/skills/spawn-quest/SKILL.md +++ b/.claude/skills/spawn-quest/SKILL.md @@ -19,9 +19,30 @@ the ones a recent branch or open PR already claims. Resolve a stale branch (old, no open PR) before keeping its quest in the start pool, rather than only mentioning it. Left in place it makes the agent's claim push fail as non-fast-forward, which the agent then reports as a lost race that -never happened. Inspect its tip: reuse the work, or delete the ref only when it -is confirmed stale, is not checked out, and has no unmerged commits. Otherwise -treat the quest as claimed and take it out of the pool. +never happened. + +Inspect what the branch carries beyond its base. A claim placeholder is itself +a commit, so "carries no work" is not the same as "has nothing unmerged", and +conflating the two strands a quest forever: + +- **Placeholder commits only.** Someone claimed the quest and abandoned it + without starting. Delete the ref and keep the quest in the pool: nothing is + lost, and treating it as claimed would retire the quest permanently on the + strength of an empty commit. Pin the remote delete to the tip you inspected, + with `git push --force-with-lease=: origin --delete `. + An unconditional delete races the owner coming back, and would erase a real + commit pushed between the inspection and the delete along with the claim it + renewed. A rejected lease means that is what happened, so treat the quest as + claimed. Delete the local ref too when it points at that same tip, or the + agent cannot cut the branch it was told to cut. +- **Real commits.** Do not cut a fresh branch over them. Hand the agent the + existing branch to continue, rebased onto its base. It still claims, with a + fresh UUID placeholder pushed under a lease pinned to the tip you inspected. + Two runs that both adopt the same branch would otherwise both skip the claim, + and a rebase that changes nothing gives neither of them the rejected push + that decides the race. +- **Checked out in a worktree, or carrying an open PR.** Treat the quest as + claimed and take it out of the pool. Put every quest to the user in priority order - the depth-first walk of the scope's `Quests` lists - with a recommendation and the one fact behind it. @@ -56,7 +77,10 @@ itself. Instruct it to: - Cut the quest branch (the quest path without `.md`) from that base and claim it with an empty placeholder commit whose message contains a freshly generated UUID, pushed immediately with `git push origin HEAD` after - pointing the upstream at the base. The UUID is what makes the claim a race: + pointing the upstream at the base. When triage hands over an existing branch to + continue, rebase it onto the base and claim it the same way, but push the + placeholder with `--force-with-lease` pinned to the tip triage inspected, so + a second adopter loses the race rather than joining it. The UUID is what makes the claim a race: two agents claiming the same quest from the same base in the same second otherwise produce the same commit object, and the loser's push succeeds as already-up-to-date instead of being rejected. A rejected push lost the race: From 89cf5df0ff67c726dcc4cee4847dad50a690bfc8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:18:03 -0700 Subject: [PATCH 14/29] chore(quest): settle the group-overflow abort plan, and split out #2999 (#3407) Co-authored-by: Claude Opus 5 --- quest/m1/README.md | 1 + quest/m1/group-overflow-abort.md | 141 +++++++++++++++++++++----- quest/m1/js-net-stream-error-codes.md | 59 +++++++++++ 3 files changed, 173 insertions(+), 28 deletions(-) create mode 100644 quest/m1/js-net-stream-error-codes.md diff --git a/quest/m1/README.md b/quest/m1/README.md index e3dd9ab333d..cf030cf0940 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -46,6 +46,7 @@ with the current dev tree before starting. - [#2709](/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md) - js/net mirrors the send-side bandwidth allocator so each publisher encodes against its own share - [#3000](/quest/m1/3000-track-teardown-on-poll-unused-is-not-atomic-against-a.md) - Track teardown on poll_unused is not atomic against a consumer reattaching - [#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md) - IETF stream resets send moq-lite error codes, so routine events read as INTERNAL_ERROR +- [JS stream codes](/quest/m1/js-net-stream-error-codes.md) - js/net: a locally raised group error reaches the wire as INTERNAL_ERROR - [#3002](/quest/m1/3002-no-test-drives-a-late-group-through-the-ietf-dispatch-loop.md) - No test drives a late group through the IETF dispatch loop - [#3187](/quest/m1/3187-preserve-structured-protocol-error-codes-across-ffi-and-c.md) - Preserve structured protocol error codes across FFI and C bindings - [#2318](/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md) - js/net: remaining capability gaps vs rs/moq-net (SETUP role, finish_at and final sequence, range controls, typed errors) diff --git a/quest/m1/group-overflow-abort.md b/quest/m1/group-overflow-abort.md index 7ba84a18efb..368a4234909 100644 --- a/quest/m1/group-overflow-abort.md +++ b/quest/m1/group-overflow-abort.md @@ -1,4 +1,4 @@ -# [M] Abort an oversized open group instead of shedding its head +# [L] Abort an oversized open group instead of shedding its head ## Goal @@ -6,8 +6,19 @@ An open group that outgrows its cache budget errors for every reader instead of evicting frames from its front. One consistent failure replaces today's split, where a reader that kept up streams the whole group while a late or new one gets `Lagged`, and the head-shedding machinery becomes dead code to -delete. Decide first: this is a semantics change to the model in both -languages, not a bug fix. +delete. The writer learns about its own overrun instead of being told nothing. + +A remote peer can tell the overrun apart from its own lag when the publisher is +Rust. From a JS publisher it still arrives as `Internal` until +[JS stream codes](/quest/m1/js-net-stream-error-codes.md) lands, because +`Writer.reset` cannot put a code on the wire for a locally raised error at all. +That is a gap in js/net rather than in this change, so it does not block this +quest; it does bound what this quest can promise, so it is stated here rather +than left to be discovered. + +This is a semantics change to the model in both languages, not a bug fix. It +was settled in a planning pass; the decisions are recorded below rather than +left open. ## Plan @@ -15,32 +26,106 @@ Today `rs/moq-net` (`model/group.rs`, `MAX_CACHE_BYTES`) and `js/net` (`group.ts`, `MAX_GROUP_CACHE_BYTES` plus `MAX_GROUP_FRAMES`) evict from the front of an open group once it passes the cap, and a reader positioned below the eviction fails with `Lagged`. Readers at or above it keep going, which the -IETF publishers use to serve a draft-20 filter whose range excludes the -evicted prefix (`group::Consumer::skip_to` in Rust, `Group.ReadOptions.from` -in JS). - -Aborting the group at the cap instead: - -- keeps the memory bound, and the group also stops growing, -- gives every reader the same terminal error at the same point, instead of - punishing only whoever subscribed late, -- deletes the per-reader eviction floors and the eviction bookkeeping - (`offset` in Rust, `start`/`evicted` in JS) outright, -- costs little in practice: a group near 32 MiB is pathological (hang groups - are GOP-sized), and shedding already denies the whole group to every new - subscriber anyway. - -Open questions to settle before implementing: - -- The abort error code: `Lagged` blames the reader for the writer's overrun; - the existing oversized-single-frame rejection (a write error) is the closer - precedent, and the writer should learn about it too. -- Whether the JS frame-count cap (1024) aborts as well or simply goes away; - Rust has only the byte budget. -- Pool-level eviction of whole idle groups is untouched; only per-group - front-shedding changes. +draft-20 IETF publisher path uses to serve a filter whose range excludes the +evicted prefix. + +### Decided + +- **A new error variant**, `Error::GroupTooLarge`, mirroring the existing + `FrameTooLarge` naming. `Lagged` is named from the consumer's side and would + keep blaming the reader for the writer's overrun; `Evicted` already means the + pool dropped a whole group under external memory pressure, and + [#3161](/quest/m1/3161-retention-should-reclaim-idle-open-groups-now-that-expiry.md) + will abort idle open groups, so these three need to stay distinguishable. +- **The writer learns synchronously.** The write that pushes the group past its + budget returns `Err(Error::GroupTooLarge)` and aborts the group, which is the + shape `FrameTooLarge` already has in `write_frame`. Today `evict()` returns + `()` and every write path returns `Ok(())` regardless, so the producer is + told nothing. +- **A new stream error code** beside `TooFarBehind` (`0x5`) and + `FrameTooLarge` (`0x25`), so a remote subscriber can tell the failure apart. + That is a wire change, so `drafts/draft-lcurley-moq-lite.md` is updated in + the same PR. `Error::to_code` has a stability test (`to_code_is_stable`) that + pins the local codes; add the new one there. +- **A frame-count cap in both languages, at 8192.** JS caps at 1024 today and + Rust has no count cap at all, so a JS publisher dies where an identical Rust + one holds 100,000 frames. 8192 gives JS eight times its current headroom and + closes the divergence. The Rust test `no_eviction_under_budget` writes + exactly 100,000 one-byte frames to assert there is no count cap; it is + rewritten rather than deleted, since the byte-budget half of what it proves + still holds. +- **The IETF wire keeps its own mapping.** Sending the new code on the + moq-transport wire belongs to + [#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md), + which is fixing the whole registry confusion rather than one variant. + +### What actually gets deleted + +Less than the original sketch claimed, so budget for it: + +- Rust: the `offset += 1` inside `evict()` and the `evict()` calls in + `write_frame`, `write_frames`, `create_frame`, and `create_frame_owned`. + `GroupState::offset` itself **stays**: it is also the `Producer::start_at` + floor, read through `live_first_frame()` by `track.rs` (`covering_group`, + `claim_sequence`) and by `resume.rs` route splicing. The two `Error::Lagged` + returns guarding `index < offset` stay for the same reason. +- JS: `state.evicted` and the eviction loop in `appendFrame`. `state.start` + **stays**: `#readBufferedFrame` increments it on every read, so it is the + running sequence counter, not an eviction floor. +- `group::Consumer::skip_to` and `Group.ReadOptions.from` **stay**. They are + doing range work, not eviction work: a draft-20 filter still has to begin at + `slice.skip` even when nothing was evicted. Only their eviction tolerance + goes, which is the clamping difference against `start_at`. Deleting the + shared cursor would push a drain loop into every publisher instead, and that + duplication is exactly what `write_fill_group` already drifted into once, as + the next section documents. `Consumer.skipped` in JS and the `js/binary` + guard on it do go, since those report eviction and nothing else. + +### Fold in: the reverted skip_to + +`f6376ed32` (#3323) added `skip_to` at two sites in +`rs/moq-net/src/ietf/publisher.rs` plus a regression test +(`skip_to_tolerates_an_eviction_below_it`). The merge commit `6947217fc` +("Merge main into dev") silently dropped one call site and the test, so +`write_fill_group` is back to the pre-fix drain-below-`skip` form and still +fails with `Lagged` on an eviction confined below the filter start. This quest +deletes that case outright, so restore nothing: instead confirm the fill path +ends up correct under the new semantics, and say in the PR that the reverted +fix was superseded rather than lost a second time. JS kept both call sites +(`ietf/publisher.ts` `#runGroup` and `#runFill`). + +### Coverage + +Rust `group.rs` tests to update: `eviction_drops_old_frames`, +`next_frame_returns_cache_full_on_tombstone`, `no_eviction_under_budget`. +Leave the `start_at` tests alone, since that floor survives. JS `group.test.ts` +tests to update: the two cap tests, `"a caught-up reader does not trip the byte +cache cap"`, `"reading a group whose frames were evicted throws Lagged"`, and +`"a read that starts above the eviction window skips the gap instead of +throwing"`. `js/net/src/broadcast.test.ts` and +`js/net/src/ietf/publisher.test.ts:886` also lean on eviction. Add a test that +the writer sees `GroupTooLarge`, which nothing covers today. + +Note `js/json/src/window/encoder.ts` keeps its own unrelated +`MAX_GROUP_FRAMES = 256`; leave it. + +### The benchmark breaks + +`rs/moq-net/benches/group.rs` sweeps `COUNTS = [512, 8_192, 32_768]` and +unwraps every write (`write_frames(..).unwrap()` and the prefill paths), so the +32,768 case panics the moment a Rust count cap exists and `just bench` fails. +Adjust the sweep, or benchmark the rejection deliberately, as part of this +quest rather than discovering it afterwards. + +The middle case sits exactly on the proposed cap, so settle the boundary and +say it in the doc comment: 8192 frames is the largest legal group, and the +8193rd write is the one that returns `GroupTooLarge`. The bench's own comment +already claims its top end "intentionally reaches the raised +`MAX_GROUP_FRAMES`", which is stale: Rust has no such constant today. ## Related - [Group charge](/quest/m0/group-charge.md) - pool-level budget accounting, unaffected by this change -- [#3161](/quest/m1/3161-retention-should-reclaim-idle-open-groups-now-that-expiry.md) - open-group lifecycle work in the same area +- [#3161](/quest/m1/3161-retention-should-reclaim-idle-open-groups-now-that-expiry.md) - also turns "open group hit a limit" into an abort, so the two must not collide on the error variant +- [JS stream codes](/quest/m1/js-net-stream-error-codes.md) - without it a JS publisher sends this new code to the wire as Internal +- [#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md) - the moq-transport half of the code mapping diff --git a/quest/m1/js-net-stream-error-codes.md b/quest/m1/js-net-stream-error-codes.md new file mode 100644 index 00000000000..e55c63a1438 --- /dev/null +++ b/quest/m1/js-net-stream-error-codes.md @@ -0,0 +1,59 @@ +# [M] js/net: a locally raised group error reaches the wire as INTERNAL_ERROR + +## Goal + +A JS publisher that resets a stream because of a group error sends the code for +that error, the same as the Rust publisher does. Today every locally minted +error arrives at the peer as `0`, which the moq-lite space reads as an internal +failure, so a routine event looks like a crash on our side. + +## Plan + +Dev only, which is why this sits in m1 rather than m0. `main` has no +`StreamCode`, no `StreamError`, and no `toTransport`: its `reset(reason)` just +calls `#writer.abort(reason)`, so there is no code-preserving path to fix +there. The structured code machinery this quest builds on exists only on dev, +so branch from dev and reconcile against that tree. + +`Writer.reset` calls `withCode(reason)` in `js/net/src/stream.ts`, which +preserves a code only when `reason instanceof StreamError`. A `Lagged` raised +locally is a plain `Error`, so `js/net/src/ietf/publisher.ts` `stream.reset(error(err))` +sends `StreamCode.Internal`. Rust does the opposite and deliberately so: +`lite/publisher.rs` aborts the stream with the real reason precisely to stop +`Writer`'s `Drop` fallback reporting every failure as `Cancel`, and maps +`Error::Lagged` to `StreamError::TooFarBehind` (`0x5`). + +The asymmetry is already load-bearing downstream. `js/json/src/window/consumer.ts` +accepts *both* the local class and the remote codes to work around it: + +```ts +const lagged = + err instanceof Moq.Group.Lagged || + (err instanceof Moq.StreamError && + (err.code === Moq.StreamCode.TooFarBehind || ...)); +``` + +So the fix is a mapping from js/net's local error classes to stream codes, +mirroring Rust's `Error::to_code` / `Error::from_transport` pair, applied where +a stream is reset rather than at each call site. It is not a matter of passing +a literal at the one `ietf/publisher.ts` site: `Lagged` is not the only local +class that reaches `reset`, and the reverse direction has to agree or two peers +disagree about what a code means. + +Scope is the moq-lite code space, matching what Rust sends today. The +moq-transport registry is a separate confusion tracked by +[#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md), +which is fixing both directions there; do not fold the two together. + +Land it with a test that a JS publisher's reset arrives at a peer as the right +code, and drop the `js/json` workaround once the class check is redundant. + +## Closes + +- [#2999](https://github.com/moq-dev/moq/issues/2999) - close this issue when the quest finishes + +## Related + +- [Group overflow](/quest/m1/group-overflow-abort.md) - adds a new stream code that a JS publisher cannot currently send +- [#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md) - the moq-transport half +- [#3187](/quest/m1/3187-preserve-structured-protocol-error-codes-across-ffi-and-c.md) - the same structured-code loss across FFI and C From 5a5ba2a85e7c6fa328656611f0f403e6c7d84403 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:19:24 -0700 Subject: [PATCH 15/29] docs(quest): audit the tree, and settle rewind, HLS identity, metadata sidecars, and clock sync (#3416) Co-authored-by: Claude Fable 5.1 --- quest/README.md | 10 +- ...c-abi-catch-up-with-the-moq-ffi-surface.md | 8 +- ...the-quiche-backend-to-quinn-noq-feature.md | 4 + ...apability-gaps-vs-rs-moq-net-setup-role.md | 8 +- ...ast-bandwidth-estimates-and-reservation.md | 2 +- ...path-predicate-for-originconsumer-0-1-x.md | 78 ------------ ...nned-source-port-can-dial-only-a-broken.md | 4 + ...reach-encoder-rate-control-so-every-non.md | 43 ------- ...te-round-trip-erases-a-routes-cold-cost.md | 65 ---------- ...o-moq-token-via-a-usage-executable-view.md | 4 +- ...y-lists-lose-to-the-environment-and-the.md | 60 ---------- ...coder-captures-the-rewind-generation-at.md | 24 +++- ...-group-delivery-order-a-handle-and-move.md | 46 ------- ...us-frame-durations-work-across-bindings.md | 6 +- ...li-defaults-from-clobbering-toml-values.md | 71 ----------- quest/m1/README.md | 13 +- quest/m1/config-provenance.md | 60 ++++++++++ quest/m1/ladder/README.md | 2 +- quest/m1/monotonic-timeline.md | 70 +++++++++++ quest/m1/plan-binding-rate-control.md | 42 +++++++ quest/m1/plan-route-cold-cost.md | 43 +++++++ quest/m1/uring-qlog.md | 8 +- ...l-clock-latency-target-for-synchronized.md | 103 ++++------------ ...te-35-ad-cue-signaling-carried-opaquely.md | 92 ++++++-------- ...-the-datagram-path-append-datagram-recv.md | 38 ------ ...rack-dynamic-so-a-browser-publisher-can.md | 35 ------ ...ser-through-moq-ffi-uniffi-instead-of-a.md | 15 +++ ...generated-media-timelines-to-wall-clock.md | 4 + ...a-revalidation-re-check-cannot-update-a.md | 37 ------ ...e-heap-activity-and-blocking-locks-from.md | 50 -------- ...d-and-coalesce-playback-driver-commands.md | 51 -------- ...lts-to-caller-constructed-configuration.md | 54 --------- quest/m2/README.md | 18 ++- quest/m2/av1-metadata.md | 9 +- quest/m2/captions-import.md | 4 - quest/m2/cli-packaging.md | 4 - quest/m2/drain/client-goaway.md | 6 + quest/m2/emsg.md | 8 +- quest/m2/flv-script.md | 6 +- quest/m2/hls-generation.md | 85 ++++++------- quest/m2/id3.md | 13 +- quest/m2/path-patterns/origin.md | 17 +++ quest/m2/plan-hls-identity.md | 56 --------- quest/m2/plan-revalidation-updates.md | 34 ++++++ quest/m2/pop-skipping/README.md | 2 +- quest/m2/qos/starvation.md | 5 - quest/m2/relay-memory.md | 37 ++++++ quest/m2/relay-memory/README.md | 112 ------------------ quest/m2/room-sdk.md | 8 ++ quest/m2/{relay-memory => }/route-gauge.md | 9 +- quest/m2/sei/README.md | 7 +- quest/m2/sei/sei.md | 25 ++-- quest/m2/timeline-wall.md | 43 +++++++ quest/m2/wildcard/README.md | 14 +-- quest/m3/697-conferencing-demo.md | 22 ---- quest/m3/README.md | 1 - quest/m3/multipath-spike.md | 15 +-- quest/m3/teleop/README.md | 4 +- quest/m3/teleop/correlation.md | 29 +++-- quest/m3/teleop/v4l2-encode.md | 2 +- quest/m3/teleop/vaapi.md | 18 --- quest/m3/video-hardware.md | 6 +- 62 files changed, 625 insertions(+), 1144 deletions(-) delete mode 100644 quest/m1/2714-per-subscriber-path-predicate-for-originconsumer-0-1-x.md delete mode 100644 quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md delete mode 100644 quest/m1/2933-moq-ffi-moqroute-round-trip-erases-a-routes-cold-cost.md delete mode 100644 quest/m1/3051-config-merge-empty-lists-lose-to-the-environment-and-the.md delete mode 100644 quest/m1/3086-refactor-net-make-group-delivery-order-a-handle-and-move.md delete mode 100644 quest/m1/3221-config-stop-cli-defaults-from-clobbering-toml-values.md create mode 100644 quest/m1/config-provenance.md create mode 100644 quest/m1/monotonic-timeline.md create mode 100644 quest/m1/plan-binding-rate-control.md create mode 100644 quest/m1/plan-route-cold-cost.md delete mode 100644 quest/m2/2822-moq-wasm-bind-the-datagram-path-append-datagram-recv.md delete mode 100644 quest/m2/2835-moq-wasm-bind-track-dynamic-so-a-browser-publisher-can.md delete mode 100644 quest/m2/3058-moq-relay-a-revalidation-re-check-cannot-update-a.md delete mode 100644 quest/m2/3164-moq-audio-remove-heap-activity-and-blocking-locks-from.md delete mode 100644 quest/m2/3165-moq-audio-bound-and-coalesce-playback-driver-commands.md delete mode 100644 quest/m2/3189-add-uniffi-defaults-to-caller-constructed-configuration.md delete mode 100644 quest/m2/plan-hls-identity.md create mode 100644 quest/m2/plan-revalidation-updates.md create mode 100644 quest/m2/relay-memory.md delete mode 100644 quest/m2/relay-memory/README.md rename quest/m2/{relay-memory => }/route-gauge.md (62%) create mode 100644 quest/m2/timeline-wall.md delete mode 100644 quest/m3/697-conferencing-demo.md delete mode 100644 quest/m3/teleop/vaapi.md diff --git a/quest/README.md b/quest/README.md index 9b2b0c637de..17925a4bc3e 100644 --- a/quest/README.md +++ b/quest/README.md @@ -8,10 +8,12 @@ grouped into milestones ordered by priority. ## Plan Milestones sort by kind, which tracks urgency here: m0 fixes what is broken, -m1 is the dev branch line, m2 grows the surface, m3 explores. Every open -GitHub issue that survived the 2026-08 grooming is represented by a quest, and -the upstream-facing questlines from the downstream moq.pro tree were migrated -in. New work joins the milestone matching its kind, at its priority rank. +m1 is the dev branch line, m2 grows the surface, m3 explores. The 2026-08 +grooming turned every surviving GitHub issue into a quest and migrated the +upstream-facing questlines from the downstream moq.pro tree; issues opened +since are imported in periodic grooming passes, and an issue already fixed on +`dev` stays open until `dev` merges. New work joins the milestone matching its +kind, at its priority rank. ## Quests diff --git a/quest/m1/2152-libmoq-c-abi-catch-up-with-the-moq-ffi-surface.md b/quest/m1/2152-libmoq-c-abi-catch-up-with-the-moq-ffi-surface.md index d822523a612..594686bafe8 100644 --- a/quest/m1/2152-libmoq-c-abi-catch-up-with-the-moq-ffi-surface.md +++ b/quest/m1/2152-libmoq-c-abi-catch-up-with-the-moq-ffi-surface.md @@ -31,10 +31,10 @@ The libmoq C ABI (`rs/libmoq/src/api.rs`) has fallen well behind moq-ffi. All of Suggest splitting off pieces as they're picked up rather than one mega-PR. Each addition also touches `cpp/obs` consumers only if used, plus `doc/lib/c` per the Cross-Package Sync table. -Video decode is the widest hole: `moq-ffi` depends on `moq-audio` but not -`moq-video`, so the UniFFI bindings only ever see raw encoded frames, and -`libmoq`'s `moq_consume_video_raw` is H.264-only with no format or resolution -knob. `moq play` is the worked example of what the shape should be. +Video decode is the widest hole on the C side: `moq-ffi` has a `video` +feature over `moq-video` (NVIDIA and VAAPI on), but `libmoq`'s +`moq_consume_video_raw` is H.264-only with no format or resolution knob. +`moq play` is the worked example of what the shape should be. ## Closes diff --git a/quest/m1/2296-moq-native-bring-the-quiche-backend-to-quinn-noq-feature.md b/quest/m1/2296-moq-native-bring-the-quiche-backend-to-quinn-noq-feature.md index a617f12e665..15cad1eca22 100644 --- a/quest/m1/2296-moq-native-bring-the-quiche-backend-to-quinn-noq-feature.md +++ b/quest/m1/2296-moq-native-bring-the-quiche-backend-to-quinn-noq-feature.md @@ -107,6 +107,10 @@ Likely upstream `web-transport-quiche` / `tokio-quiche` / `quiche` work, or a lo - \#686 tracks congestion control/BBR, where quinn and noq do not currently behave the same. - \#679 tracks multi-threaded UDP receive scaling, which is a reason to use quiche rather than a parity gap. +## Required + +- [noq parity gate](/quest/m2/quic/noq-parity.md) - decides whether quiche stays a supported backend; if it is retired this quest is abandoned with the verdict + ## Closes - [#2296](https://github.com/moq-dev/moq/issues/2296) - close this issue when the quest finishes diff --git a/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md b/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md index e6fe70c7454..b2337b3bc79 100644 --- a/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md +++ b/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md @@ -8,10 +8,10 @@ within the issue's stated scope and boundaries. ## Plan Rescoped during the 2026-08 grooming: SETUP role, frame naming, typed errors, -and start/end cursors landed on dev. Two gaps survive: track end semantics -(finishAt / finished(), still collapsed into close(abort?)), and the -producer-side `announce(prefix)` a Rust publisher has and a browser one does -not. +and start/end cursors landed on dev. One gap survives here: track end +semantics (finishAt / finished(), still collapsed into close(abort?)). The +producer-side prefix serving a Rust publisher has and a browser one does not +is [JS announce](/quest/m1/js-announce.md), not this quest. ### Issue context diff --git a/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md b/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md index ceac56e0dc4..28fb1ef83a1 100644 --- a/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md +++ b/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md @@ -60,4 +60,4 @@ Branch from `dev`, where the shared connection and `forward.ts` live. - [#2848](/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md) - audio following its grant, the Rust half - [#2859](/quest/m1/2859-passthrough-imports-reserve-no-bandwidth-so-a-co-resident.md) - passthrough imports reserving nothing -- [#2857](/quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md) - the same gap for the native bindings +- [#2857](/quest/m1/plan-binding-rate-control.md) - the same gap for the native bindings diff --git a/quest/m1/2714-per-subscriber-path-predicate-for-originconsumer-0-1-x.md b/quest/m1/2714-per-subscriber-path-predicate-for-originconsumer-0-1-x.md deleted file mode 100644 index bd91dbcb6ea..00000000000 --- a/quest/m1/2714-per-subscriber-path-predicate-for-originconsumer-0-1-x.md +++ /dev/null @@ -1,78 +0,0 @@ -# [M] Per-subscriber path predicate for OriginConsumer (0.1.x) / AnnounceConsumer (0.2.x) - -## Goal - -Implement and verify the behavior tracked in [#2714](https://github.com/moq-dev/moq/issues/2714) -within the issue's stated scope and boundaries. - -## Plan - -Rescoped during the 2026-08 grooming: answer the design question against -dev's reshaped announce/origin surface only; the 0.1.x backport half is -dropped unless a maintenance branch appears. See also the pattern-scoped -origin work at /quest/m2/path-patterns/origin.md. - -### After prefix routes - -A per-path predicate does not mean what it did when this was filed. -[moq#3225](https://github.com/moq-dev/moq/pull/3225) made `announced()` yield -covering prefixes rather than broadcast paths, so a predicate applied to the -announce stream is deciding about a *set*: a route at `room/` carries every -path under it, including the ones a deafen rule wants blocked, and blocking the -whole prefix hides paths the subscriber may use. The filter also cannot be the -enforcement point on its own, because `request_broadcast` resolves against the -route table directly. - -So this needs the enforcement boundary settled before an API: a predicate over -announcements (discovery), a predicate at resolution (access), or the -pattern-scoped consumer in -[origin](/quest/m2/path-patterns/origin.md). The reporter's requirement, which a -forked client cannot bypass, points away from a discovery-time filter. - -### Issue context - -Hi @kixelated 👋 - -We're building [WolvaneSfu](https://github.com/postanteGames/WolvaneSfu) - a MoQ-primary SFU backend (Rust media-engine + Elixir signaling) using `moq-net` for the relay side. We need a per-subscriber dynamic path exclusion filter to implement server-authoritative moderation (deafen enforcement - audio path subscribes filtered per-user based on server state) that a malicious/forked client cannot bypass. - -Current `scope(&[Path])` is inclusion-only and immutable - it doesn't compose with a runtime-updated exclusion set. We'd like to add: - -**Proposed API (0.1.x)** - `src/model/origin.rs::OriginConsumer`: - -```rust -impl OriginConsumer { - /// Returns a new consumer that drops paths from the announce stream when - /// the predicate returns false. Unannounce events pass unconditionally - /// (to preserve memory-cleanup semantics for previously announced paths). - pub fn subscribe_filter(self, predicate: F) -> OriginConsumer - where F: FnMut(&Path) -> bool + Send + 'static; -} -``` - -Injection point: `OriginConsumerState::apply_announce` - predicate gate at the top. - -**0.2.x equivalent** (`AnnounceConsumer`): a `with_filter(F)` chain builder after `Consumer::announced()`, applied inside `next() / poll_next()` drainage. - -**Questions before I open a PR:** - -1. Would you accept this API? Any alternative design you'd prefer (different predicate signature, or a fundamentally different approach)? -2. Should the PR target the `0.1.x` maintenance branch (we're pinned to 0.1.18) or `0.2.x` main? Happy to backport both directions if maintenance branches are still cut. -3. Is `0.1.x` still maintained, or should we be planning a `0.2.x` migration first? - -If you'd rather I just open the PR and discuss inline, that works too - just wanted to check intent first since a "please migrate to 0.2.x" answer changes our plan meaningfully. - -Test suite I'm planning (5 tests, matching the `#[tokio::test]` + `assert_next_wait` style in existing `origin.rs::tests`): - -- `filter_blocks_matching_path` - paths where predicate returns false don't appear in `announced()` -- `filter_allows_unannounce_of_blocked` - unannounce passes unconditionally (memory cleanup) -- `filter_composable_with_scope` - `scope(&paths).subscribe_filter(pred)` chains correctly -- `filter_clone_independence` - filter is per-consumer, doesn't leak to siblings -- `filter_late_announce_gate` - announces arriving after subscribe still pass through the filter - -Thanks - MoQ is finally starting to feel like it's ready for prod SFU work. - -- Mustafa (WolvaneSfu) - -## Closes - -- [#2714](https://github.com/moq-dev/moq/issues/2714) - close this issue when the quest finishes diff --git a/quest/m1/2853-quiche-with-a-pinned-source-port-can-dial-only-a-broken.md b/quest/m1/2853-quiche-with-a-pinned-source-port-can-dial-only-a-broken.md index 4d9686921da..ff2881be300 100644 --- a/quest/m1/2853-quiche-with-a-pinned-source-port-can-dial-only-a-broken.md +++ b/quest/m1/2853-quiche-with-a-pinned-source-port-can-dial-only-a-broken.md @@ -35,6 +35,10 @@ For the `limit(1)` path specifically, wait for the authoritative answer rather t Found while reviewing #2852 (merging main into dev); the code is identical on `main`, so this is not a merge regression and was left out of that PR. Originally raised by the Codex connector bot as an inline comment there. +## Required + +- [noq parity gate](/quest/m2/quic/noq-parity.md) - decides whether quiche stays a supported backend; if it is retired this quest is abandoned with the verdict + ## Closes - [#2853](https://github.com/moq-dev/moq/issues/2853) - close this issue when the quest finishes diff --git a/quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md b/quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md deleted file mode 100644 index 48175f201a3..00000000000 --- a/quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md +++ /dev/null @@ -1,43 +0,0 @@ -# [M] Bindings can't reach encoder rate control, so every non-Rust publisher ignores congestion - -## Goal - -Implement and verify the behavior tracked in [#2857](https://github.com/moq-dev/moq/issues/2857) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Noticed while wiring [#2854](https://github.com/moq-dev/moq/pull/2854). - -#### Problem - -Every non-Rust publisher encodes at its configured bitrate regardless of congestion. There is no way to reach the encoder rate control from a binding: - -- `moq-ffi` and `libmoq` publish video (`moq_publish_video_raw`) and audio (`moq_audio::encode::Options::default()`), but never set `Options::bandwidth`. -- The send estimate is exposed only as a **snapshot**, on `moq_connection_stats` / `Session::stats` (`send_rate_bps` + a `_valid` flag). A caller can read a number; it cannot hand a live handle to an encoder. - -So an OBS plugin, or a Python/Swift/Kotlin/Go publisher, overshoots a closing uplink exactly as far as its configured bitrate is above what the link can carry, and keeps overshooting until the operator changes the setting by hand. The Rust path has followed the estimate since `moq_video::encode::rate` landed. - -This predates the bandwidth allocator; [#2854](https://github.com/moq-dev/moq/pull/2854) didn't regress it, it just made the hole visible by giving the Rust side a second thing bindings can't reach. - -#### What a fix needs - -The awkward part is that both `bandwidth::Consumer` and `bandwidth::Allocator` are live handles with wakeups, which is not a shape UniFFI or a C ABI carries naturally. Options, roughly cheapest first: - -- **A flag on the publish call**: "follow this connection's estimate", with the binding creating the allocator and registering the track on the caller's behalf. No new handle type crosses the boundary, and it's the behavior nearly every caller wants. Loses the ability to share one allocator across separately-created publishers. -- **An opaque allocator handle** (`moq_allocator_*` / a UniFFI object) minted from a session and passed into each publish call. Mirrors the Rust API exactly and composes across publishers, at the cost of a new object in five wrappers. -- **A polled getter** returning the current target for a track, leaving the caller to drive their own encoder. Fits an application that owns its encoder (OBS does), useless for the built-in encode path. - -The first two are not exclusive: the flag can be the default and the handle an escape hatch. - -Whichever way, this touches `rs/moq-ffi`, `rs/libmoq`, `{py,swift,kt}/`, `go/wrapper/moq/`, and `doc/lib/{py,swift,kt,go,c}` per the Cross-Package Sync table, plus `cpp/obs` if the OBS plugin should adopt it. - -## Closes - -- [#2857](https://github.com/moq-dev/moq/issues/2857) - close this issue when the quest finishes diff --git a/quest/m1/2933-moq-ffi-moqroute-round-trip-erases-a-routes-cold-cost.md b/quest/m1/2933-moq-ffi-moqroute-round-trip-erases-a-routes-cold-cost.md deleted file mode 100644 index 71cfb941047..00000000000 --- a/quest/m1/2933-moq-ffi-moqroute-round-trip-erases-a-routes-cold-cost.md +++ /dev/null @@ -1,65 +0,0 @@ -# [M] moq-ffi: MoqRoute round-trip erases a route's cold cost - -## Goal - -Implement and verify the behavior tracked in [#2933](https://github.com/moq-dev/moq/issues/2933) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### After prefix routes - -[moq#3225](https://github.com/moq-dev/moq/pull/3225) deleted both APIs the -issue names. `MoqBroadcastConsumer::route_updates` and -`MoqBroadcastProducer::set_route` are gone; a broadcast no longer carries a -route at all. - -The bug survived the move. `MoqOriginProducer::announce(prefix, MoqRoute)` is -the input, `MoqAnnouncement.route` is the output, and the two conversions are -unchanged: `From` reports `route.cost.warm` and drops `cold`, -while `TryFrom` calls `with_cost(u64)`, which sets both halves. So -observing a route through `announced()` and re-announcing it still rewrites a -truthful `{warm: 0, cold: N}` into `{warm: 0, cold: 0}`, claiming to be the -publisher. - -The severity argument in the issue no longer holds either: it rested on -`FrontState::handover_allowed` only consulting cold at two or more hops, and -that gate was deleted with the rest of the handover machinery. Cold is still -ranked on by `route_order`, so an understated cold still wins ties it should -lose. Re-read the options below against `announce`/`announced` rather than -`set_route`/`route_updates`. - -### Issue context - -Found by the Codex reviewer during [#2925](https://github.com/moq-dev/moq/pull/2925), which splits the route cost into `broadcast::Cost { warm, cold }`. Filing rather than fixing in that PR because it is a public-API decision spanning five language bindings. - -#### The gap - -`rs/moq-ffi/src/origin.rs` maps the pair onto one scalar in each direction: - -- `From for MoqRoute` reports `route.cost.warm` and drops `cold`. -- `TryFrom for broadcast::Route` calls `with_cost(u64)`, which sets *both* halves to that scalar. - -So a caller that observes a route through `MoqBroadcastConsumer::route_updates` and feeds it back into `MoqBroadcastProducer::set_route` converts a truthful `{warm: 0, cold: N}` into `{warm: 0, cold: 0}`. Cold 0 means "I am the publisher", so the republished route understates how far the content actually is, and a relay forwarding it advertises a cold cost lower than the truth. - -This was deliberately scoped out of #2925 to avoid rippling `cold` through `libmoq`, `py/`, `swift/`, `kt/`, `go/`, and `doc/lib/*` for a field no application routes on. The round-trip being silently lossy in the *unsafe* direction is what makes it worth revisiting. - -#### Severity is narrower than it first looks - -The handover gate only consults an advertised cold cost when `hops.len() >= 2` (see `FrontState::handover_allowed`). A route published by an application arrives at its relay with a one-hop chain, so it is treated as a direct publisher route and never enters the rank comparison. An application can also already misreport its production cost through the existing scalar, so this is not a new trust boundary - the relay mesh has always taken a publisher's seeded cost at face value. - -What is new is that the *observe then republish* path silently rewrites a value the app did not choose, rather than the app choosing to lie. - -#### Options - -1. Add `cold` to the `MoqRoute` uniffi record. Additive for the generated bindings (`rs/libmoq` does not expose routes over the C ABI, so no C break), but per the Cross-Package Sync table it touches `{py,swift,kt}/`, `go/wrapper/moq/*.go`, and `doc/lib/{py,swift,kt,go}`. An omitted cold could default to the supplied warm value, which is correct for a publisher seeding a production cost. -2. Keep the surface scalar and make the asymmetry unrepresentable instead, e.g. by separating the seed-a-production-cost input from the observe-a-route output so an observed route cannot be passed to `set_route`. -3. Accept it and document `MoqRoute.cost` as a production cost that is not meant to be echoed. Weakest option; `CLAUDE.md` prefers making misuse unrepresentable over documenting it. - -## Closes - -- [#2933](https://github.com/moq-dev/moq/issues/2933) - close this issue when the quest finishes diff --git a/quest/m1/3046-fold-moq-token-into-moq-token-via-a-usage-executable-view.md b/quest/m1/3046-fold-moq-token-into-moq-token-via-a-usage-executable-view.md index f50aa5f61a3..1ca85f08f60 100644 --- a/quest/m1/3046-fold-moq-token-into-moq-token-via-a-usage-executable-view.md +++ b/quest/m1/3046-fold-moq-token-into-moq-token-via-a-usage-executable-view.md @@ -23,7 +23,7 @@ Fold the standalone `moq-token` binary into `moq token`, so there is one binary #### Why now -If #3030 lands, Usage has the mechanism for exactly this: an **executable view**. A view is argv0 dispatch against a single binary's own spec, so one binary can present a different root depending on the name it was invoked under. `moq-token generate ...` and `moq token generate ...` become one spec with two surfaces, and help renders the right prefix for each (`page_view` / `render_failure_view` already exist for this). +Now that #3030 has landed on `dev`, Usage has the mechanism for exactly this: an **executable view**. A view is argv0 dispatch against a single binary's own spec, so one binary can present a different root depending on the name it was invoked under. `moq-token generate ...` and `moq token generate ...` become one spec with two surfaces, and help renders the right prefix for each (`page_view` / `render_failure_view` already exist for this). The blocker today is not the CLI plumbing, it is that they are separate build artifacts. Views cannot span two binaries. @@ -44,7 +44,7 @@ The second point may be the one that kills it. Worth measuring `moq` vs `moq-tok #### Depends on -\#3030 (Usage migration). Without it there is no view mechanism, and the shared-`Args` arrangement we have is about as good as clap allows. +\#3030 (Usage migration), merged to `dev` on 2026-08-26, which is why this branches from `dev`. ## Closes diff --git a/quest/m1/3051-config-merge-empty-lists-lose-to-the-environment-and-the.md b/quest/m1/3051-config-merge-empty-lists-lose-to-the-environment-and-the.md deleted file mode 100644 index 493b633206a..00000000000 --- a/quest/m1/3051-config-merge-empty-lists-lose-to-the-environment-and-the.md +++ /dev/null @@ -1,60 +0,0 @@ -# [M] Config merge: empty lists lose to the environment, and the file outranks it - -## Goal - -Implement and verify the behavior tracked in [#3051](https://github.com/moq-dev/moq/issues/3051) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -#### Summary - -Two related defects in how `moq-relay` and `moq-bench` resolve configuration. Both predate the Usage migration (#3030); one of them that migration briefly widened, and #3030 fixes that part. - -#### 1. A bare `Vec` loses to the environment - -The TOML merge parses CLI + env + defaults, overlays the file, then re-parses the CLI so explicit flags win. That last step fills any field the parser reads as *empty* -- from the environment and from declared defaults, not just from the command line. - -Emptiness is a property of the type. A `Vec` reads empty when it has no items, so a config file that deliberately sets an empty list is refilled from the environment: - -```toml -[connect] -version = [] # "offer every supported version" -``` - -With `MOQ_CONNECT_VERSION=moq-lite-01` set, this resolves to `moq-lite-01` instead. - -Affects roughly fifteen env-bound list fields: TLS certificate/key/root lists, accepted versions on both the connect and listen sides, the Unix peer allowlists (`uid`/`gid`/`pid`), cluster peers, auth domains, and the web HTTPS material. - -Booleans had the same bug and are now `Option` with the default resolved in code. The same fix works for lists (`Option>`) but multiplies a convention rather than removing the cause -- see below. - -#### 2. The file outranks the environment - -Precedence today is `CLI > TOML > env > defaults`. That ordering was never chosen: it falls out of the implementation, which folds env in during the parse and then overlays the file on top of everything. The sentence documenting it in `doc/bin/relay/config.md` was written to describe the behavior after the fact. - -Nearly every comparable tool does `CLI > env > file` (12-factor, Viper, and `usage-config`'s own default). The reasoning is deployment: the file ships inside the artifact and the environment varies per deployment, so file-wins means rebuilding an image to change one setting, and means a secret injected by an orchestrator cannot override a placeholder in a checked-in file. - -#### Why they are one issue - -Both come from the same missing thing: nothing records *which source set a value*. Presence is inferred from the value itself, so "set to false", "set to the empty list", and "never set" are indistinguishable -- and any fix built on comparing values inherits the same blind spot one level down ("set to the default value" vs "never set"). - -`usage-config` exists for this. `CliLayer` is built from what the parser saw rather than from the parsed struct, `EnvLayer` reads variables against a registry that declares which key each one backs, and `Layers` takes the order from the caller, so the precedence flip in (2) becomes a declared line rather than an emergent property. It also yields provenance, so the relay could answer "where did this setting come from". - -The cost is real and worth stating: `usage::Cli` and `usage::Config` share the `#[usage(...)]` attribute namespace and each rejects the other's attributes, so one struct cannot carry both derives. A registry means declaring every merged setting a second time -- roughly 100 for the relay, 20 for the bench -- with `Registry::drift` as a test that the two declarations stay in step. - -#### Suggested order - -1. `Option>` for the fifteen list fields, as a contained fix for (1) with a regression test. -2. Then evaluate the registry for (2), where the duplication buys the precedence flip and provenance rather than just a bug fix. - -Doing (2) first would make (1) unnecessary, so it is worth deciding before spending on (1). - -## Closes - -- [#3051](https://github.com/moq-dev/moq/issues/3051) - close this issue when the quest finishes diff --git a/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md b/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md index 66e1d1d2327..fe28b2769bc 100644 --- a/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md +++ b/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md @@ -1,15 +1,27 @@ -# [M] watch: video decoder captures the rewind generation at output time, not submit time +# [S] watch: the video decoder resets on a declared discontinuity ## Goal -Implement and verify the behavior tracked in [#3056](https://github.com/moq-dev/moq/issues/3056) -within the issue's stated scope and boundaries. +A frame submitted before a declared discontinuity never surfaces after it. +The decoder is reset (and reconfigured) when the container consumer reports a +discontinuity, the way the audio decoder already does, so queued pictures from +the old epoch are discarded rather than parked against the new clock. ## Plan -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. +Rescoped in the 2026-09 planning pass: undeclared rewinds are going away +([Monotonic timeline](/quest/m1/monotonic-timeline.md)), so the generation +mismatch below no longer produces the sixty-second park, but the bug it +describes still applies to a declared discontinuity, where a stale frame must +not be presented at all. Fix it the first way the issue suggests: call +`decoder.reset()` and re-`configure()` in `#onDiscontinuity`, and drop the +output-time generation guard, which then has nothing left to catch. The +regression needs a real WebCodecs decoder, so it lives in a browser harness +rather than a bun unit test. + +## Required + +- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - settles what a discontinuity is before the decoder's reaction to one is pinned ### Issue context diff --git a/quest/m1/3086-refactor-net-make-group-delivery-order-a-handle-and-move.md b/quest/m1/3086-refactor-net-make-group-delivery-order-a-handle-and-move.md deleted file mode 100644 index 3a2563e6f10..00000000000 --- a/quest/m1/3086-refactor-net-make-group-delivery-order-a-handle-and-move.md +++ /dev/null @@ -1,46 +0,0 @@ -# [M] refactor(net): make group delivery order a handle, and move timestamp-based skipping into moq-net - -## Goal - -Implement and verify the behavior tracked in [#3086](https://github.com/moq-dev/moq/issues/3086) -within the issue's stated scope and boundaries. - -## Plan - -Rescoped during the 2026-08 grooming: the ordered-handle half landed on dev -as #3099. What remains: move timestamp-based group skipping from moq-mux -(poll_min/max_timestamp) into moq-net, settling the three open design -questions below. - -### Issue context - -`track::Subscriber` exposes two delivery orders as two methods on one handle: - -- `recv_group` / `read_frame` walk an arrival index (`PlainSubscriber::index`) -- `next_group` seeks by sequence (`PlainSubscriber::next_sequence`) - -The cursors are not independent. `recv_datagram` bumps `next_sequence`, and `read_frame` bumps both `index` and `next_sequence`. Interleaving the two orders on one subscriber is expressible today and the result is hard to reason about, so the rule lives in doc comments rather than in the type. - -Proposal: make the order a handle rather than a method choice, e.g. `track::Consumer::ordered()` returning a cursor that only yields groups in sequence order, with the arrival-order cursor keeping the plain path. That makes mixing the two unrepresentable instead of merely discouraged, per the Public API Scrutiny rules in `CLAUDE.md`. - -##### Timestamp-based group skipping - -The longer-term goal is to move the group-skipping logic out of `moq-mux` and into `moq-net`. Today it lives in `rs/moq-mux/src/container/consumer.rs` (`poll_read`), which decides whether to abandon a stalled group by comparing `poll_min_timestamp` / `poll_max_timestamp` across buffered groups against a latency budget. - -The reason it lives in `moq-mux` is that those timestamps come from parsing the container format through `F: Format`. That constraint has weakened: on Lite05+ the wire carries per-frame timestamps at the track's own timescale (TRACK\_INFO plus zigzag-delta frame timestamps), so `moq-net` can read them without knowing anything about the media. That keeps the "the relay does not know anything about media" rule intact. - -Open questions before this is designed: - -- What happens on wires that cannot carry the timescale (pre-Lite05 moq-lite, IETF moq-transport), where timestamps fall back to local monotonic milliseconds? A skip policy keyed on those means something different than one keyed on author timestamps. -- Does the skip policy belong on the ordered cursor itself, on `Subscription` (so the publisher's aggregate sees it), or both? -- What does `moq-mux` keep? Discontinuity marking and the empty-group boundary handling in `poll_read` are container concerns even if the skip decision moves down. - -##### Context - -Split out of the `poll_next_in_range` fix, which made the sequence-ordered path a `BTreeMap` seek instead of a full cache scan. That removed the performance argument for treating sequence order as the slower opt-in path; the API argument above stands on its own. - ---- - -## Closes - -- [#3086](https://github.com/moq-dev/moq/issues/3086) - close this issue when the quest finishes diff --git a/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md b/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md index 2cb2de0baaa..804034403ef 100644 --- a/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md +++ b/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md @@ -39,7 +39,7 @@ Validate that the value is exactly one of `2.5, 5, 10, 20, 40, 60`. #### Implementation plan 1. Change the FFI and C scalar types to floating-point milliseconds and perform exact supported-value validation before converting to `Duration`. -2. Add the UniFFI 20 ms default in coordination with #3189 so the field is changed once. +2. Keep the UniFFI 20 ms default that [#3227](https://github.com/moq-dev/moq/pull/3227) added on the integer field when the type changes, so the field moves once. 3. Preserve the exact JavaScript frame duration separately from catalog metadata. 4. Encode catalog jitter as the ceiling of the duration because jitter is an integer upper-bound hint, not the exact encoder cadence. 5. Update Python, Swift, Kotlin, Go, and C documentation and examples generated or wrapped from these types. @@ -52,7 +52,3 @@ Changing the published FFI and C field types is source-breaking, so this targets ## Closes - [#3208](https://github.com/moq-dev/moq/issues/3208) - close this issue when the quest finishes - -## Related - -- [#3189: Add UniFFI defaults to caller-constructed configuration records](/quest/m2/3189-add-uniffi-defaults-to-caller-constructed-configuration.md) - related open work diff --git a/quest/m1/3221-config-stop-cli-defaults-from-clobbering-toml-values.md b/quest/m1/3221-config-stop-cli-defaults-from-clobbering-toml-values.md deleted file mode 100644 index a6e0faf8a92..00000000000 --- a/quest/m1/3221-config-stop-cli-defaults-from-clobbering-toml-values.md +++ /dev/null @@ -1,71 +0,0 @@ -# [M] config: stop CLI defaults from clobbering TOML values - -## Goal - -Implement and verify the behavior tracked in [#3221](https://github.com/moq-dev/moq/issues/3221) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -#### Problem - -Target branch: `dev`. - -`rs/moq-relay/src/config.rs::Config::parse_and_merge` parses the CLI, loads the selected TOML file, then calls `config.update_from(&args)` so explicit CLI and environment values win. The final clap parse also reapplies `default_value` and `default_value_t` entries when the user did not supply those arguments, so defaults silently overwrite file values. - -Known affected fields include: - -- `moq_native::ClientConfig::bind` -- `moq_native::Backoff::{initial, multiplier, max, timeout}` -- `moq_native::websocket::Client::{enabled, delay}` -- `moq_native::Log::level` -- `moq_relay::WebConfig::ws` - -The existing `Option` convention only protects fields whose clap declaration has no default. For example, `websocket.delay` is optional but still has `default_value = "200ms"`, so a TOML value is replaced during `update_from`. - -#### Failure example - -Given a TOML file with values such as: - -```toml -[web] -ws = false - -[client] -bind = "127.0.0.1:12345" - -[client.websocket] -enabled = false -delay = "3s" - -[client.backoff] -initial = "7s" -multiplier = 3 -max = "30s" -timeout = "2m" - -[log] -level = "debug" -``` - -running `moq-relay ` without matching CLI overrides resets those fields to clap defaults. The most serious example is `web.ws = false` becoming `true`, which can expose a listener mode the operator explicitly disabled. - -#### Expected - -Only values explicitly supplied through CLI arguments or environment variables should override TOML. Parser defaults should fill fields only when no source provided a value. - -#### Suggested direction - -Fix the merge centrally instead of relying on every flattened config field to use a particular Rust type. Possible shapes are an all-optional CLI overlay, or using clap value-source metadata so only command-line and environment values are applied after TOML deserialization. - -Add table-driven regression coverage for nested flattened structs, bare scalar defaults, optional fields with clap defaults, environment overrides, and explicit CLI overrides. - -## Closes - -- [#3221](https://github.com/moq-dev/moq/issues/3221) - close this issue when the quest finishes diff --git a/quest/m1/README.md b/quest/m1/README.md index cf030cf0940..55f9f481fc9 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -5,7 +5,8 @@ Everything that lands on the dev branch or with its merge to main: the thread-per-core runtime (moq-uring, moq-tokio, quiche), the net model and allocator follow-ups, breaking bindings work, and the archive line that gates -the merge itself. +the merge itself, because moq.pro needs archive-backed recording on the release +that `dev` produces before it can adopt it. ## Plan @@ -27,8 +28,8 @@ with the current dev tree before starting. - [#2980](/quest/m1/2980-moq-relay-nothing-guards-the-socket-capturing-acceptor.md) - moq-relay: nothing guards the socket-capturing acceptor being installed on a listener - [#2853](/quest/m1/2853-quiche-with-a-pinned-source-port-can-dial-only-a-broken.md) - quiche with a pinned source port can dial only a broken IPv4 address - [#2624](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md) - moq-native: GOAWAY redirect guard classifies hosts by name, not by resolved address -- [#3086](/quest/m1/3086-refactor-net-make-group-delivery-order-a-handle-and-move.md) - refactor(net): make group delivery order a handle, and move timestamp-based skipping into moq-net - [#3161](/quest/m1/3161-retention-should-reclaim-idle-open-groups-now-that-expiry.md) - Retention should reclaim idle open groups now that expiry is timestamp-only +- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - a track's timestamps never fall below its live edge; publishers declare a discontinuity and continue forward, consumers stop detecting rewinds - [Group overflow](/quest/m1/group-overflow-abort.md) - an oversized open group aborts for every reader instead of shedding its head - [#2895](/quest/m1/2895-add-an-atomic-readiness-gate-for-origin-broadcasts.md) - Add an atomic readiness gate for Origin broadcasts - [#2985](/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md) - js/net: path-keyed publisher state goes stale when a broadcast is replaced @@ -42,7 +43,7 @@ with the current dev tree before starting. - [#2848](/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md) - Follow the bandwidth grant in moq-audio instead of holding a fixed reservation - [#2859](/quest/m1/2859-passthrough-imports-reserve-no-bandwidth-so-a-co-resident.md) - Passthrough imports reserve no bandwidth, so a co-resident encoder over-targets - [Ladder](/quest/m1/ladder/README.md) - a transcode ladder adapts to the uplink it publishes over, instead of encoding every live rung at its ceiling -- [#2857](/quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md) - Bindings can't reach encoder rate control, so every non-Rust publisher ignores congestion +- [Plan: binding rate control](/quest/m1/plan-binding-rate-control.md) - settle how a non-Rust publisher follows the send estimate before wiring five bindings - [#2709](/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md) - js/net mirrors the send-side bandwidth allocator so each publisher encodes against its own share - [#3000](/quest/m1/3000-track-teardown-on-poll-unused-is-not-atomic-against-a.md) - Track teardown on poll_unused is not atomic against a consumer reattaching - [#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md) - IETF stream resets send moq-lite error codes, so routine events read as INTERNAL_ERROR @@ -51,20 +52,18 @@ with the current dev tree before starting. - [#3187](/quest/m1/3187-preserve-structured-protocol-error-codes-across-ffi-and-c.md) - Preserve structured protocol error codes across FFI and C bindings - [#2318](/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md) - js/net: remaining capability gaps vs rs/moq-net (SETUP role, finish_at and final sequence, range controls, typed errors) - [#2774](/quest/m1/2774-collapse-reload-and-shared-into-one-connection-class.md) - Collapse Reload and Shared into one Connection class -- [#2714](/quest/m1/2714-per-subscriber-path-predicate-for-originconsumer-0-1-x.md) - Per-subscriber path predicate for OriginConsumer (0.1.x) / AnnounceConsumer (0.2.x) - [#2870](/quest/m1/2870-moq-hls-a-named-sibling-rendition-is-pinned-too-late-to.md) - moq-hls: a named sibling rendition is pinned too late to survive a same-path republish - [#2075](/quest/m1/2075-mirror-catalog-reservation-gating-in-moq-hang-js-hang.md) - Mirror catalog reservation gating in @moq/hang (js/hang) - [#933](/quest/m1/933-video-rotation-metadata-not-propagated-from-mobile-camera.md) - Video rotation metadata not propagated from mobile camera publish to watch renderer - [#3056](/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md) - watch: video decoder captures the rewind generation at output time, not submit time -- [#3221](/quest/m1/3221-config-stop-cli-defaults-from-clobbering-toml-values.md) - config: stop CLI defaults from clobbering TOML values -- [#3051](/quest/m1/3051-config-merge-empty-lists-lose-to-the-environment-and-the.md) - Config merge: empty lists lose to the environment, and the file outranks it +- [Config provenance](/quest/m1/config-provenance.md) - the merge records which source set a value, so TOML survives CLI defaults and empty lists, and env outranks the file - [#3046](/quest/m1/3046-fold-moq-token-into-moq-token-via-a-usage-executable-view.md) - Fold moq-token into moq token via a Usage executable view - [#3126](/quest/m1/3126-moq-bench-every-readme-example-fails-to-parse-and.md) - moq-bench: every README example fails to parse, and cumulative latency percentiles cannot be windowed to steady state - [#816](/quest/m1/816-expose-transportconfig.md) - QUIC flow-control windows on quic::Client and quic::Server, applied or refused per backend - [#3188](/quest/m1/3188-make-every-blocking-go-operation-cancellable-with-context.md) - Make every blocking Go operation cancellable with context.Context - [#3208](/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md) - Make 2.5 ms Opus frame durations work across bindings - [#2152](/quest/m1/2152-libmoq-c-abi-catch-up-with-the-moq-ffi-surface.md) - libmoq: C ABI catch-up with the moq-ffi surface -- [#2933](/quest/m1/2933-moq-ffi-moqroute-round-trip-erases-a-routes-cold-cost.md) - moq-ffi: MoqRoute round-trip erases a route's cold cost +- [Plan: route cold cost](/quest/m1/plan-route-cold-cost.md) - settle how a route's cold cost crosses the bindings without being rewritten on the way back - [#3049](/quest/m1/3049-moq-net-outbound-hop-path-construction-is-never.md) - moq-net: validate a hop chain where it is emitted, not only where it is parsed - [#3060](/quest/m1/3060-moq-net-ban-hop-id-0-from-hop-chains.md) - moq-net: ban Hop ID 0 from hop chains - [#2248](/quest/m1/2248-moq-mux-rebase-fmp4-export-timestamps-for-late-subscribers.md) - moq-mux: rebase fMP4 export timestamps for late subscribers diff --git a/quest/m1/config-provenance.md b/quest/m1/config-provenance.md new file mode 100644 index 00000000000..9a5d0e11bf7 --- /dev/null +++ b/quest/m1/config-provenance.md @@ -0,0 +1,60 @@ +# [L] Config: track which source set each value + +## Goal + +`moq-relay` and `moq-bench` resolve configuration with recorded provenance. +Only a value the command line or the environment actually supplied overrides +the TOML file; a list the file sets to empty, a flag it sets to false, and a +field the parser merely defaults all survive the merge. Precedence is +`CLI > env > file > defaults`, declared in one place, and the relay can say +which source set a setting. + +## Plan + +Two shipped defects share one cause. The merge parses CLI plus environment +plus defaults, overlays the file, then re-parses argv so explicit flags win. +Presence is inferred from the value, so "set to the empty list", "set to +false", "set to the declared default", and "never set" are indistinguishable: + +- A `Vec` reads empty when it has no items, so a file that deliberately + sets `version = []` is refilled from the environment. Roughly fifteen + env-bound list fields are affected: TLS certificate, key, and root lists, + accepted versions on both sides, the Unix peer allowlists, cluster peers, + auth domains, and the web HTTPS material. +- The final re-parse reapplies every declared default, so a flag with a + default (`websocket.delay = "200ms"`, `web.ws`, the backoff fields, the log + level, the client bind) overwrites what the file said. `web.ws = false` + becoming `true` re-exposes a listener the operator disabled. +- The file outranks the environment, which no comparable tool does and which + means a secret injected by an orchestrator cannot override a placeholder in + a checked-in file. + +The `Option` convention that patched the boolean half is a per-field +workaround; `Option>` would multiply it. Fix the merge instead of the +types: + +- Build the layers from what the parser saw rather than from the parsed + struct. `usage-config`'s `CliLayer`, `EnvLayer` over a registry that + declares which key each variable backs, and `Layers` in caller-chosen order + give exactly that, plus provenance. The cost is real: `usage::Cli` and + `usage::Config` reject each other's attributes, so every merged setting is + declared a second time, about a hundred for the relay and twenty for the + bench, with `Registry::drift` as the test that the two declarations stay in + step. An all-optional overlay struct is the fallback if the registry proves + too heavy; either way, presence comes from the source, never the value. +- Flip precedence to `CLI > env > file > defaults` and state it in + `doc/bin/relay/config.md`, replacing the sentence that describes today's + emergent order. +- Delete the `Option` resolve-in-code convention once the merge no + longer needs it, so a plain field is safe again. +- Table-driven regression coverage: empty lists, false booleans, optional + fields with declared defaults, nested flattened structs, environment + overrides, and explicit CLI overrides, each asserting both the value and + its source. + +Branch from `dev`: the relay config lives on `usage` there. + +## Closes + +- [#3051](https://github.com/moq-dev/moq/issues/3051) - close this issue when the quest finishes +- [#3221](https://github.com/moq-dev/moq/issues/3221) - close this issue when the quest finishes diff --git a/quest/m1/ladder/README.md b/quest/m1/ladder/README.md index 39a97c696eb..0918b55ee4f 100644 --- a/quest/m1/ladder/README.md +++ b/quest/m1/ladder/README.md @@ -79,4 +79,4 @@ encoders on every target change. - [#2848](/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md) - the other sender that reserves but never follows its grant - [#2859](/quest/m1/2859-passthrough-imports-reserve-no-bandwidth-so-a-co-resident.md) - passthrough tracks that reserve nothing at all -- [#2857](/quest/m1/2857-bindings-cant-reach-encoder-rate-control-so-every-non.md) - non-Rust publishers cannot reach rate control regardless of what the ladder does +- [#2857](/quest/m1/plan-binding-rate-control.md) - non-Rust publishers cannot reach rate control regardless of what the ladder does diff --git a/quest/m1/monotonic-timeline.md b/quest/m1/monotonic-timeline.md new file mode 100644 index 00000000000..fcce20b0292 --- /dev/null +++ b/quest/m1/monotonic-timeline.md @@ -0,0 +1,70 @@ +# [M] Timelines only move forward + +## Goal + +A track's timestamps never fall below the live edge its earlier groups +reached. A publisher that has to reset (a flush, a seek, a source restart) +declares an empty-group discontinuity and continues forward from where it +was; it can no longer rewind. Inside a group timestamps still reorder freely, +since B-frames present before the frames that precede them in decode order, +and open-GOP leading pictures still qualify because they sit above the +previous group's reach. + +Consumers stop detecting and re-anchoring on undeclared rewinds. A group that +breaks the rule is a malformed track, not a timeline event. + +## Plan + +Today both container consumers (`Rewind` in +`rs/moq-mux/src/container/consumer.rs` and in +`js/hang/src/container/consumer.ts`) classify a newer group whose timestamps +land before the live edge as a rewind, drop the reneged buffer, and bump the +same counter a declared discontinuity bumps. The hang draft says an empty +group declares a discontinuity "whether the resumed timestamps move backward +or forward". That machinery is what goes. + +- **Publisher side, in both container producers.** The track producer refuses + a frame whose timestamp is below the live edge established by the groups + before its own, returning an error the way an oversized frame does, so a + rewind never reaches the wire. That is `moq-mux`'s producer and the + `js/hang` container producer (`js/hang/src/container/legacy.ts` has no + cross-group live-edge check today), so a browser publisher cannot emit a + sequence the updated consumers reject. moqsink already re-anchors forward + on a flush and rejects a rewinding base; check the CLI importers, the + capture publishers, and `js/publish` do the same on a source restart, and + re-anchor rather than refuse where the source is trusted. +- **Consumer side, in hang.** Delete the rewind boundary and its + classification in both languages. A group below the live edge aborts the + track as malformed. The discontinuity counter stays, counting declared + discontinuities only; `js/watch` keeps resetting its decoders on it + ([#3056](/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md)). +- **Draft.** `drafts/draft-lcurley-moq-hang.md` loses the backward clause and + states the rule: after a discontinuity the timeline continues forward. The + moq-lite draft is unchanged; enforcement lives above the relay, in the + media layer that knows what a group is. +- **TS export.** [moq#3375](https://github.com/moq-dev/moq/pull/3375) lands + first and keys its reset on the discontinuity counter, which keeps working + once the counter only counts declared discontinuities. After it, remove the + `last_psi` / `last_si` / `last_pcr` reset a forward jump makes redundant and + reword its docs; keep `discontinuity_indicator` on the PCR packet, since a + PCR jump over 100 ms without it is a TR 101 290 error whichever direction + the jump goes. + +Tests, in both languages: the producer refuses a group below the live edge; +the consumer aborts on one; a group with reordered B-frames is accepted; an open-GOP group whose +leading pictures sit below its keyframe but above the previous group passes; +a declared discontinuity followed by a forward jump passes and bumps the +counter once; moqsink's flush re-anchor produces a forward timeline. + +Branch from `dev`, where the container consumers carry the current `Rewind` +state. + +## Required + +- [moq#3375](https://github.com/moq-dev/moq/pull/3375) has merged, so the TS export reset it adds is keyed on the discontinuity counter before this quest trims it + +## Related + +- [#3056](/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md) - the watch decoder reset that keeps mattering for declared discontinuities +- [#3115](/quest/m2/3115-moqsink-the-publication-has-no-generation-so-a-flush.md) - moqsink's generation model after EOS, the same publisher +- [#2833](/quest/m0/2833-moq-export-ts-a-rewound-timeline-stalls-the-si-table.md) - the TS export fix this trims once it lands diff --git a/quest/m1/plan-binding-rate-control.md b/quest/m1/plan-binding-rate-control.md new file mode 100644 index 00000000000..3353af59887 --- /dev/null +++ b/quest/m1/plan-binding-rate-control.md @@ -0,0 +1,42 @@ +# [S] Plan: encoder rate control from the bindings + +## Goal + +A settled shape for how a non-Rust publisher follows the connection's send +estimate, so an OBS, Python, Swift, Kotlin, or Go publisher stops encoding at +its configured bitrate regardless of congestion. Run `/plan-quest`; the settled +plan becomes the implementing quest that closes the issue. + +## Plan + +`moq-ffi` and `libmoq` publish video (`moq_publish_video_raw`) and audio with +default encoder options and never set `Options::bandwidth`. The send estimate +reaches a binding only as a snapshot on `moq_connection_stats` / +`Session::stats`, which a caller can read but cannot hand to an encoder. The +Rust path has followed the estimate since `moq_video::encode::rate` landed, and +the allocator ([moq#2854](https://github.com/moq-dev/moq/pull/2854)) gave it a +second live handle bindings cannot reach. + +The awkward part is that `bandwidth::Consumer` and `bandwidth::Allocator` are +live handles with wakeups, which UniFFI and a C ABI do not carry naturally. +Candidates, cheapest first, and the first two compose: + +- A flag on the publish call: "follow this connection's estimate", with the + binding creating the allocator and registering the track. No new handle + crosses the boundary; loses sharing one allocator across separately created + publishers. +- An opaque allocator handle minted from a session and passed into each + publish call. Mirrors the Rust API and composes, at the cost of a new object + in five wrappers. +- A polled getter returning the current target for a track, for applications + that own their encoder (OBS does); useless for the built-in encode path. + +Whichever shape wins touches `rs/moq-ffi`, `rs/libmoq`, every wrapper, and +`doc/lib/*` per the Cross-Package Sync table, plus `cpp/obs` if the plugin +adopts it. + +## Related + +- [#2857](https://github.com/moq-dev/moq/issues/2857) - the issue the implementing quest closes +- [#2709](/quest/m1/2709-per-broadcast-bandwidth-estimates-and-reservation.md) - the same allocator mirrored in js/net +- [Ladder](/quest/m1/ladder/README.md) - the transcode consumer of the same estimate diff --git a/quest/m1/plan-route-cold-cost.md b/quest/m1/plan-route-cold-cost.md new file mode 100644 index 00000000000..c36e1999505 --- /dev/null +++ b/quest/m1/plan-route-cold-cost.md @@ -0,0 +1,43 @@ +# [S] Plan: the route cold cost across the bindings + +## Goal + +A settled shape for how a route's `{warm, cold}` cost crosses `moq-ffi` and +the language wrappers, so observing a route and re-announcing it cannot +silently rewrite a truthful cold cost. Run `/plan-quest`; the settled plan +becomes the implementing quest that closes the issue. + +## Plan + +`MoqOriginProducer::announce(prefix, MoqRoute)` is the input and +`MoqAnnouncement.route` the output. `From` reports +`route.cost.warm` and drops `cold`, while `TryFrom` calls +`with_cost(u64)`, which sets both halves. So a truthful `{warm: 0, cold: N}` +observed through `announced()` and announced again becomes +`{warm: 0, cold: 0}`, claiming to be the publisher. `route_order` still ranks +on cold, so an understated cold wins ties it should lose. + +Candidates, none chosen: + +- Add `cold` to the `MoqRoute` record. Additive for the generated bindings + and no C break, but it touches `{py,swift,kt,go,dart}/` and + `doc/lib/*`; an omitted cold could default to the warm value, which is right + for a publisher seeding a production cost. +- Keep the scalar and make the asymmetry unrepresentable: separate the + seed-a-production-cost input from the observe-a-route output so an observed + route cannot be re-announced as-is. +- Document `MoqRoute.cost` as a production cost that must not be echoed. The + weakest option; `CLAUDE.md` prefers unrepresentable over documented. + +Decide against `announce` / `announced` on `dev`, after +[Announce handle](/quest/m1/announce-handle.md) settles the surface the +bindings mirror. + +## Required + +- [Announce handle](/quest/m1/announce-handle.md) - the `announce` / `announced` surface the bindings mirror must be settled before the cost shape is decided against it + +## Related + +- [#2933](https://github.com/moq-dev/moq/issues/2933) - the issue the implementing quest closes +- [#3190](/quest/m1/3190-align-origin-broadcast-creation-naming-across-language.md) - the bindings surface this rides on diff --git a/quest/m1/uring-qlog.md b/quest/m1/uring-qlog.md index 23fe90c8a12..3ac99f29aaf 100644 --- a/quest/m1/uring-qlog.md +++ b/quest/m1/uring-qlog.md @@ -13,11 +13,11 @@ none of the machinery. ## Plan -Both backends, not just the default: quiche compiles qlog support in -unconditionally and quinn-proto has its own `qlog` feature, so +Every backend, not just the default: noq-proto and quinn-proto have their own +`qlog` feature and quiche compiles qlog support in unconditionally, so `--server-quic-qlog` must mean the same thing whether the relay was built with -`io-uring` or `io-uring-quinn`. Drop the bail in `transport()` once both are -wired. +`io-uring` (noq), `io-uring-quinn`, or `io-uring-quiche`. Drop the bail in +`transport()` once all three are wired. The open piece is where the bytes land. A pinned worker must not block on a file, so write through the ring if the adaptation is clean: the QUIC stacks diff --git a/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md b/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md index ea6a9630f2d..e2c3f98e572 100644 --- a/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md +++ b/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md @@ -1,91 +1,36 @@ -# [L] watch: absolute wall-clock latency target for synchronized playback across viewers +# [S] hang: a timeline consumer exposes the wall anchor ## Goal -Implement and verify the behavior tracked in [#2278](https://github.com/moq-dev/moq/issues/2278) -within the issue's stated scope and boundaries. +A browser application can read a rendition's `Timeline.wall` and its records +through a `Timeline.Consumer` in `js/hang`, so an application that knows its +viewers share a clock can compute the delay that renders one frame at one +instant everywhere, and a DVR view can map presentation time to wall time. -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Let every viewer render the same frame at the same instant. Watch parties, sports bars, betting, second-screen, and multi-camera sync all need it. HLS does it badly via `EXT-X-PROGRAM-DATE-TIME`; WebRTC doesn't try. - -The pieces are closer than they look: **the PTS → wall-clock mapping already exists in the catalog and the player just never reads it.** - -#### What exists today - -**`Timeline.wall` is the mapping, and it's already shipping.** `rs/hang/src/catalog/timeline.rs`: - -```rust -pub struct Timeline { - pub track: String, - pub timescale: u32, // default 1000 (ms) - pub wall: Option, // wall-clock of pts 0, in timescale units since the MoQ epoch -} -pub const MOQ_EPOCH_UNIX_MILLIS: u64 = 1_577_836_800_000; // 2020-01-01 -``` - -Set via `moq_mux::timeline::Producer::set_wall(pts, wall)` (`rs/moq-mux/src/timeline.rs`), attached per-rendition on `VideoConfig.timeline` / `AudioConfig.timeline`. JS mirrors the schema (`js/hang/src/catalog/timeline.ts`) and the producer (`js/hang/src/container/timeline.ts` - `setWall(pts, wall)`). - -So `wall + pts` → Unix time, per rendition, today. - -**Frame timestamps are PTS, deliberately, and must not be repurposed.** Worth stating clearly because it's easy to get wrong: - -- `hang::container::Frame::timestamp` doc (`rs/hang/src/container/frame.rs:27`): "the presentation timestamp... This is NOT a wall clock time." -- `rs/moq-net/src/model/time.rs:119`: "All timestamps within a track are relative, so zero for one track is not zero for another." -- `Timestamp::now()` is a deliberately one-way bridge with **no inverse**, and the anchor is 2020-01-01 **minus a per-process random 0..69420ms jitter**, explicitly "to deter nerds trying to use timestamp as wall clock time" (`time.rs:412`). -- The lite-05 wire field is a **zigzag delta of PTS at the track timescale** (`rs/moq-net/src/lite/publisher.rs:828` `encode_frame_timing`), not a wall-clock value. +The library itself never synchronizes playback on wall time. That is the +decision behind [#2278](https://github.com/moq-dev/moq/issues/2278): frame +timestamps are relative by design, `Timestamp::now()` is a one-way bridge with +a per-process jitter to deter wall-clock readings, and two machines' clocks are +only comparable when the application already knows they are synced. No +absolute `delay` mode, no client clock estimation from the session RTT, and no +sync exchange over a track. -The codebase actively fights wall-clock interpretation of frame timestamps. `Timeline.wall` is the sanctioned channel and this issue should stay on it. - -#### What's missing - -**Latency is relative (buffer depth), never absolute.** `js/watch/src/sync.ts`: - -```ts -export type Bound = "real-time" | Time.Milli; -export type Latency = Bound | { min?: Bound; max?: Bound }; -``` - -`Sync.received(timestamp)` computes `ref = Time.Milli.now() - timestamp`, and `Time.Milli.now()` is `performance.now()` (`js/net/src/time.ts:68`) - monotonic since page load, explicitly "not wall-clock time". So `reference` is an arbitrary local-clock↔PTS offset anchored by whichever frame happened to arrive first. Two viewers who tune in 3 seconds apart are 3 seconds apart forever, and nothing in the model can tell. - -`Sync` never reads `Timeline.wall`. There is no JS `Timeline.Consumer` at all (producer only). - -#### Proposed shape - -1. **A JS `Timeline.Consumer`** in `js/hang` (needed by the DVR work too). -2. **A third `Bound` variant, or a new absolute mode**: something like `latency: { absolute: Time.Milli }` meaning "render pts P at wall time `wall + P + absolute`". `Sync` then anchors on `Timeline.wall` instead of first-frame arrival, and `now()` becomes a wall-clock computation rather than an offset. -3. **Client clock sync.** This is the real work. `Timeline.wall` is the *publisher's* wall clock; the viewer's `Date.now()` can be off by seconds. Options, roughly in order of cost: - - trust `Date.now()` (fine for a watch party, useless for betting) - - estimate offset from the session RTT (we already track min RTT via PROBE in `#runJitter` for exactly this kind of anchoring) - - NTP-ish round trips over a MoQ track - - let the app inject a clock - Suggest: an injectable clock with an RTT-based default, so the app can bring its own. -4. **Degrade honestly.** If the target wall time has already passed (viewer joined late, or clock skew is large) the player must either skip forward or admit it can't hit the target. Silently drifting back to relative is worse than an error. -5. **Surface the achieved offset** so an app can show "you are 240ms behind the reference". - -#### Open questions - -- Absolute latency and the existing `min`/`max` bounds interact awkwardly: an absolute target is a *point*, the bounds are a *range*. Does absolute override the range, or clamp within it? -- `Timeline.wall` is per-rendition. Are audio and video guaranteed consistent? A rendition switch must not re-anchor. -- Does this need anything on the publisher, or is `set_wall` sufficient? Today nothing in `moq-video`/`js/publish` calls `set_wall` as far as I can tell - so step 0 might be "actually populate `wall`". - -#### Naming note - -`Latency` is already doubly overloaded: the `Bound | {min,max}` type (`js/watch/src/sync.ts:16`) and `class Latency` (`js/hang/src/util/latency.ts:21`, jitter+buffer). Different packages, both exported. Worth resolving before adding a third meaning. +## Plan -#### Branch +`js/hang` has a timeline producer (`setWall(pts, wall)`) and no consumer; +`js/watch/src/sync.ts` anchors on first-frame arrival against +`performance.now()`. Add the consumer beside the producer, mirroring the Rust +timeline consumer's records and exposing `wall` as a signal, and keep `Sync` +untouched. Document in `doc/concept` how an application derives a `delay` +from `wall + pts` against its own clock, and what it gives up when the clocks +are not synced. -`main` - additive on the player. Unless `Timeline` gains fields, in which case check `#[non_exhaustive]` (it has it) and the JS schema. +Nothing populates `wall` from the built-in publishers today, so this waits on +[Publishers anchor the timeline](/quest/m2/timeline-wall.md). -#### Cross-package sync +## Required -`js/watch`, `js/hang`, `rs/hang`; `demo/web` if it exposes the knob; `doc/concept`. +- [Publishers anchor the timeline](/quest/m2/timeline-wall.md) - there is no anchor to expose until publishers set one ## Closes diff --git a/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md b/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md index 1bcabd33e61..c9c14f51a94 100644 --- a/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md +++ b/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md @@ -1,65 +1,47 @@ -# [L] hang: typed SCTE-35 / ad cue signaling (carried opaquely today, unreadable by players) +# [L] hang: SCTE-35 cues a player can act on ## Goal -Implement and verify the behavior tracked in [#2279](https://github.com/moq-dev/moq/issues/2279) -within the issue's stated scope and boundaries. +An ad cue carried through MoQ is readable by a browser without knowing MPEG-TS +exists, and a player can raise an event on it. Today `splice_info_section` +bytes ride byte-faithfully on a TS-specific verbatim track that only the TS +exporter understands. Server-side ad insertion is a separate future quest. ## Plan -Rescoped during the 2026-08 grooming: typed signaling plus a player-side cue -event only, prototyped as a CatalogExt. SSAI splits into its own future quest. - -### Issue context - -Ad insertion: nobody streams commercially without it, and MoQ has no ad story. The pieces to signal a splice already exist but only as opaque MPEG-TS byte carriage, so a cue can ride *through* MoQ but nothing can *act* on one. - -MoQ should also be able to do this better than HLS: switching broadcasts is instant, so an ad transition needn't stall or re-buffer the way a discontinuity does. - -#### What exists today - -**SCTE-35 is carried, byte-faithfully, but never parsed.** It lives entirely in the `moq-mux` MPEG-TS container via the `mpegts` catalog extension: - -- `rs/moq-mux/src/container/ts/catalog.rs` - `Verbatim { stream_type: u8, framing: Framing, stream_id: Option }`, `enum Framing { Pes, Section }`. SCTE-35 is `Verbatim::new(0x86, Framing::Section)`. Plus `Descriptor { tag, data }` for the program-level `CUEI` registration. -- **Import**: `rs/moq-mux/src/container/ts/import.rs:404` routes section-framed verbatim PIDs to one MoQ track per ES, each frame a complete `splice_info_section`, byte for byte. Cues are stamped with **video PTS** (`import.rs:2218`). Requires an extension catalog - a `Catalog<()>` routes the CUEI PID to `Stream::Ignored`. -- **Export**: `rs/moq-mux/src/container/ts/export.rs:596-613` re-derives the `CUEI` registration; `:834` packetizes private sections. Note section-framed export **requires a video track** for the program clock. -- Real coverage: `ts/import_test.rs:352` (6 real `splice_insert`s from `test_data/scte35/kyrion_dirtystart.ts`, TSDuck-verified), `ts/export_test.rs:565` (TS→MoQ→TS byte-identical round trip), `rs/moq-mux/examples/scte35_inject.rs`. PRs #1617, #1685, #1696. - -So: TS in → TS out preserves cues perfectly. But a **browser watching that same broadcast sees nothing**, because the cue is an undecoded private section on a TS-specific track that only the TS exporter understands. - -**No ID3, no `emsg`, no generic timed-metadata track type anywhere.** The `scte35` name in `rs/hang/src/catalog/root.rs:363`, `rs/moq-mux/src/catalog/hang/ext.rs:155`, and `rs/moq-json/src/snapshot.rs:667` is **test fixture / doc example only** - an illustrative extension, not a shipped schema. - -#### What's missing - -Three separable things: - -1. **A typed, container-independent cue model.** Parse `splice_info_section` into something an application can read (`splice_insert`, `time_signal`, segmentation descriptors, `out_of_network_indicator`, duration, `splice_time` → PTS). A cue should be readable by a browser without knowing MPEG-TS exists. -2. **A catalog section describing where cues live**, so a consumer can find the track. The `mpegts` section is the worked example to copy: `rs/moq-mux/src/container/ts/catalog.rs` defines `struct Ext { mpegts: Mpegts }` + `impl CatalogExt for Ext`, plus a `trait Catalog: CatalogExt` composition hook implemented for `()`/`Extra`/`Ext`. -3. **Something that acts on a cue.** Player-side (CSAI: fire an event, let the app swap sources) and/or server-side (SSAI: splice a different broadcast into the output). Relative broadcast refs (`VideoConfig.broadcast`, `RelativeBroadcastSchema`) already let a catalog point at another broadcast, which is most of the mechanism for a server-side splice. - -#### Proposed shape - -- A `moq-json::stream` track of typed cue records, mirroring the timeline track's shape (`hang::timeline::Record` on a `moq_json::stream`, one record per frame). Cues are sparse, self-contained, and want lossless ordered delivery - `stream` is exactly right, and `RecordExt` is the extension hook. -- Records carry PTS in the same timescale as the timeline section, so cue → group resolution reuses the timeline machinery. -- Prototype **out of tree as a `CatalogExt` first** (exactly like `ts::catalog::Ext`) before touching base `hang`. `rs/hang/src/catalog/root.rs:352` has an `extension_roundtrip` test showing the serde-flatten pattern; JS uses `z.extend(RootSchema, ...)` (`js/hang/src/catalog/root.test.ts:15`) and `RootSchema` is a `z.looseObject` so unknown sections pass through untouched. -- The TS importer keeps its verbatim lane (contribution needs byte fidelity), and **additionally** emits typed cues. The two are complementary, not either/or. - -#### Open questions - -1. **Typed cues vs. verbatim, or both?** Both, I think: verbatim for TS→TS fidelity, typed for everything else. But then the TS exporter must not double-emit. -2. **Which SCTE-35 subset?** `splice_insert` + `time_signal` + segmentation descriptors covers most real usage. Full SCTE-35 is large. Prefer a maintained crate over hand-rolling if one exists at acceptable quality. -3. **Does this generalize to timed metadata?** ID3, `emsg`, and custom app cues want the same track shape. A generic "timed metadata track" with SCTE-35 as one record type may be the better primitive than an SCTE-35-specific section - cf. the Public API Scrutiny rule about composable building blocks over bespoke one-offs. -4. **SSAI is a much bigger scope** (ad decision server, per-viewer manifests, tracking beacons). Suggest this issue covers signaling + a player-side event, and SSAI gets its own. -5. Cue PTS is stamped from **video** PTS on import - check that's right for audio-only broadcasts. - -#### Branch - -`main` if it lands as a `CatalogExt` extension or a new optional section that doesn't break struct literals. Note `hang::catalog::Catalog` (`rs/hang/src/catalog/root.rs:16`) is **not `#[non_exhaustive]`** and has pub `video`/`audio` fields, so adding a field to base `hang` breaks struct-literal construction → that variant is `dev`. - -#### Cross-package sync - -`rs/hang` ↔ `js/hang`; `rs/moq-mux`; `doc/concept`. If a catalog section lands, `drafts/draft-lcurley-moq-hang.md` in the same PR. +Rescoped in the 2026-09 planning pass to the sidecar rule every metadata +section follows (see [SEI section](/quest/m2/sei/sei.md)): a metadata track +belongs to one rendition, uses that rendition's group sequence, stamps each +frame with the wire timestamp of the media it applies to, and carries raw +bytes. Not a `moq-json` stream of typed records. + +- **A `scte35` catalog section**, as its own `CatalogExt` like the `mpegts` + one, naming one sidecar track per rendition that carries cues. Cues from a + TS program attach to its video rendition, or to its audio rendition when + there is no video; the import already stamps them with video PTS. +- **Framing.** Each frame is one complete `splice_info_section`, byte for + byte, in the group of the media it applies to, stamped with the splice + time's wire timestamp. Cues are sparse, so most groups are empty; a joiner + at a keyframe gets that GOP's cues. +- **Typed decode is a helper, not the wire.** A parser for `splice_insert`, + `time_signal`, and segmentation descriptors (a maintained crate if one is + adequate) lives beside the section in Rust and JS, and `js/watch` raises a + cue event from it. Applications that want the rest parse the bytes. +- **The TS lanes stay.** The importer keeps the verbatim `mpegts` track for + contribution fidelity and additionally emits the sidecar; the exporter must + not double-emit, and section-framed export still needs the video clock. + +Wire and catalog schema: a new optional section, additive, so it lands on +`main`, with `drafts/draft-lcurley-moq-hang.md` updated in the same PR as the +schema: the draft is the normative spec, and a section Rust and JS emit must +be in it the day it ships. Cross-package sync: `rs/hang`, `js/hang`, +`rs/moq-mux`, `doc/concept`. ## Closes - [#2279](https://github.com/moq-dev/moq/issues/2279) - close this issue when the quest finishes + +## Related + +- [SEI section](/quest/m2/sei/sei.md) - defines the sidecar rule this follows +- [ID3 catalog section](/quest/m2/id3.md) - the other typed timed-metadata section, same rule diff --git a/quest/m2/2822-moq-wasm-bind-the-datagram-path-append-datagram-recv.md b/quest/m2/2822-moq-wasm-bind-the-datagram-path-append-datagram-recv.md deleted file mode 100644 index b56f3a4922d..00000000000 --- a/quest/m2/2822-moq-wasm-bind-the-datagram-path-append-datagram-recv.md +++ /dev/null @@ -1,38 +0,0 @@ -# [S] moq-wasm: bind the datagram path (append_datagram / recv_datagram) - -## Goal - -Implement and verify the behavior tracked in [#2822](https://github.com/moq-dev/moq/issues/2822) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Split out of #2814, where the Codex reviewer raised it. - -\#2814 expands `rs/moq-wasm` to bind moq-net's model: publish, discovery, on-demand serving, fetch, subscription options, track properties, cursors, frame timestamps. Datagrams are the one part of the **track** model it leaves out. - -Unbound today: - -- `moq_net::track::Producer::append_datagram` / `write_datagram` -- `moq_net::track::Subscriber::recv_datagram` -- the `moq_net::Datagram` value type (`sequence`, `timestamp`, `payload`) - -So a browser publisher cannot emit datagrams and a browser subscriber has no way to observe them, even though `@moq/net` exposes `appendDatagram` / `writeDatagram` / `datagrams` and the browser transport carries them. - -This is reachable, not theoretical: datagrams flow on moq-lite-05 over a transport that supports them, and `rs/moq-wasm/src/transport.rs` already implements the datagram side of `web_transport_trait::Session`. They are dropped on IETF moq-transport, moq-lite before 05, and stream-only transports like WebSocket, so any binding should make that conditional obvious rather than looking like a silent no-op. - -Deliberately deferred out of #2814 rather than rushed in: everything else in that PR was verified in a browser against a relay, and shipping datagram bindings without the same treatment would be the one untested corner of a new public surface. A datagram round trip needs its own harness (publish a datagram, confirm it arrives, and confirm the documented drop on a version that can't carry it), which is more than a review pass should bolt on. - -`rs/moq-wasm/README.md` lists this under "Not covered" so the surface doesn't overclaim in the meantime. - -Related: #2816 (no browser test harness at all), which is what would make verifying this cheap. - -## Closes - -- [#2822](https://github.com/moq-dev/moq/issues/2822) - close this issue when the quest finishes diff --git a/quest/m2/2835-moq-wasm-bind-track-dynamic-so-a-browser-publisher-can.md b/quest/m2/2835-moq-wasm-bind-track-dynamic-so-a-browser-publisher-can.md deleted file mode 100644 index 99365afa6b8..00000000000 --- a/quest/m2/2835-moq-wasm-bind-track-dynamic-so-a-browser-publisher-can.md +++ /dev/null @@ -1,35 +0,0 @@ -# [S] moq-wasm: bind track::Dynamic so a browser publisher can serve cache-miss fetches - -## Goal - -Implement and verify the behavior tracked in [#2835](https://github.com/moq-dev/moq/issues/2835) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Split out of #2814, where the Codex reviewer raised it. - -\#2814 binds `TrackConsumer.fetchGroup(sequence, options)`, the *requesting* side of a FETCH. The serving side is missing. - -When a browser publisher is asked for a group it no longer has cached, moq-net resolves the fetch only if the publisher holds a `moq_net::track::Dynamic` and answers its `requested_group()` with a `GroupRequest` (`accept(info)` / `reject(err)`). Otherwise the fetch fails with `NotFound`. `rs/moq-wasm` exposes neither, so from JavaScript every cache-miss fetch is unserveable. - -Note the asymmetry this leaves: #2814 *does* bind the broadcast-level equivalent, `BroadcastProducer.requestedTrack()` over `broadcast::Dynamic`, so on-demand serving works for tracks but not for groups within a track. - -Roughly what's needed: - -- a `TrackProducer.requestedGroup()` entry point over `track::Dynamic`, minted lazily like `requestedTrack` does for `broadcast::Dynamic` (creating a `Dynamic` registers an on-demand handler, so it shouldn't happen for callers who never serve) -- a `GroupRequest` binding with `sequence`, `priority`, `accept(info)` -> `GroupProducer`, and `reject(code)` - -Deliberately deferred out of #2814 rather than rushed in: it is new public surface, and everything else in that PR was verified in a browser against a relay. Serving a cache-miss fetch needs a harness that evicts or never caches a group and then fetches it, which is more than a review pass should bolt on. - -Related: #2822 (datagrams, the other unbound part of the track model) and #2816 (no browser test harness, which is what would make verifying this cheap). - -## Closes - -- [#2835](https://github.com/moq-dev/moq/issues/2835) - close this issue when the quest finishes diff --git a/quest/m2/2907-bind-the-browser-through-moq-ffi-uniffi-instead-of-a.md b/quest/m2/2907-bind-the-browser-through-moq-ffi-uniffi-instead-of-a.md index 6493dbac8d9..76136601c1c 100644 --- a/quest/m2/2907-bind-the-browser-through-moq-ffi-uniffi-instead-of-a.md +++ b/quest/m2/2907-bind-the-browser-through-moq-ffi-uniffi-instead-of-a.md @@ -78,6 +78,21 @@ Nothing in this spike ran in a browser. Generated and type-checking is not worki - \#2816, no browser test harness. Blocks step 4 and is the root cause of #2814 needing four hand-verified review rounds. - \#2822 and #2835 are `moq-wasm` gaps that this approach would close by construction, since `moq-ffi` already binds datagrams and `track::Dynamic`. +### Verification this quest owns + +The two remaining `moq-wasm` gaps close by construction, because `moq-ffi` +already binds them: datagrams (`append_datagram` / `recv_datagram`, dropped on +IETF moq-transport, pre-05 moq-lite, and stream-only transports, which the +binding must make visible rather than a silent no-op) and `track::Dynamic` +(`requestedGroup()` plus a `GroupRequest` with accept and reject, so a browser +publisher can serve a cache-miss fetch). Each needs its own browser round trip +in the harness from step 4: a datagram published and received, and the +documented drop on a version that cannot carry it; a group evicted or never +cached, then fetched and served. No hand-written wasm surface is added for +either. + ## Closes - [#2907](https://github.com/moq-dev/moq/issues/2907) - close this issue when the quest finishes +- [#2822](https://github.com/moq-dev/moq/issues/2822) - close this issue when the quest finishes +- [#2835](https://github.com/moq-dev/moq/issues/2835) - close this issue when the quest finishes diff --git a/quest/m2/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md b/quest/m2/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md index 8054f8c5e67..e73ec1c5333 100644 --- a/quest/m2/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md +++ b/quest/m2/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md @@ -90,3 +90,7 @@ When the reference timestamp represents capture time, receivers can estimate cap ## Closes - [#3021](https://github.com/moq-dev/moq/issues/3021) - close this issue when the quest finishes + +## Related + +- [Publishers anchor the timeline](/quest/m2/timeline-wall.md) - the same anchor from the built-in publishers; the anchor is data, never a sync source diff --git a/quest/m2/3058-moq-relay-a-revalidation-re-check-cannot-update-a.md b/quest/m2/3058-moq-relay-a-revalidation-re-check-cannot-update-a.md deleted file mode 100644 index 9e1e9864eac..00000000000 --- a/quest/m2/3058-moq-relay-a-revalidation-re-check-cannot-update-a.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] moq-relay: a revalidation re-check cannot update a session's tier, and changes its alias only by closing it - -## Goal - -Implement and verify the behavior tracked in [#3058](https://github.com/moq-dev/moq/issues/3058) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Found while reviewing #3041, which merged as 7047347. Not a regression - it is what the new re-check does and does not do with a reply whose *non-scope* fields changed. - -`Auth::recheck` scores a reply by `Scope::covered_by`, which compares root, subscribe and publish. The resulting `AuthToken` is then dropped and only the `CacheHints` propagate: - -```rust -Fetched::Ok { resp, hints } => match self.authorize(&grant.params, &resp) { - Ok(token) if grant.scope.covered_by(&token) => Recheck::Valid { hints }, -``` - -So two fields the auth API can legitimately change behave inconsistently: - -**`tier` is silently ignored.** An endpoint that re-buckets a connection - moving it to a named billing tier, or off one - has no effect on a live session. The session keeps the tier it was admitted with until it reconnects. That matters because the tier decides which meter pays. - -**`alias` closes the session.** The alias becomes the token's `root`, so `covered_by`'s `self.root == token.root` fails and the session is revoked. That is arguably correct - the broadcast would otherwise keep announcing under a root the API no longer assigns - but it is a silent hard disconnect for what may be a benign rename, and it is not obviously the intended contract. - -The general question is what a re-check is allowed to *update in place* versus what forces a reconnect. Scope narrowing is settled (close, and the client reconnects into the narrower grant). `tier` and `alias` are not. - -`tier` looks the harder half: the session's stats handle is built from the admission token in `connection.rs`, so applying a new tier mid-session means rebuilding that handle, and usage already recorded under the old tier stays there. Worth deciding deliberately rather than inheriting whichever behaviour fell out. - -## Closes - -- [#3058](https://github.com/moq-dev/moq/issues/3058) - close this issue when the quest finishes diff --git a/quest/m2/3164-moq-audio-remove-heap-activity-and-blocking-locks-from.md b/quest/m2/3164-moq-audio-remove-heap-activity-and-blocking-locks-from.md deleted file mode 100644 index 0f66cc98d07..00000000000 --- a/quest/m2/3164-moq-audio-remove-heap-activity-and-blocking-locks-from.md +++ /dev/null @@ -1,50 +0,0 @@ -# [M] moq-audio: remove heap activity and blocking locks from the capture callback - -## Goal - -Implement and verify the behavior tracked in [#3164](https://github.com/moq-dev/moq/issues/3164) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Follow-up to #2479, found while confirming the full-duplex audio parity tracked by #2481. #2487 bounded the callback-to-encoder queue, but #2479 explicitly left the per-callback allocation for later. The capture path is bounded now, but it is not yet real-time safe in the sense the iroh-live source path was. - -#### Current failure mechanism - -Every cpal callback allocates a new `Vec`: - -- F32 uses [`data.to_vec()`](https://github.com/moq-dev/moq/blob/85c9f489195dde93fc1d4619c7ec00864975fc3a/rs/moq-audio/src/capture.rs#L219-L224). -- I16 and U16 use [`collect()`](https://github.com/moq-dev/moq/blob/85c9f489195dde93fc1d4619c7ec00864975fc3a/rs/moq-audio/src/capture.rs#L225-L234). - -When the depth-8 queue is full, `Sender::push` drops the rejected `Vec` on the same callback thread, which can run the allocator again: - -- [`capture/channel.rs::Sender::push`](https://github.com/moq-dev/moq/blob/85c9f489195dde93fc1d4619c7ec00864975fc3a/rs/moq-audio/src/capture/channel.rs#L53-L65) - -With AEC enabled, the callback also takes a blocking `std::sync::Mutex` in [`Canceller::process`](https://github.com/moq-dev/moq/blob/85c9f489195dde93fc1d4619c7ec00864975fc3a/rs/moq-audio/src/aec.rs#L218-L228). The playback driver takes the same state lock when it replaces the render-reference channel. Callback buffers above the pre-sized threshold can also grow AEC scratch storage on that thread. - -Heap contention or lock contention can delay the cpal callback long enough to cause the underrun or overrun that these queues are meant to prevent. Bounded memory alone does not make the callback path real-time safe. - -#### Required behavior - -- Move microphone samples through preallocated bounded storage. A fixed-resample SPSC endpoint or a buffer pool are both plausible, as long as callback push never allocates, frees, waits, or logs. -- Convert non-f32 device formats in fixed preallocated chunks rather than collecting a new buffer. -- Preserve the current drop and gap semantics. Overflow must remain observable so the producer re-anchors its timeline. -- Make AEC callback-owned state exclusive to the callback. Device-switch updates should cross a bounded nonblocking handoff rather than contend on its state lock. -- Size scratch storage before `stream.play()`, or process oversized callbacks in bounded chunks, so callback size cannot trigger a one-time allocation. -- Keep allocations on the async consumer side acceptable. That side is not the real-time thread. - -#### Tests - -Factor the callback adapter so hardware-free tests can feed F32, I16, and U16 buffers through it. Assert bounded overflow and gap reporting, chunk sizes above the normal host period, and AEC reference replacement. Add an allocation-counting test or equivalent instrumentation proving the callback adapter performs no allocation or free after construction. - -Refs #2479, #2481. - -## Closes - -- [#3164](https://github.com/moq-dev/moq/issues/3164) - close this issue when the quest finishes diff --git a/quest/m2/3165-moq-audio-bound-and-coalesce-playback-driver-commands.md b/quest/m2/3165-moq-audio-bound-and-coalesce-playback-driver-commands.md deleted file mode 100644 index d7b2cafbd39..00000000000 --- a/quest/m2/3165-moq-audio-bound-and-coalesce-playback-driver-commands.md +++ /dev/null @@ -1,51 +0,0 @@ -# [M] moq-audio: bound and coalesce playback driver commands - -## Goal - -Implement and verify the behavior tracked in [#3165](https://github.com/moq-dev/moq/issues/3165) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Part of the full-duplex audio parity audit from #2481. #2478 called for a bounded driver command channel, matching iroh-live's bounded device-control path. The landed playback mixer and sample rings are bounded, but the device driver's outer command queue is not. - -#### Current failure mechanism - -[`Engine::open`](https://github.com/moq-dev/moq/blob/85c9f489195dde93fc1d4619c7ec00864975fc3a/rs/moq-audio/src/playback.rs#L98-L126) creates the driver channel with `std::sync::mpsc::channel()`. This queue is unbounded. Clones of its sender are used for: - -- output-device switches, -- sink and AEC synchronization wakes, -- shutdown, -- cpal error notifications from every stream generation. - -The driver performs blocking host operations while opening or switching devices. During one of those stalls, repeated cpal errors or caller commands can continue allocating queue nodes without a memory bound. The cpal error callback also uses the allocating `send` path before returning. - -The inner mixer queue is correctly bounded and retried, so the unbounded outer queue is now the only control-plane exception in the playback engine. - -#### Required behavior - -- Give the driver command path a fixed memory bound and never block a cpal callback on capacity. -- Coalesce idempotent `Sync` wakes instead of queueing one per event. -- Coalesce or otherwise bound failure notifications by live stream generation. A stale generation must not evict the signal for the current stream. -- Preserve reliable switch completion. A switch request must either enter the bounded driver path or return a clear overload or stopped error to its awaiting caller. -- Preserve reliable last-handle shutdown without requiring an unbounded emergency lane. -- Keep the existing mixer retry invariant. A dropped wake must not leave a sink or AEC tap permanently unattached. - -#### Tests - -- Flood each notification class while the driver receiver is paused and assert storage never exceeds the fixed bound. -- Confirm a saturated path cannot strand an awaiting switch response. -- Confirm a live-generation failure still schedules restart when stale failures and sync wakes are already pending. -- Confirm dropping the final engine or sink shuts the driver down under saturation. - -Refs #2478, #2481. - -## Closes - -- [#3165](https://github.com/moq-dev/moq/issues/3165) - close this issue when the quest finishes diff --git a/quest/m2/3189-add-uniffi-defaults-to-caller-constructed-configuration.md b/quest/m2/3189-add-uniffi-defaults-to-caller-constructed-configuration.md deleted file mode 100644 index aa8993c9164..00000000000 --- a/quest/m2/3189-add-uniffi-defaults-to-caller-constructed-configuration.md +++ /dev/null @@ -1,54 +0,0 @@ -# [S] Add UniFFI defaults to caller-constructed configuration records - -## Goal - -Implement and verify the behavior tracked in [#3189](https://github.com/moq-dev/moq/issues/3189) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -#### Problem - -Several caller-constructed UniFFI records describe fields as optional or defaultable but do not declare UniFFI defaults: - -- [`MoqAudioEncoderOutput` and `MoqAudioDecoderOutput`](https://github.com/moq-dev/moq/blob/7494084aaf7e2fa6abe553ac83101ac4ef19f33a/rs/moq-ffi/src/audio.rs#L68-L100) -- [`MoqVideoHint`, whose documentation says every field is optional](https://github.com/moq-dev/moq/blob/7494084aaf7e2fa6abe553ac83101ac4ef19f33a/rs/moq-ffi/src/media.rs#L159-L176) - -The video encoder and decoder output records already use `#[uniffi(default = None)]` consistently: - -- [Video output record defaults](https://github.com/moq-dev/moq/blob/7494084aaf7e2fa6abe553ac83101ac4ef19f33a/rs/moq-ffi/src/video.rs#L97-L114) -- [Video decoder defaults](https://github.com/moq-dev/moq/blob/7494084aaf7e2fa6abe553ac83101ac4ef19f33a/rs/moq-ffi/src/video.rs#L375-L388) - -Python, Swift, Kotlin, and Go publicly expose or alias the generated records. Without generated-language defaults, callers must spell every optional field, and adding a new field can become a source compatibility break. - -#### Proposed direction - -Audit every caller-constructed UniFFI record and declare defaults for all fields whose omission has defined behavior. At minimum: - -- optional values should default to `None` -- audio encoder frame duration should use its documented 20 ms default -- enum defaults should be declared where the API already defines a canonical automatic choice - -Keep returned data records distinct from caller-constructed options. Returned records do not need construction defaults. - -#### Acceptance criteria - -- Optional configuration fields can be omitted in Python, Swift, and Kotlin. -- Generated signatures expose the intended defaults rather than only documenting them. -- Adding a new optional field to these records remains source-compatible where UniFFI supports it. -- Binding tests construct each options record with only its required fields. -- The audit covers all public records in `moq-ffi`, not only the three known examples. - -## Closes - -- [#3189](https://github.com/moq-dev/moq/issues/3189) - close this issue when the quest finishes - -## Related - -- [#3208: Make 2.5 ms Opus frame durations work across bindings](/quest/m1/3208-make-2-5-ms-opus-frame-durations-work-across-bindings.md) - related open work diff --git a/quest/m2/README.md b/quest/m2/README.md index b1d9d50bb24..1a266389d04 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -18,8 +18,7 @@ can act on. Each still carries its own plan and regression test. ## Quests -- [Plan: HLS identity](/quest/m2/plan-hls-identity.md) - settle where a cacheable generation ID comes from via /plan-quest -- [Generation](/quest/m2/hls-generation.md) - media URLs carry a generation ID so segment caching can be re-enabled +- [Generation](/quest/m2/hls-generation.md) - init URLs follow the rendition config and segment URLs carry an embedder-supplied generation, so caching can be re-enabled - [Wildcard](/quest/m2/wildcard/README.md) - a service advertises a path pattern it could serve instead of enumerating broadcasts - [Path patterns](/quest/m2/path-patterns/README.md) - one versioned matcher for every predicate over broadcast paths: tokens, origins, interest - [Keyframe trigger](/quest/m2/keyframe-trigger.md) - an application can ask the built-in capture encoder for a keyframe @@ -29,13 +28,15 @@ can act on. Each still carries its own plan and regression test. ACK progress, reliable reset, hierarchical scheduling, and qmux - [Bandwidth estimate release](/quest/m2/web-transport-bandwidth-estimate.md) - web-transport-quinn reports quinn's BBR bandwidth estimate and ships a release carrying it - [#2847](/quest/m2/2847-the-quinn-backends-send-bandwidth-estimate-is-cwnd-rtt.md) - quinn backend: bump to the releases that report the controller bandwidth estimate instead of cwnd/rtt -- [Relay memory](/quest/m2/relay-memory/README.md) - relay memory scales with what it serves, not what the fleet knows +- [Relay memory](/quest/m2/relay-memory.md) - remeasure what an announcement costs after prefix routes +- [Route gauge](/quest/m2/route-gauge.md) - an operator sees how many routes a relay holds for a path - [PoP skipping](/quest/m2/pop-skipping/README.md) - short cold paths for unpopular broadcasts without losing warm backhaul dedup - [E2EE](/quest/m2/e2ee/README.md) - TypeScript and Rust peers interoperate over encrypted broadcasts no relay can decrypt - [SEI](/quest/m2/sei/README.md) - H.26x SEI moves into its own track, readable without subscribing to video - [Processor](/quest/m2/processor/README.md) - a customer-run worker publishes an on-demand contribution with scoped access -- [#2278](/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md) - watch: absolute wall-clock latency target for synchronized playback across viewers -- [#2279](/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md) - hang: typed SCTE-35 / ad cue signaling (carried opaquely today, unreadable by players) +- [Timeline wall](/quest/m2/timeline-wall.md) - every built-in publisher anchors its timeline to wall time; the anchor is data, never a sync source +- [#2278](/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md) - hang: a timeline consumer exposes the wall anchor, and the library never syncs playback on it +- [#2279](/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md) - hang: SCTE-35 cues ride a per-rendition sidecar a player can act on - [Caption import](/quest/m2/captions-import.md) - fMP4 and MKV subtitle tracks import as text renditions instead of erroring or being dropped - [MSF caption roles](/quest/m2/captions-msf.md) - an MSF caption, subtitle, or sign-language track survives conversion to a hang catalog - [CEA-608/708](/quest/m2/captions-cea.md) - captions carried inside video SEI become a real text rendition at import @@ -61,15 +62,10 @@ can act on. Each still carries its own plan and regression test. - [Dart codec parity](/quest/m2/dart-codecs.md) - Dart is the one binding that cannot originate media - [Direct3D11 render import](/quest/m2/render-d3d11.md) - Windows presents without downloading every frame to system memory - [#2147](/quest/m2/2147-moq-video-10-bit-hevc-and-av1-support-in-the-nvidia-codec.md) - moq-video: 10-bit HEVC and AV1 support in the NVIDIA codec path -- [#3164](/quest/m2/3164-moq-audio-remove-heap-activity-and-blocking-locks-from.md) - moq-audio: remove heap activity and blocking locks from the capture callback -- [#3165](/quest/m2/3165-moq-audio-bound-and-coalesce-playback-driver-commands.md) - moq-audio: bound and coalesce playback driver commands - [#2907](/quest/m2/2907-bind-the-browser-through-moq-ffi-uniffi-instead-of-a.md) - Bind the browser through moq-ffi/UniFFI instead of a second hand-written wasm API -- [#2822](/quest/m2/2822-moq-wasm-bind-the-datagram-path-append-datagram-recv.md) - moq-wasm: bind the datagram path (append_datagram / recv_datagram) -- [#2835](/quest/m2/2835-moq-wasm-bind-track-dynamic-so-a-browser-publisher-can.md) - moq-wasm: bind track::Dynamic so a browser publisher can serve cache-miss fetches - [#2850](/quest/m2/2850-js-net-give-reader-a-synchronous-decode-so-the-publisher.md) - js/net: decode messages synchronously from buffered bytes and delete the publisher read-ahead queue (dev) -- [#3189](/quest/m2/3189-add-uniffi-defaults-to-caller-constructed-configuration.md) - Add UniFFI defaults to caller-constructed configuration records - [Cluster flags](/quest/m2/cluster-flags.md) - a discovery mechanism carries its own prerequisites, so an incomplete cluster config cannot be expressed -- [#3058](/quest/m2/3058-moq-relay-a-revalidation-re-check-cannot-update-a.md) - moq-relay: a revalidation re-check cannot update a session's tier, and changes its alias only by closing it +- [Plan: revalidation updates](/quest/m2/plan-revalidation-updates.md) - settle which fields an auth re-check may update on a live session and which force a reconnect - [#3137](/quest/m2/3137-moqsrc-bound-the-pending-rendition-subscriptions-a.md) - moqsrc: bound the pending rendition subscriptions a catalog can open - [#3115](/quest/m2/3115-moqsink-the-publication-has-no-generation-so-a-flush.md) - moqsink: a flushing restart after EOS opens a new publication generation - [#709](/quest/m2/709-automatic-letsencrypt-support.md) - the relay provisions and renews its own ACME certificate over HTTP-01, persisted on disk diff --git a/quest/m2/av1-metadata.md b/quest/m2/av1-metadata.md index 11802ab2f14..b5c762da9bd 100644 --- a/quest/m2/av1-metadata.md +++ b/quest/m2/av1-metadata.md @@ -14,9 +14,12 @@ are neither separated nor addressable. This is the AV1 analogue of SEI in H.26x, and it should follow whatever the `sei` section settles rather than inventing a second shape. -Decide first whether AV1 metadata belongs in its own section or extends the -`sei` one, given the payloads overlap heavily (both carry ITU-T T.35 user data, -both carry timecode) but the framing does not. +It gets its own section, like every timed-metadata format, but follows the +sidecar rule the `sei` section defines: one track per AV1 rendition, the +rendition's group sequence, the temporal unit's wire timestamp on each frame, +raw OBU bytes. The payloads overlap with SEI (both carry ITU-T T.35 user data +and timecode), so the typed helpers can be shared even though the framing is +not. Preserve `metadata_type`, the payload bytes, and ordering within the temporal unit. Test HDR10+ T.35 metadata, timecode, an unknown metadata type, several diff --git a/quest/m2/captions-import.md b/quest/m2/captions-import.md index 875e7dabf43..d4685d2f850 100644 --- a/quest/m2/captions-import.md +++ b/quest/m2/captions-import.md @@ -30,10 +30,6 @@ for a cue that may be minutes away. Cover a file with subtitles and no supported text format, a file with several subtitle languages, and a file whose subtitle track is empty. -## Closes - -- [#2280](https://github.com/moq-dev/moq/issues/2280) - close this issue when the quest finishes - ## Related - [MSF caption roles](/quest/m2/captions-msf.md) - the catalog half of the same diff --git a/quest/m2/cli-packaging.md b/quest/m2/cli-packaging.md index bba48fc1b51..7ab89e16ad0 100644 --- a/quest/m2/cli-packaging.md +++ b/quest/m2/cli-packaging.md @@ -27,7 +27,3 @@ dependencies present, and a Docker image without them fails at build rather than at run. Then verify the shipped artifact runs `moq devices` and `moq play` on each platform, since a feature that compiles into the binary and then fails to open a device is the same gap one layer down. - -## Closes - -- [#2272](https://github.com/moq-dev/moq/issues/2272) - close this issue when the quest finishes diff --git a/quest/m2/drain/client-goaway.md b/quest/m2/drain/client-goaway.md index 9cec9cca5f1..6868220f457 100644 --- a/quest/m2/drain/client-goaway.md +++ b/quest/m2/drain/client-goaway.md @@ -17,6 +17,11 @@ lands clients on a healthy relay. Everything below describes `dev`, which is where this quest lands: `main` still has `moq-native`'s close-only `Reconnect` and no `Connection.Shared`. +It lands after [#2774](/quest/m1/2774-collapse-reload-and-shared-into-one-connection-class.md) +collapses `Reload` and `Shared` into one `Connection`, so the drain loop is +written once into that class and consumers migrate their call sites once; +where the text below says `Reload` or `Connection.Shared`, read the single +class and its pool. ### Rust is done except the proof @@ -82,4 +87,5 @@ down in its effect cleanup, and `Connection.Shared` ## Required +- [#2774](/quest/m1/2774-collapse-reload-and-shared-into-one-connection-class.md) - one Connection class first, so GOAWAY is built into it rather than into two - [Redirect guard by name](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md) - pin validated DNS results before enabling cross-host redirects by default diff --git a/quest/m2/emsg.md b/quest/m2/emsg.md index 4fa92a440e0..83c50e7ce34 100644 --- a/quest/m2/emsg.md +++ b/quest/m2/emsg.md @@ -17,9 +17,11 @@ time, duration, id, and the opaque message payload, for both version 0 and version 1 timing. Do not parse the payload into a vocabulary; an application decodes it with its own library, and a new scheme stays forward-compatible. -Anchor each event on the media timeline so a consumer can correlate it without -reproducing fMP4's timescale arithmetic. Export rebuilds the box in the fragment -whose media time contains it. +Carry them on a sidecar of the video rendition (the audio rendition when there +is none) following the rule in [SEI section](/quest/m2/sei/sei.md): the +rendition's group sequence, each box stamped with the wire timestamp of its +presentation time, raw bytes. Export rebuilds the box in the fragment whose +media time contains it. Test version 0 and version 1 boxes, an emsg before the first moof, several on one fragment, an unknown scheme, a zero duration, and a round trip that is diff --git a/quest/m2/flv-script.md b/quest/m2/flv-script.md index 850701ccb07..dc49fc43ee3 100644 --- a/quest/m2/flv-script.md +++ b/quest/m2/flv-script.md @@ -12,8 +12,10 @@ on, which drops every AMF data message an encoder sends. `onMetaData` is the one every RTMP publisher emits, and applications routinely push their own cues through the same channel. -Carry the tag payload byte-faithfully with its timestamp rather than decoding -AMF into a fixed vocabulary, so a custom message type needs no further work. +Carry the tag payload byte-faithfully rather than decoding AMF into a fixed +vocabulary, so a custom message type needs no further work, on a sidecar of +the video rendition following the rule in [SEI section](/quest/m2/sei/sei.md): +the rendition's group sequence and the tag's timestamp on the wire. Where `onMetaData` duplicates something the catalog already models (dimensions, framerate, bitrate), prefer the value the bitstream actually carries and treat the script tag as opaque data, not a second source of truth for configuration. diff --git a/quest/m2/hls-generation.md b/quest/m2/hls-generation.md index 864eebd17a1..6106cad1706 100644 --- a/quest/m2/hls-generation.md +++ b/quest/m2/hls-generation.md @@ -1,52 +1,45 @@ -# [L] HLS generation +# [M] HLS generation ## Goal -Media URLs carry a generation ID so segment caching can be re-enabled safely. The -same ID keeps a restarted publisher from being decoded against the previous run's -init segment. +`moq-hls` emits media URLs a CDN can cache. The init segment URL changes +whenever the rendition it initializes changes, and an embedder can hand the +exporter a generation string that every segment URL carries, so a restarted +publisher is never decoded against the previous run's init or served under +the previous run's segment numbers. A publisher that provides no generation +keeps today's unversioned URLs. ## Plan -Two bugs, one missing ingredient. A CDN edge must serve `.../seg/{n}.m4s` and -`init.mp4` with `no-store` today because the URLs are reused across publisher -generations. And `Rendition::init` caches the init for the life of the pooled -rendition with no invalidation, so an inline-parameter-set codec (`avc3`) that -restarts at a new resolution decodes new segments against the old parameter -sets; the segment window's `push` (`rs/moq-hls/src/export/segments.rs`) detects -that restart already and resets only the playlist. - -The missing ingredient is the identity itself: nothing in the stack mints a -generation today. Where it comes from is -[Plan: HLS identity](/quest/m2/plan-hls-identity.md)'s job. Whatever it turns -out to be, it must be minted by the publisher and travel with the content; -never a wall-clock or a relay-local counter, since a CDN needs one URL to mean -the same bytes on every edge. - -So this quest is the consumption: the moq-hls playlist and MPD renderers EMIT -generation-versioned `init.mp4` and segment URLs, and the cached init is -invalidated when the generation changes. Both are CONDITIONAL on the identity -being present: a publisher that declares none cannot version anything, so those -broadcasts keep unversioned URLs rather than being cached against an identity -that does not exist. The cache-header half is moq.pro (downstream) work: its -edge parses the versioned paths and restores the long `max-age` (where -`immutable` is finally true) once the renderers emit them; versioning the -accepted paths without versioning the emitted ones changes nothing. - -There are two collision sources, and a per-broadcast identity only covers one -of them. A live publisher that reconfigures a rendition (a resolution change, -say) reuses the same `init.mp4` URL and segment numbers for different bytes -without restarting the broadcast, so caching on a restart-scoped generation -alone would let a CDN keep serving the previous configuration. Either the URL -gains a rendition-generation component or the identity is required (and tested) -to move on every reconfigure. Settle it in -[Plan: HLS identity](/quest/m2/plan-hls-identity.md); it is the reason that -plan exists rather than a detail of this one. - -Acceptance: a publisher restart mid-playback exercised end to end, a publisher -declaring no identity still served with unversioned URLs, and a mid-broadcast -reconfigure that cannot be decoded against the previous configuration. - -## Required - -- [Plan: HLS identity](/quest/m2/plan-hls-identity.md) - a generation ID has to exist before a URL can carry it +Two collision sources, settled separately after a 2026-09 planning pass. + +**A live reconfigure** reuses `init.mp4` for different bytes: an +inline-parameter-set codec (`avc3`) that restarts at a new resolution keeps +its URL while `Rendition::init` caches the old init for the life of the +pooled rendition, and the segment window's `push` +(`rs/moq-hls/src/export/segments.rs`) resets only the playlist. Version the +init URL with a hash of the rendition config (codec string, dimensions, and +the parameter sets the init carries), so a reconfigure changes the URL and +the cached init is invalidated with it. A hash rather than a counter because +two edges rendering the same broadcast must agree without coordinating. + +**A publisher restart** reuses segment numbers: `moq-net` splices a +re-attached source into the existing broadcast by hop ID so consumers never +observe it, and group sequences restart at zero. No protocol carries an +identity for this and none will: an epoch on the announce messages was +rejected (announcements are capability routes that must keep stitching by hop +ID), and a `TRACK_INFO` epoch, a catalog `generation`, and an epoch path +segment were all declined in favor of the managed edge minting one per +publisher session downstream. So `moq-hls` gains an optional generation input, +one string per broadcast the embedder supplies, and when it is present the +playlist renderers in `rs/moq-hls/src/export` emit it in `init.mp4` and +segment URLs and the export resets its window when it changes. There is no +DASH renderer in the tree, so nothing here promises an MPD. Absent means unversioned URLs, never +a wall-clock or a relay-local counter. The cache-header half stays downstream: +the edge restores its long `max-age` once the paths it accepts are the ones the +renderers emit. + +Acceptance: a mid-broadcast reconfigure that cannot be decoded against the +previous init gets a new init URL on both edges; a supplied generation appears +in every media URL and a changed one resets the export; no generation keeps +the URLs byte-identical to today. diff --git a/quest/m2/id3.md b/quest/m2/id3.md index 69bb519f9c9..c7bf7c9e410 100644 --- a/quest/m2/id3.md +++ b/quest/m2/id3.md @@ -10,11 +10,14 @@ stream. ## Plan -Define the catalog section and frame contract for complete ID3 tags. Carry -original tag bytes, track identity, source presentation time when present, -the media rendition or timeline the tag accompanies, and the raw stream-level -PMT descriptor loop plus any application or program registration metadata the -exporter needs to reproduce the input. Do not parse the tag into a fixed +Define the catalog section and frame contract for complete ID3 tags, +following the sidecar rule in [SEI section](/quest/m2/sei/sei.md): the ID3 +track belongs to the program's video rendition (its audio rendition when there +is no video), uses that rendition's group sequence, and stamps each frame with +the wire timestamp of the tag's presentation time. Carry original tag bytes, +and in the section the raw stream-level PMT descriptor loop plus any +application or program registration metadata the exporter needs to reproduce +the input. Do not parse the tag into a fixed metadata vocabulary; applications can decode frames with their ID3 library and new tag types remain forward-compatible. diff --git a/quest/m2/path-patterns/origin.md b/quest/m2/path-patterns/origin.md index 7b540e484d6..afe0aa4f31a 100644 --- a/quest/m2/path-patterns/origin.md +++ b/quest/m2/path-patterns/origin.md @@ -25,6 +25,23 @@ beneath them. - Prove holes and suffix grants at the model layer, including concurrent announcements outside the grant never reaching a scoped consumer. +This is also the answer to the per-subscriber exclusion filter +[#2714](https://github.com/moq-dev/moq/issues/2714) asked for, to enforce +server-authoritative moderation (a deafened user's audio path) that a forked +client cannot bypass. A predicate over the announce stream cannot be that +boundary: announcements are prefix routes, so a filter decides about a set and +either hides paths the subscriber may use or leaks the ones it must not, and +`request_broadcast` resolves against the route table regardless. The +enforcing shape is a pattern-scoped grant on the consumer handle, narrowed at +resolution and announce fan-out alike, and changed at runtime through +[relay revalidation](/quest/m2/path-patterns/relay-auth.md); prove the deafen +case (a grant excluding one audio path under a room prefix) in the model-layer +tests here. + ## Required - [Matcher](/quest/m2/path-patterns/matcher.md) + +## Closes + +- [#2714](https://github.com/moq-dev/moq/issues/2714) - close this issue when the quest finishes diff --git a/quest/m2/plan-hls-identity.md b/quest/m2/plan-hls-identity.md deleted file mode 100644 index 9d2cfcc63cb..00000000000 --- a/quest/m2/plan-hls-identity.md +++ /dev/null @@ -1,56 +0,0 @@ -# [S] Plan: a generation identity for cacheable media URLs - -## Goal - -A settled answer to where a generation ID comes from, so -[HLS generation](/quest/m2/hls-generation.md) can start. The broadcast epoch -was going to supply it; -[moq#3225](https://github.com/moq-dev/moq/pull/3225) removed Epoch from the -drafts as spec-only and never implemented, retired -`draft-lcurley-moq-broadcast`, and made announcements prefix routes with no -per-broadcast identity to hang a generation on. - -## Plan - -Deliberately unsettled after a 2026-09 planning pass. The open question is -which layer owns the identity at all: moq-net, hang, or only a managed -deployment (moq.pro) that already keys recordings itself. Until that is -decided this quest stays a plan quest; do not start -[HLS generation](/quest/m2/hls-generation.md) against a guess. - -The pass settled the following points, so they do not need to be re-derived: - -- The requirement is unchanged: a CDN edge needs one URL to mean the same - bytes on every edge, so the ID has to be minted by the publisher and travel - with the content, never a wall-clock or a relay-local counter. -- The two collision sources stay distinct. A live rendition reconfigure under - the same track name is not banned by any rule (MSF forbids it, hang does - not) and `moq-hls` already tears the pooled rendition down and rebuilds it - when the catalog config changes, so only the URL reuse remains there. A - publisher restart is the harder half: `moq-net` splices a re-attached - source into the existing broadcast by hop ID so consumers never observe - it, and group sequences restart at 0. -- An epoch on ANNOUNCE_START / ANNOUNCE_UPDATE is rejected. Announcements are - capability routes, and a `transcode/*` route followed by a more specific - `transcode/foo` route must keep stitching by hop ID; a per-route epoch would - break that. `main`'s copy of `draft-lcurley-moq-lite.md` still carries the - old Epoch text on those messages and on TRACK, TRACK_INFO, SUBSCRIBE, and - FETCH; `dev` deleted all of it in [moq#3225](https://github.com/moq-dev/moq/pull/3225), - and `dev` is the draft this decision is made against. It was never - implemented on either branch. -- The candidates left, none chosen: an `Epoch` field on TRACK_INFO stamped - from `broadcast::Info` (per-broadcast value, per-track carriage, fits the - existing rule that TRACK_INFO is fixed for a track's lifetime, and a - takeover across epochs would end the subscription instead of splicing); a - hang catalog root `generation` (no wire change, routing-blind, so a - restart's groups can land on the old handle before the new catalog); an - epoch segment in the broadcast path under prefix routes (relays need - nothing, every consumer and token must resolve the newest, random values do - not order); or no protocol identity, with the managed edge minting one per - publisher session downstream. -- If a wire epoch is chosen it folds into lite-06-wip, is minted at random - on `broadcast::Info` with an application override so an archive replays a - recording under its recorded value. On moq-lite it rides TRACK_INFO (lite's - SUBSCRIBE_OK carries only the resolved group); on moq-transport, which has - no TRACK_INFO, it rides a SUBSCRIBE_OK parameter, and the adapter maps - both onto the same `track::Info` field. diff --git a/quest/m2/plan-revalidation-updates.md b/quest/m2/plan-revalidation-updates.md new file mode 100644 index 00000000000..5cd9495dd5b --- /dev/null +++ b/quest/m2/plan-revalidation-updates.md @@ -0,0 +1,34 @@ +# [S] Plan: what an auth re-check may update in place + +## Goal + +A settled contract for which fields a revalidation reply can change on a live +session and which force a reconnect. Scope narrowing is settled (close, and +the client reconnects into the narrower grant); `tier` and `alias` are not. +Run `/plan-quest`; the settled plan becomes the implementing quest that closes +the issue. + +## Plan + +`Auth::recheck` scores a reply by `Scope::covered_by` (root, subscribe, +publish), then drops the `AuthToken` and propagates only the `CacheHints`. Two +fields the auth API can legitimately change therefore behave inconsistently: + +- `tier` is silently ignored. An endpoint that re-buckets a connection onto or + off a billing tier has no effect until the session reconnects, and the tier + decides which meter pays. Applying it mid-session means rebuilding the + session's stats handle, and usage already recorded under the old tier stays + there. +- `alias` closes the session, because the alias becomes the token's `root` and + `covered_by` fails. Arguably correct, since the broadcast would otherwise keep + announcing under a root the API no longer assigns, but it is a silent hard + disconnect for what may be a benign rename. + +Decide deliberately, per field: update in place, close, or refuse the change. +Then the implementing quest applies it in `rs/moq-relay/src/auth.rs` with a +test per field and documents the contract in `doc/bin/relay/auth.md`. + +## Related + +- [#3058](https://github.com/moq-dev/moq/issues/3058) - the issue the implementing quest closes +- [Auth verdict](/quest/m2/auth-verdict.md) - the proxy mode whose re-check this also governs diff --git a/quest/m2/pop-skipping/README.md b/quest/m2/pop-skipping/README.md index e77ef3ec8d0..4ec79010012 100644 --- a/quest/m2/pop-skipping/README.md +++ b/quest/m2/pop-skipping/README.md @@ -155,4 +155,4 @@ costs of one bidirectional session, which one `?cost=` cannot split. - [drain](/quest/m2/drain/README.md) - a second relay per PoP makes the same-PoP link price and its connection cardinality operationally important - [wildcard](/quest/m2/wildcard/README.md) - it reuses this questline's route cost, and needs a cluster on Lite06 -- [relay-memory](/quest/m2/relay-memory/README.md) - a denser mesh multiplies whatever a non-selected route costs +- [relay-memory](/quest/m2/relay-memory.md) - a denser mesh multiplies whatever a non-selected route costs diff --git a/quest/m2/qos/starvation.md b/quest/m2/qos/starvation.md index 1c98cf3c8ae..419f3cbc2ee 100644 --- a/quest/m2/qos/starvation.md +++ b/quest/m2/qos/starvation.md @@ -79,11 +79,6 @@ already taken while it was outstanding; a session close mid-group counts as dropped; two subscriptions of one broadcast sum into one row; the aggregate consumer sums two nodes bucket by bucket. -## Closes - -- [#2733](https://github.com/moq-dev/moq/issues/2733) - close this issue when - the quest finishes - ## Related - [Starvation at frame granularity](/quest/m2/qos/starvation-frames.md) - diff --git a/quest/m2/relay-memory.md b/quest/m2/relay-memory.md new file mode 100644 index 00000000000..3c9a3508525 --- /dev/null +++ b/quest/m2/relay-memory.md @@ -0,0 +1,37 @@ +# [S] Relay memory per announcement + +## Goal + +A relay's memory scales with what it serves, not with what the mesh knows, +and the numbers behind that claim are current. + +## Plan + +Every published figure predates two changes and is untrustworthy: + +- [moq#2989](https://github.com/moq-dev/moq/pull/2989) cut `kio`'s inline + waiter slots from 32 to 4, so a `kio::State<()>` went from 896 B to about + 200 B. Going lower trades memory for an allocation per wake, which + `kio`'s `tests/waiter_allocs.rs` pins; that lever is spent. +- [moq#3225](https://github.com/moq-dev/moq/pull/3225) made a received + announcement a `RouteEntry` plus one `ServeState` whose cache materializes + a broadcast only when something requests a path. A standby route is a + table entry, not an object graph, so degree stopped being a multiplier. + +The old baseline was 8.8 KB per announced broadcast plus 4.3 KB per extra +route on `adad52b`, measured with two throwaway `moq-net` examples driving an +origin under a counting allocator and reading `/proc/self/statm`. Neither is +committed, since they need `#[doc(hidden)]` size probes on private types. +Rebuild them from this description and restate the per-broadcast and +per-route cost, the per-peer session bookkeeping (`announce_ids`, `held`, +`watched`), and the shed threshold on a degree-5, 1 GB node, before anyone +quotes a number again. Two committed directions depend on the answer: +chat-shaped traffic (one broadcast per channel or per chatter) and +[PoP skipping](/quest/m2/pop-skipping/README.md), which triples average +degree and adds a second, more specific route per carried broadcast. + +## Related + +- [Routes per broadcast gauge](/quest/m2/route-gauge.md) - the operator-facing count, shippable on its own +- [Group charge](/quest/m0/group-charge.md) - the group cache charges what a cached group really costs; a group holds one of the same state cells +- [Perf](/quest/m1/perf/README.md) - the hot-path work that owns the remaining per-cell cost diff --git a/quest/m2/relay-memory/README.md b/quest/m2/relay-memory/README.md deleted file mode 100644 index 2b80393af74..00000000000 --- a/quest/m2/relay-memory/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Relay memory per announcement - -## Goal - -A relay's memory scales with what it serves, not with what the mesh knows. -Every relay currently materializes a full broadcast object graph for every -announcement anywhere in the cluster, subscribed or not, once per neighbor that -advertises it. - -Measured on `moq-net` (`adad52b`) with an RSS plus allocator-counting harness: -8.8 KB per announced broadcast with one route, plus 4.3 KB per additional -route, in ~30 allocations and 2 spawned tasks. The announcement itself (the -path string) is 18 B of that, and `PathOwned` already shares one `Arc` -across every copy, so the wire object is not the problem. - -## Plan - -### Root cause - -`kio::State` carries three `WaiterList`s, each an inline array of weak wakers -paid unconditionally inside the `Arc>`. A broadcast holds four to -five of those cells (the source's `Shared` and `Alive`, the front's `Shared` and -`Alive`, and `FrontState`), so empty waker slots used to dominate its footprint: -at 32 inline slots a `WaiterList` was 296 B and `size_of::>()` was 896 B -before `T`. - -That lever is spent. -[moq#2989](https://github.com/moq-dev/moq/pull/2989) cut the inline count to 4, -which is 64 to 72 B and 200 to 224 B respectively, the range depending on -whether anything in the build graph enables `smallvec/union`. Going lower trades -memory for an allocation on every wake, because `WaiterList::take()` hands its -spilled buffer to the snapshot that wakes it, so a spilled list re-allocates per -cycle rather than keeping capacity. `kio`'s `tests/waiter_allocs.rs` and -`the_list_stays_small` now pin both sides of that trade. - -The remaining 48 B per cell is therefore gated on [cheaper parking and -waking](/quest/m1/perf/kio-wake.md), which owns the buffer reuse that would make -a smaller inline count free. Shrinking the slots before that lands only moves -cost from memory to the fan-out path. - -The per-route half is closed. It used to be that every received announcement ran -the whole `create_broadcast` path, minting its own `broadcast::Producer` and -lifecycle task even for a standby route that was never selected, so a relay of -degree `d` attached `d` sources to one front and degree was a straight -multiplier. [moq#3225](https://github.com/moq-dev/moq/pull/3225) removed that: -`lite::subscriber` now calls `origin::Producer::announce_served`, so an -announcement costs a `RouteEntry` plus one `ServeState` whose `served` cache -materializes a broadcast only when something requests a path. That is what the -standby-routes quest asked for, delivered as a side effect, and the quest is -closed. - -A group holds one of the same cells, which is why [group -charge](/quest/m0/group-charge.md) is an accounting bug and not only a footprint one. - -### What it costs today - -Per broadcast at degree `d` a relay pays `8.8 KB + (d - 1) * 4.3 KB` of origin -state, plus about `d * 0.5 KB` of per-peer session bookkeeping: every peer we -advertise the path to holds an `announce_ids` entry (32 B), a `held` entry -(144 B) and a `watched` entry (248 B), plus hashbrown's load factor. That -second term is summed from `size_of` rather than measured end to end, unlike -the origin figures, so treat it as good to a KB rather than exact. On a 1 GB -node at degree 5 (moq.pro's downstream APAC hub is exactly that, with half of -RAM given to the group cache and health shedding at 75%), announce state has -roughly 200 MB of headroom against a warm cache, so the node sheds itself out -of serving near 7,000 concurrent cluster-wide broadcasts. - -Every figure above predates both prefix routes and the inline-slot cut, so none -of it is trustworthy: it was measured on `adad52b`, where an announcement built -a broadcast out of 896 B state cells. Remeasure before treating any of it as -current. - -### Why now rather than later - -Nothing is near the ceiling today. Two committed directions move it: - -- Chat-shaped traffic (a moq.pro downstream direction) names announcement - churn as the control-plane risk. One broadcast per channel is ~10k - (~270 MB on a 1 GB hub, at the shed threshold); one per chatter is ~700k - (~19 GB, not a tuning problem). -- [Cache-aware PoP skipping](/quest/m2/pop-skipping/README.md) densifies the - relay mesh, roughly tripling average degree on the topology it was designed - against, and its warm advertisement adds a second, more specific route per - carried broadcast. Both multiply whatever a route costs, which is now a table - entry rather than an object graph. - -### Expected result - -The published table compared today, waiter slots, and standby routes at ~29 KB, -~19 KB, and ~9 KB per cluster-wide broadcast on a degree-5, 1 GB relay. Both of -those levers have since landed by other routes, and the baseline they were -measured against no longer exists, so no claim survives. Rebuild the harness and -restate the table before quoting a shed threshold. - -### Reproducing the measurements - -Two throwaway `moq-net` examples drive an origin and read `/proc/self/statm` -alongside a counting global allocator, with `N` paths, `ROUTES` sources per -path (distinct second hops so they splice onto one front), and `PEERS` announce -cursors. Neither is committed: they need `#[doc(hidden)]` size probes on private -types, and the numbers above are what they were written to produce. Rebuild them -from this quest rather than expecting them in the tree. - -## Quests - -- [Routes per broadcast gauge](/quest/m2/relay-memory/route-gauge.md) - expose - how many routes a relay holds for a path - -## Related - -- [Group charge](/quest/m0/group-charge.md) - the group cache charges what a - cached group really costs; it undercounts chat-shaped traffic today diff --git a/quest/m2/room-sdk.md b/quest/m2/room-sdk.md index 5e8170b8f47..6e0032b4b83 100644 --- a/quest/m2/room-sdk.md +++ b/quest/m2/room-sdk.md @@ -37,6 +37,14 @@ AccessToken analogue). hang.live migrates onto the package as proof. a crude demo): a headless library, no element. Custom elements and framework bindings are follow-ups if wanted. +The conferencing demo [#697](https://github.com/moq-dev/moq/issues/697) +asked for is this package's proof: hang.live's room, migrated onto it, is the +demo, and `` is not coming back as an element. + +## Closes + +- [#697](https://github.com/moq-dev/moq/issues/697) - close this issue when the quest finishes + ## Related - [LiveKit client shim](/quest/m2/livekit-shim.md) - builds its diff --git a/quest/m2/relay-memory/route-gauge.md b/quest/m2/route-gauge.md similarity index 62% rename from quest/m2/relay-memory/route-gauge.md rename to quest/m2/route-gauge.md index 603f507ef2d..d4a2f31ab55 100644 --- a/quest/m2/relay-memory/route-gauge.md +++ b/quest/m2/route-gauge.md @@ -7,12 +7,11 @@ An operator can see how many routes a relay holds for a path. moq.pro's ## Plan -This rode the standby-routes quest as a passenger, because that quest had the -route table open anyway. [moq#3225](https://github.com/moq-dev/moq/pull/3225) -completed standby routes by making a non-selected route a table entry, so the -quest was closed and the accessor needs its own change. - Expose the count of routes covering a path from `origin`, next to whatever `best_server` already walks, and surface it wherever `moq-relay` reports node state. Prefer a count over exposing the entries themselves: the caller is a gauge, and a routes iterator is a much larger surface to keep stable. + +## Related + +- [Relay memory](/quest/m2/relay-memory.md) - the remeasurement this gauge helps read diff --git a/quest/m2/sei/README.md b/quest/m2/sei/README.md index c3edf450bec..8c2bfb0d503 100644 --- a/quest/m2/sei/README.md +++ b/quest/m2/sei/README.md @@ -33,9 +33,10 @@ still joins two independently scheduled tracks, so it owns a stated buffer budget and reports what missed it. An old exporter drops SEI from its output; upgrading is the answer. -Correlate by group sequence and frame ordinal, which is exact by construction, -and carry the video frame's timestamp alongside as data so an application -syncing to presentation time gets it without subscribing to video. Preserve the +Correlate by the video group sequence and the frame's wire timestamp, with +the frame ordinal disambiguating access units that share a timestamp, so an +application syncing to presentation time gets it without subscribing to video. +This is the sidecar rule every timed-metadata section shares. Preserve the original NAL bytes, prefix or suffix placement, and in-access-unit order so reinsertion is byte-faithful without interpreting payload types. Placement must be exact: misplaced `recovery_point` misdirects tune-in, and reordered caption diff --git a/quest/m2/sei/sei.md b/quest/m2/sei/sei.md index 8565deeee8c..dfdbc78a5db 100644 --- a/quest/m2/sei/sei.md +++ b/quest/m2/sei/sei.md @@ -9,16 +9,21 @@ subscribes to the sidecar alone. ## Plan -Specify one sidecar track per video rendition, or an equally unambiguous -mapping, and key each sample by the video group sequence and frame ordinal. -That identity is exact by construction and independent of how a splitter -assigns clocks, which a timestamp is not: on the raw Annex B path -`h264::Split::decode` resolves one wall-clock value per call and gives it to -every access unit in that chunk, so several frames can share a timestamp. - -Carry the video frame's timestamp on the sample as well, as data rather than -identity. An application syncing to presentation time reads it directly instead -of joining against the video track it deliberately did not subscribe to. +This section defines the sidecar rule every timed-metadata section follows +(ID3, SCTE-35, emsg, FLV script tags, AV1 metadata OBUs each keep their own +section but share the shape): a metadata track belongs to exactly one +rendition, uses that rendition's group sequence, stamps each frame with the +wire timestamp of the media it accompanies, and carries raw bytes. A 1080p and +a 360p rendition carry different SEI, so there is one sidecar per video +rendition, and group 7 of the sidecar holds the SEI for group 7 of its video. + +Within a group the frame's wire timestamp is the key, so an application +syncing to presentation time reads it directly instead of joining against the +video track it deliberately did not subscribe to. Several access units can +share a timestamp on the raw Annex B path (`h264::Split::decode` resolves one +clock value per call and gives it to every access unit in that chunk), so the +frame's ordinal within the group disambiguates those; it is a tie-break, not +the identity. Preserve prefix or suffix placement, original NAL bytes, and order when several SEI units accompany one access unit. The codec is the mapped video diff --git a/quest/m2/timeline-wall.md b/quest/m2/timeline-wall.md new file mode 100644 index 00000000000..7314e4232b9 --- /dev/null +++ b/quest/m2/timeline-wall.md @@ -0,0 +1,43 @@ +# [S] Publishers anchor the timeline to wall time + +## Goal + +Every built-in publisher sets `Timeline.wall` so a consumer can read when +presentation timestamp zero happened on the publisher's clock: the moq-video +capture publisher, `moq_audio::encode::publish_capture` and its public +`Producer` (an audio-only broadcast advertises a timeline too), the moq-cli +import paths, and `js/publish`. moq-gst has its own quest for the same +anchor. + +The anchor is data, not a synchronization source. Nothing in the library +times playback against it: frame timestamps stay relative, and two machines' +wall clocks are only comparable when the application already knows they are +synced. Metrics and applications that control both ends may inspect it. + +## Plan + +`moq_mux::timeline::Producer::set_wall(pts, SystemTime)` and the JS +`setWall(pts, Date)` exist and nothing outside tests calls them. Anchor on the +first frame each publisher emits, with the wall time it observed that frame, +and republish the rendition's catalog entry once the anchor is set. + +One `wall` value describes one linear mapping, so a declared discontinuity +that breaks linearity (a seek, a source restart) must not overwrite it: +records already advertised on the timeline track would then resolve against +the wrong epoch in DVR, HLS `PROGRAM-DATE-TIME`, and correlation consumers. +Instead the first timeline record after a discontinuity carries the new +anchor itself, and a consumer resolves a record against the latest anchor at +or before it. The catalog `wall` stays the anchor for records before the +first such carried one. That is a record-schema addition, so it lands with a +`drafts/draft-lcurley-moq-hang.md` update in the same PR. + +Tests: each publisher's catalog carries `wall` after its first frame; +`(wall + pts) / timescale` resolves to the observed duration since the MoQ +epoch; after a discontinuity, records before it still resolve against the old +anchor and records after it against the one they carry. + +## Related + +- [#3021](/quest/m2/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - the moq-gst half, with the reference-timestamp rules +- [#2278](/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md) - exposes the anchor to browser applications +- [Cross-track correlation](/quest/m3/teleop/correlation.md) - reads the anchor on machines that share a clock diff --git a/quest/m2/wildcard/README.md b/quest/m2/wildcard/README.md index e947eed3e64..9f7ef4d44bc 100644 --- a/quest/m2/wildcard/README.md +++ b/quest/m2/wildcard/README.md @@ -17,13 +17,13 @@ subtree pattern `/chat/**` expresses them. An archive serving recordings over FETCH wants to say "if nobody is publishing this live, I have it", which is the catch-all `**`, a claim about every path at once. -The cost of enumerating is measured, not assumed. -[relay-memory](/quest/m2/relay-memory/README.md) puts one announcement at -8.8 KB per relay with one route, plus 4.3 KB per additional route, and every -relay that hears it materializes that whether or not anything there subscribes. -So "workers times broadcasts" is not just a large number of messages; it is that -number multiplied across the fleet in resident memory, which is the same growth -that questline exists to stop. +The cost of enumerating is real even though its last measurement is stale. +[relay-memory](/quest/m2/relay-memory.md) measured one announcement at 8.8 KB +per relay plus 4.3 KB per additional route before prefix routes made a +standby route a table entry, and owns remeasuring it. Whatever the current +number, every relay that hears an announcement pays it whether or not anything +there subscribes, so "workers times broadcasts" is that number multiplied +across the fleet in resident memory. ## Plan diff --git a/quest/m3/697-conferencing-demo.md b/quest/m3/697-conferencing-demo.md deleted file mode 100644 index b1795235607..00000000000 --- a/quest/m3/697-conferencing-demo.md +++ /dev/null @@ -1,22 +0,0 @@ -# [M] Conferencing Demo - -## Goal - -Implement and verify the behavior tracked in [#697](https://github.com/moq-dev/moq/issues/697) -within the issue's stated scope and boundaries. - -## Plan - -Rescoped during the 2026-08 grooming: the hang-meet component this issue -builds on no longer exists. Compose the demo from the current js/publish, -js/watch, and demo/web primitives instead. - -### Issue context - -There is a [hang-meet](https://github.com/kixelated/moq/tree/main/js/hang/src/meet) web component but it's extremely simple. - -We should steal the best, generic stuff from `hang.live` to create a proper conferencing demo. I could set up the rendering but I would want some help on the rest, especially the UI. Let me know if you're interested and I can set up the base. - -## Closes - -- [#697](https://github.com/moq-dev/moq/issues/697) - close this issue when the quest finishes diff --git a/quest/m3/README.md b/quest/m3/README.md index 4f62fa49e64..0d88cca6a0b 100644 --- a/quest/m3/README.md +++ b/quest/m3/README.md @@ -21,7 +21,6 @@ its verdict lands and the follow-on work becomes concrete. - [QUIC GCC](/quest/m3/quic-gcc.md) - a measured verdict on delay-based congestion control for media egress - [QUIC FEC](/quest/m3/quic-fec.md) - a measured verdict on transport-level FEC vs retransmission - [GOP overhead](/quest/m3/gop-overhead.md) - price the I-frames a short GOP pays for, deciding whether a long GOP plus a keyframe request is worth designing -- [#697](/quest/m3/697-conferencing-demo.md) - Conferencing Demo - [#703](/quest/m3/703-experimental-webgpu-renderer.md) - Experimental WebGPU renderer - [#823](/quest/m3/823-svc-support.md) - SVC support? - [#1838](/quest/m3/1838-tr-101-290-monitoring-requirements-broadcast-contribution.md) - TR 101 290 monitoring: requirements (broadcast/contribution health metrics) diff --git a/quest/m3/multipath-spike.md b/quest/m3/multipath-spike.md index 32aef8cc5ec..6eb85802ab4 100644 --- a/quest/m3/multipath-spike.md +++ b/quest/m3/multipath-spike.md @@ -28,11 +28,12 @@ browser WebTransport backend could never implement it. **What the spike is really for is the constraint nobody has written down.** `max_concurrent_multipath_paths` maps to the `initial_max_path_id` transport parameter, which both peers negotiate, so a multipath session needs noq on -*both* ends. noq is not the default backend on either branch -(`default = ["quinn", ...]`), and no other backend has the extension: quinn -0.11 lacks it, quiche's `probe_path`/`migrate` is RFC 9000 single-path -migration, and iroh's path logic is its own. So bonded contribution is not an -opt-in client feature; it constrains the relay too. Establish what that +*both* ends. noq is the default backend on `dev` (`moq-tokio`, `moq-uring`, +and `moq-relay` all default to it) but `main` still defaults to quinn, and no +other backend has the extension: quinn 0.11 lacks it, quiche's +`probe_path`/`migrate` is RFC 9000 single-path migration, and iroh's path +logic is its own. So bonded contribution is not an opt-in client feature; it +constrains every relay and client on the path to run noq. Establish what that actually costs before anyone builds a fan-out socket. Prove it on a multi-homed host, which is enough to negotiate and open a second @@ -66,7 +67,3 @@ Target `dev`, where the crate is `moq-tokio`; the rename has not reached - [Establish the noq relationship](/quest/m2/quic/parent.md) - noq's multipath support is one reason to invest in the upstream relationship - -## Closes - -- [#2276](https://github.com/moq-dev/moq/issues/2276) - close this issue when the quest finishes diff --git a/quest/m3/teleop/README.md b/quest/m3/teleop/README.md index 858ed6cb6f1..ba9d767a372 100644 --- a/quest/m3/teleop/README.md +++ b/quest/m3/teleop/README.md @@ -105,8 +105,6 @@ stating plainly because it is what a builder is comparing against. - [Cross-track correlation](/quest/m3/teleop/correlation.md) - a command, the telemetry it produced, and the video frame showing the result share one timebase -- [Validate VAAPI encoding](/quest/m3/teleop/vaapi.md) - prove `moq-video`'s - VAAPI backend on real hardware so it stops being opt-in ## Related @@ -116,3 +114,5 @@ stating plainly because it is what a builder is comparing against. - [Publisher-reported media stats](/quest/m2/qos/publisher-stats.md) - publisher-reported stats as a catalog section (moq#2734); teleop's latency instrumentation extends that track rather than adding a second one +- [Video hardware validation](/quest/m3/video-hardware.md) - the VAAPI run + that covers Intel ground robots and NUC companions diff --git a/quest/m3/teleop/correlation.md b/quest/m3/teleop/correlation.md index 9c26e96c031..3bb6f4a5fda 100644 --- a/quest/m3/teleop/correlation.md +++ b/quest/m3/teleop/correlation.md @@ -8,23 +8,26 @@ data and an operator can be shown what the machine actually saw. ## Plan -The hard part is time transfer between two hosts, and it is the part with no -existing answer. `rs/moq-mux/src/clock.rs` is a process-local `Instant` epoch -for relating capture threads inside one process; it has no wall anchor and -cannot relate an operator's command timestamp to a robot's frame timestamp. The -wall anchor is `timeline.rs`'s `Producer::set_wall`. - -So the open question is how the two clocks are related: assume NTP on both -ends, derive an offset from session RTT, or stamp only at the robot and have -the operator echo its stamps back the way `moq-boy`'s `status` track already -does. Settle that before designing the contract; the third option needs no -external dependency and is the one already proven in-tree. +The two hosts' clocks are related by assumption, not by the library: a robot +deployment already runs NTP or PTP on both ends, and the `moq-robot` contract +says so. Each track's `Timeline.wall` anchor is the bridge, read as data from +the catalog; the library offers no clock-sync mechanism and the crate must +not add one (see [#2278](/quest/m2/2278-watch-absolute-wall-clock-latency-target-for-synchronized.md) +for the decision). `rs/moq-mux/src/clock.rs` is a process-local `Instant` +epoch for relating capture threads inside one process and cannot relate an +operator's command to a robot's frame; `timeline.rs`'s `set_wall` is what can. + +So the contract is: the robot anchors its video and telemetry timelines, the +operator anchors its command track, and a recording joins the three on +`wall + pts`. State the assumption beside the API, and report the anchor's +presence so a consumer can tell "unsynced" from "synced". This is also the answer to Kyber's headline claim of continuous drift computation onto one unified timeline. Worth answering on the merits: -correlating sensor, command and video is the actual product need, and it is the -same property that makes an MCAP recording valuable. +correlating sensor, command and video is the actual product need, and it is +the same property that makes an MCAP recording valuable. ## Required - [Robot teleoperation primitive](/quest/m3/teleop/robot.md) +- [Publishers anchor the timeline](/quest/m2/timeline-wall.md) - the anchor every track needs before it can be joined diff --git a/quest/m3/teleop/v4l2-encode.md b/quest/m3/teleop/v4l2-encode.md index ad2cda784c1..c53c8c4a6c3 100644 --- a/quest/m3/teleop/v4l2-encode.md +++ b/quest/m3/teleop/v4l2-encode.md @@ -39,4 +39,4 @@ codes as 1088 rows and the compose rectangle crops it back, and the other ## Related -- [vaapi](/quest/m3/teleop/vaapi.md) - the other unproven encoder backend +- [Video hardware validation](/quest/m3/video-hardware.md) - the other unproven encoder backend, VAAPI diff --git a/quest/m3/teleop/vaapi.md b/quest/m3/teleop/vaapi.md deleted file mode 100644 index b473f081374..00000000000 --- a/quest/m3/teleop/vaapi.md +++ /dev/null @@ -1,18 +0,0 @@ -# [M] Validate VAAPI encoding - -## Goal - -`moq-video`'s VAAPI backend is proven on real hardware and stops being opt-in -for the reason it is opt-in today. - -## Plan - -The backend is in-tree and its own comment says NOT YET VALIDATED ON HARDWARE, -which is why the `vaapi` feature is off by default. Run it on Intel and AMD -graphics, fix what it finds, and either turn the feature on or record what -still blocks it. - -Independent of the V4L2-M2M work: different vendor, different code path, -different hardware to test on. It covers Intel-based ground robots and NUC -companions, which is why it belongs to teleoperation rather than sitting -unowned. diff --git a/quest/m3/video-hardware.md b/quest/m3/video-hardware.md index 75d5eb59da9..8a32f8c72ac 100644 --- a/quest/m3/video-hardware.md +++ b/quest/m3/video-hardware.md @@ -11,7 +11,11 @@ Every item here is blocked on a physical machine rather than on code, which is why they sit together and why they sit in m3. - **VAAPI encode on an Intel or AMD box**: low-power against full entrypoint, - the NV12 upload round trip, and `cargo deny` license resolution. + the NV12 upload round trip, and `cargo deny` license resolution. The + backend's own comment says NOT YET VALIDATED ON HARDWARE even though the + `vaapi` feature is on by default (moq-vaapi dlopens libva), so this run + either clears that comment or records what still blocks it. It covers the + Intel-based ground robots and NUC companions the teleoperation line needs. - **VAAPI zero-copy dmabuf input**: the backend uses an NV12 surface upload today. Exercise the `Surface::DmaBuf` path with a V4L2 `VIDIOC_EXPBUF` source instead. From c2c5558de24a9e003a4964832f5d87797086d1d2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:30:59 -0700 Subject: [PATCH 16/29] chore(quest): claim 2985 js/net path-keyed publisher state Claim UUID: cc3f0536-0789-44ea-8321-fb8e0f4d734e From babac7c6e23b347885968e85713740f6da424ba8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:44:51 -0700 Subject: [PATCH 17/29] fix(js/net): key the publisher's TRACK_INFO cache by broadcast, not path The lite publisher memoized each track's immutable TRACK_INFO in a Map keyed by `broadcast\0track`. The key is a path, and a path outlives the broadcast on it: `Origin.publish` swaps the front under an existing key, so a republish left the entry in place and every later TRACK request was answered with the predecessor's priority, maxAge, and timescale. FETCH reads the same cache to pick the timescale it converts frame timestamps into, so the successor's frames were quantized onto the predecessor's grid: a 1234us timestamp on a MICRO track came back as 1000us once a MILLI predecessor had primed the cache. SUBSCRIBE resolves `track.info()` fresh, so the two disagreed about the same track on the same connection. Key the cache on the routing front instead. Immutability holds for one broadcast, which is exactly the lifetime a WeakMap keyed by the front gives it, and it is the same identity the announce loop already diffs on. `runFetch` now passes the front it already resolved, so the metadata and the frames come off one generation rather than two independent map lookups that a replacement can land between. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLtoPv9B3779kGxLFTUGHR --- js/net/src/integration.test.ts | 80 +++++++++++++++++ js/net/src/lite/publisher.ts | 57 ++++++------ ...keyed-publisher-state-goes-stale-when-a.md | 86 ------------------- quest/m1/README.md | 1 - quest/m1/js-announce.md | 1 - 5 files changed, 112 insertions(+), 113 deletions(-) delete mode 100644 quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index e90f4a1a828..bec59644c1f 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -638,6 +638,86 @@ test("integration: lite draft-05 fetches an in-progress group", async () => { server.close(); }); +// The publisher caches TRACK_INFO so it only asks the application once per track. The cache +// has to expire with the broadcast that answered it: a republish puts a different producer +// on the path, and its immutable properties are its own. +test("integration: lite draft-05 track info follows a republished broadcast", async () => { + const pair = createMockTransportPair(Lite.ALPN_05); + const origin = new OriginProducer(); + + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { publish: origin.consume() }), + ]); + + const path = Path.from("test"); + const first = origin.publish(path); + first.createTrack("video", { priority: 1, timescale: Timescale.MILLI, maxAge: 1000 }); + + const remote = client.consume(path); + const before = await remote.track("video").info(); + expect(before.priority).toBe(1); + expect(before.timescale).toBe(Timescale.MILLI); + expect(before.maxAge).toBe(1000); + + // Replace the broadcast on the same path with one whose track declares different + // immutable properties. + const second = origin.publish(path); + second.createTrack("video", { priority: 7, timescale: Timescale.MICRO, maxAge: 5000 }); + + const after = await remote.track("video").info(); + expect(after.priority).toBe(7); + expect(after.timescale).toBe(Timescale.MICRO); + expect(after.maxAge).toBe(5000); + + first.close(); + second.close(); + remote.close(); + client.close(); + server.close(); +}); + +// FETCH reads the same cache to decide the timescale it serves frames in, so a stale entry +// quantizes the successor's timestamps to the predecessor's resolution. +test("integration: lite draft-05 fetch uses the republished track's timescale", async () => { + const enc = new TextEncoder(); + const pair = createMockTransportPair(Lite.ALPN_05); + const origin = new OriginProducer(); + + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { publish: origin.consume() }), + ]); + + const path = Path.from("test"); + const first = origin.publish(path); + const firstTrack = first.createTrack("video", { timescale: Timescale.MILLI }); + const firstGroup = firstTrack.appendGroup(); + firstGroup.writeFrame({ payload: enc.encode("alpha"), timestamp: Timestamp.fromMillis(10) }); + firstGroup.close(); + + // Prime the publisher's TRACK_INFO cache against the predecessor. + const remote = client.consume(path); + expect((await remote.track("video").info()).timescale).toBe(Timescale.MILLI); + + const second = origin.publish(path); + const secondTrack = second.createTrack("video", { timescale: Timescale.MICRO }); + const secondGroup = secondTrack.appendGroup(); + secondGroup.writeFrame({ payload: enc.encode("beta"), timestamp: Timestamp.fromMicros(1234) }); + secondGroup.close(); + + // A millisecond timescale would round this to 1000us. + const fetched = await remote.track("video").fetchGroup(0); + const frame = await fetched.readFrame(); + expect(frame?.timestamp.asMicros()).toBe(1234); + + first.close(); + second.close(); + remote.close(); + client.close(); + server.close(); +}); + test("integration: ietf fetch group is unsupported", async () => { const pair = createMockTransportPair(Ietf.ALPN.DRAFT_18); const origin = new OriginProducer(); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index b8320f3ff12..2074c3afae8 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -327,9 +327,11 @@ export class Publisher { // TRACK_INFO is immutable per track, so resolve it from the application once // (via a throwaway subscribe whose info() resolves when the app calls accept) - // and reuse it for every later TRACK request of the same track. Keyed by - // `broadcast\0track`. A rejected lookup is evicted so a retry can re-probe. - #trackInfo = new Map>(); + // and reuse it for every later TRACK request of the same track. Keyed by the + // routing front rather than the path: immutability holds for one broadcast, and a + // republish puts a different one on the path, so its entries must not be reused. + // A rejected lookup is evicted so a retry can re-probe. + #trackInfo = new WeakMap>>(); /** * Creates a new Publisher instance. @@ -480,14 +482,14 @@ export class Publisher { * @internal */ async runSubscribe(msg: Subscribe, stream: Stream) { - const broadcast = this.#broadcasts.peek()?.get(msg.broadcast); - if (!broadcast) { + const front = this.#broadcasts.peek()?.get(msg.broadcast); + if (!front) { console.debug(`publish unknown: broadcast=${msg.broadcast}`); stream.writer.reset(new Error("not found")); return; } - const track = broadcast.subscribe(msg.track, { + const track = front.subscribe(msg.track, { priority: msg.priority, maxAge: servingMaxAge(this.version, msg.maxAge), startGroup: msg.startGroup, @@ -586,8 +588,8 @@ export class Publisher { return; } - const broadcast = this.#broadcasts.peek()?.get(msg.broadcast); - if (!broadcast) { + const front = this.#broadcasts.peek()?.get(msg.broadcast); + if (!front) { console.debug(`fetch unknown: broadcast=${msg.broadcast}`); stream.writer.reset(new Error("not found")); return; @@ -599,9 +601,10 @@ export class Publisher { let group: group.Consumer | undefined; try { - // The timescale is immutable, so serve exactly what TRACK_INFO advertised. - const info = await this.#resolveTrackInfo(msg.broadcast, msg.track); - group = await broadcast.track(msg.track).fetchGroup(msg.group, { priority: msg.priority }); + // The timescale is immutable, so serve exactly what TRACK_INFO advertised. Both + // come off the same front, so the metadata and the frames are one generation. + const info = await this.#resolveTrackInfo(front, msg.track); + group = await front.fetchGroup(msg.track, msg.group, { priority: msg.priority }); await this.#runFetchGroup(group, stream.writer, { timescale: Timescale(info.timescale), start: msg.startFrame, @@ -774,7 +777,10 @@ export class Publisher { */ async runTrackInfo(msg: TrackMessage, stream: Stream) { try { - const info = await this.#resolveTrackInfo(msg.broadcast, msg.track); + const front = this.#broadcasts.peek()?.get(msg.broadcast); + if (!front) throw new Error("not found"); + + const info = await this.#resolveTrackInfo(front, msg.track); await info.encode(stream.writer, this.version); console.debug(`track info: broadcast=${msg.broadcast} track=${msg.track}`); stream.close(); @@ -785,20 +791,21 @@ export class Publisher { } // Resolve (and cache) a track's immutable TRACK_INFO by asking the application. - // `broadcast.track(name).info()` triggers a TrackRequest the app answers with - // accept(TrackInfo); only the immutable properties are needed (not the groups). - // Cached because they're fixed for the track's lifetime. Rejects if the broadcast - // or track is unavailable. - #resolveTrackInfo(broadcast: Path.Valid, track: string): Promise { - const key = `${broadcast}\0${track}`; - const cached = this.#trackInfo.get(key); + // `resolveTrackInfo` triggers a TrackRequest the app answers with accept(TrackInfo); + // only the immutable properties are needed (not the groups). Cached because they're + // fixed for the track's lifetime. Rejects if the track is unavailable. + #resolveTrackInfo(front: broadcast.Consumer, track: string): Promise { + let tracks = this.#trackInfo.get(front); + if (!tracks) { + tracks = new Map(); + this.#trackInfo.set(front, tracks); + } + + const cached = tracks.get(track); if (cached) return cached; const pending = (async () => { - const published = this.#broadcasts.peek()?.get(broadcast); - if (!published) throw new Error("not found"); - - const info = await published.track(track).info(); + const info = await front.resolveTrackInfo(track); return new TrackInfoMessage({ priority: info.priority, // Publisher Max Age: the publisher's retention bound, advertised so @@ -811,8 +818,8 @@ export class Publisher { })(); // Don't poison the cache on failure: a later request may succeed. - pending.catch(() => this.#trackInfo.delete(key)); - this.#trackInfo.set(key, pending); + pending.catch(() => tracks.delete(track)); + tracks.set(track, pending); return pending; } diff --git a/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md b/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md deleted file mode 100644 index 357aca8a0fc..00000000000 --- a/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md +++ /dev/null @@ -1,86 +0,0 @@ -# [M] js/net: path-keyed publisher state goes stale when a broadcast is replaced - -## Goal - -Implement and verify the behavior tracked in [#2985](https://github.com/moq-dev/moq/issues/2985) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### After prefix routes - -The remedy this quest points at no longer exists. -[moq#3225](https://github.com/moq-dev/moq/pull/3225) removed `Epoch` from -`draft-lcurley-moq-lite` as spec-only and never implemented, and retired -`draft-lcurley-moq-broadcast`. So "#2610 specs exactly this as the broadcast -epoch" is now a dead pointer, and TRACK_INFO epoch-keying is not coming from -there. - -Two of the three sites are also already fixed on `dev`, by keying on the routing -front rather than on the path: - -- `lite/publisher.ts`'s `runAnnounce` holds `Map` and diffs on the front, so a republish emits - ended-then-active instead of nothing. -- `ietf/publisher.ts` records the front each refusal was about in `offered`, so - a republish at the same path clears the refusal. - -What remains is the third: `#resolveTrackInfo` caches on -`` `${broadcast}\0${track}` `` with no generation, so a republish serves the old -track's `TRACK_INFO`. Rescope to that one cache. It is local state with a local -fix (invalidate on the front change the other two sites already observe), and it -needs no wire-level identity, so do not block it on the Rust side. - -### Issue context - -#### Problem - -Three pieces of publisher state in `js/net` are keyed by **path** and never reconciled when a path's producer is *replaced*. Replacement keeps the key present, so every one of them silently keeps serving the predecessor's generation. - -Surfaced while reviewing #2976, which fixes an adjacent race (a same-tick close + republish let the predecessor's close handler evict the live successor from the lookup). That fix is correct and these are **not** regressions from it: the diff there only changes the body of `broadcast.closed.then(...)`, so it is inert unless a broadcast closes. All three reproduce on `main` today with a plain replacement over a still-open predecessor, since `Publisher.publish` overwrites the map entry without closing its predecessor: - -```ts -publisher.publish(path, first); -publisher.publish(path, second); // first never closed; key unchanged, value swapped -``` - -##### 1. Same-path replacement is invisible to discovery - -`runAnnounce` builds its active set from `broadcasts.keys()` and diffs it with `Set.difference`, so a value swap under an existing key produces neither an `ended` nor a new `active` announcement. Discovery consumers stay attached to the predecessor's `Consumer`. - -- `js/net/src/lite/publisher.ts` - `runAnnounce` -- the IETF advertisement loops reconcile the same way - -##### 2. `TRACK_INFO` is cached across generations - -`#trackInfo` is a `Map>` keyed by path + track name and is never invalidated by `publish`. A successor with different immutable track properties (timescale, ordering) is answered with the predecessor's metadata. `FETCH` consumes the same cache, so successor frames can be interpreted against predecessor metadata. - -- `js/net/src/lite/publisher.ts` - `#trackInfo`, `runTrackInfo` - -##### 3. An IETF refusal permanently suppresses the successor - -The advertisement loop holds `refused` keyed by `Path.Valid` and clears an entry only when the path is absent from the live set (`if (!live.has(path)) refused.delete(path)`). Because replacement keeps the key present, a `PUBLISH_NAMESPACE` refusal against the predecessor is never cleared, and a live successor stays unadvertised for the rest of the session even though a direct `SUBSCRIBE` resolves it. - -- `js/net/src/ietf/publisher.ts` - the advertisement loop's `refused` map - -#### Suggested direction - -All three are the same root cause: no producer identity or generation is tracked, only the path. #2610 specs exactly this as the broadcast **epoch**, and already lists TRACK\_INFO epoch-keying ("`TRACK_INFO` returns the resolved epoch so metadata caches key by generation and requests cannot race a replacement"). This issue tracks the three concrete `js/net` sites that are reachable today, so they don't get lost if the epoch work lands wire-first or in stages. - -A local fix is possible ahead of the wire work: compare producer identity (not just the key) during announce reconciliation and emit the replacement transition, evict the path's `#trackInfo` entries in `publish`, and key `refused` by producer identity so it clears on replacement. - -#### Tests to encode the root cause - -- Replacing a path's producer (both same-tick-after-close and over a still-open predecessor) emits a replacement transition and `announcedBroadcast.active` becomes a distinct `Consumer`. -- Republishing with a different timescale returns the successor's `TRACK_INFO`, not the predecessor's, over both `SUBSCRIBE` and `FETCH`. -- A `PUBLISH_NAMESPACE` refusal followed by replacement advertises the successor. - -Found by Codex during the adversarial review on #2976, verified against the code. - -## Closes - -- [#2985](https://github.com/moq-dev/moq/issues/2985) - close this issue when the quest finishes diff --git a/quest/m1/README.md b/quest/m1/README.md index 0e51adba618..9f0591c4865 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -31,7 +31,6 @@ with the current dev tree before starting. - [#3161](/quest/m1/3161-retention-should-reclaim-idle-open-groups-now-that-expiry.md) - Retention should reclaim idle open groups now that expiry is timestamp-only - [Group overflow](/quest/m1/group-overflow-abort.md) - an oversized open group aborts for every reader instead of shedding its head - [#2895](/quest/m1/2895-add-an-atomic-readiness-gate-for-origin-broadcasts.md) - Add an atomic readiness gate for Origin broadcasts -- [#2985](/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md) - js/net: path-keyed publisher state goes stale when a broadcast is replaced - [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - net: coalesce dynamic tracks and preserve sequences across replacements - [Announce handle](/quest/m1/announce-handle.md) - announce(prefix, route) advertises and serves requests; create_broadcast plus set_announce publishes - [#3190](/quest/m1/3190-align-origin-broadcast-creation-naming-across-language.md) - every native binding exposes create_broadcast, set_announce, and announce with one meaning diff --git a/quest/m1/js-announce.md b/quest/m1/js-announce.md index 65a5b2b55f2..8f60bdb7649 100644 --- a/quest/m1/js-announce.md +++ b/quest/m1/js-announce.md @@ -38,5 +38,4 @@ Branch from `dev`, where the origin table lives; the rename is breaking. ## Related - [#3190](/quest/m1/3190-align-origin-broadcast-creation-naming-across-language.md) - the native bindings half -- [#2985](/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md) - publisher state on the same origin table - [#2318](/quest/m1/2318-js-net-remaining-capability-gaps-vs-rs-moq-net-setup-role.md) - other js/net gaps vs rs/moq-net From 9e4be8707f5526ab17ff81880fe4473c05e4fe26 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:48:28 -0700 Subject: [PATCH 18/29] docs(drafts): scope a TRACK_INFO cache to the announcement, not the path The Track Stream section told a subscriber to cache TRACK_INFO keyed by broadcast path and track name for the life of the session, which is the exact shape that goes stale when a broadcast is replaced on that path. Say what the immutability is actually scoped to. The wire format is unchanged; this only qualifies existing caching advice. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLtoPv9B3779kGxLFTUGHR --- drafts/draft-lcurley-moq-lite.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 1ebad80ae44..bf386c6b384 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -412,6 +412,8 @@ A subscriber opens a Track Stream (0x6) to learn a Track's immutable publisher p The subscriber sends a TRACK message containing the broadcast path and track name. The publisher replies with a single TRACK_INFO message and then FINs the stream, or resets the stream on error (e.g. the track does not exist). The returned properties are fixed for the lifetime of the track, so the subscriber SHOULD cache TRACK_INFO keyed by broadcast path and track name, and reuse it across every SUBSCRIBE and FETCH of the same track over the session that served it. +The properties are fixed for one track, not for the path it arrived on: a path outlives the broadcast on it, and a different broadcast reaching the same path brings its own. +Anything cached against a path, on either side, is therefore scoped to the announcement that carried it and is discarded when that announcement is retracted. If FRAME messages cannot be decoded against the cached TRACK_INFO, the subscriber MUST reset the affected stream with a protocol violation and re-request it. Because a subscriber cannot parse buffered group frames until TRACK_INFO arrives, the publisher SHOULD prioritize TRACK_INFO ahead of group data on the connection. From 9a5e8452226e26725d65339c1e1a6464d8782e8e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 18:59:31 -0700 Subject: [PATCH 19/29] docs(quest): land the review fixes auto-merge left out of #3416 (#3425) Co-authored-by: Claude Fable 5.1 --- ...coder-captures-the-rewind-generation-at.md | 8 +++++-- quest/m1/plan-binding-rate-control.md | 6 ++++-- ...te-35-ad-cue-signaling-carried-opaquely.md | 21 +++++++++++++++---- quest/m2/emsg.md | 6 ++++-- quest/m2/flv-script.md | 3 ++- quest/m2/hls-generation.md | 16 ++++++++------ quest/m2/id3.md | 4 +++- quest/m2/path-patterns/origin.md | 8 ++++--- quest/m2/plan-revalidation-updates.md | 13 +++++++++++- quest/m2/relay-memory.md | 5 ++++- quest/m2/sei/sei.md | 6 ++++++ quest/m3/teleop/correlation.md | 8 +++++-- 12 files changed, 79 insertions(+), 25 deletions(-) diff --git a/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md b/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md index fe28b2769bc..87d8b1a0642 100644 --- a/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md +++ b/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md @@ -14,8 +14,12 @@ Rescoped in the 2026-09 planning pass: undeclared rewinds are going away mismatch below no longer produces the sixty-second park, but the bug it describes still applies to a declared discontinuity, where a stale frame must not be presented at all. Fix it the first way the issue suggests: call -`decoder.reset()` and re-`configure()` in `#onDiscontinuity`, and drop the -output-time generation guard, which then has nothing left to catch. The +`decoder.reset()` and re-`configure()` in `#onDiscontinuity`, so queued chunks +never surface. Keep the post-await generation guard as well: `reset()` cannot +cancel an output callback that already holds a frame and is parked in +`#park()` or `sync.wait()`, and `sync.reset()` releases exactly that wait, so +the guard is what stops a stale callback writing `timestamp` and `frame` +after the reset. The regression needs a real WebCodecs decoder, so it lives in a browser harness rather than a bun unit test. diff --git a/quest/m1/plan-binding-rate-control.md b/quest/m1/plan-binding-rate-control.md index 3353af59887..b0b2184416c 100644 --- a/quest/m1/plan-binding-rate-control.md +++ b/quest/m1/plan-binding-rate-control.md @@ -3,8 +3,10 @@ ## Goal A settled shape for how a non-Rust publisher follows the connection's send -estimate, so an OBS, Python, Swift, Kotlin, or Go publisher stops encoding at -its configured bitrate regardless of congestion. Run `/plan-quest`; the settled +estimate with its configured bitrate as the ceiling, the contract +`moq_video::encode::Options::{bitrate, bandwidth}` already gives Rust, so an +OBS, Python, Swift, Kotlin, or Go publisher stops holding that ceiling through +congestion. Run `/plan-quest`; the settled plan becomes the implementing quest that closes the issue. ## Plan diff --git a/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md b/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md index c9c14f51a94..a00eaf6fb3b 100644 --- a/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md +++ b/quest/m2/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md @@ -18,18 +18,31 @@ bytes. Not a `moq-json` stream of typed records. - **A `scte35` catalog section**, as its own `CatalogExt` like the `mpegts` one, naming one sidecar track per rendition that carries cues. Cues from a TS program attach to its video rendition, or to its audio rendition when - there is no video; the import already stamps them with video PTS. + there is no video. One timestamp contract for both: a cue's wire timestamp + is its `splice_time` (or the section's arrival time when it has none) + converted to the owning rendition's clock, so the importer, which stamps + from the video `last_pts` today, learns to stamp from audio PTS for an + audio-only program, and the draft, both implementations, and the export + tests use the same rule. - **Framing.** Each frame is one complete `splice_info_section`, byte for byte, in the group of the media it applies to, stamped with the splice - time's wire timestamp. Cues are sparse, so most groups are empty; a joiner - at a keyframe gets that GOP's cues. + time's wire timestamp. A section with no splice time (`splice_null`, + `private_command`, `bandwidth_reservation`, an immediate `splice_insert`) + keeps the arrival media clock the importer stamps it with today and lands + in the group current at arrival, so raw commands stay available. Cues are + sparse, so most groups are empty; a joiner at a keyframe gets that GOP's + cues. - **Typed decode is a helper, not the wire.** A parser for `splice_insert`, `time_signal`, and segmentation descriptors (a maintained crate if one is adequate) lives beside the section in Rust and JS, and `js/watch` raises a cue event from it. Applications that want the rest parse the bytes. - **The TS lanes stay.** The importer keeps the verbatim `mpegts` track for contribution fidelity and additionally emits the sidecar; the exporter must - not double-emit, and section-framed export still needs the video clock. + not double-emit. Section-framed export takes its clock from the video + rendition today and rejects an audio-only program + (`scte35_without_video_export_is_rejected`); extend it to derive the clock + from the mapped audio rendition, or the audio-only mapping above is a + promise export cannot keep. Wire and catalog schema: a new optional section, additive, so it lands on `main`, with `drafts/draft-lcurley-moq-hang.md` updated in the same PR as the diff --git a/quest/m2/emsg.md b/quest/m2/emsg.md index 83c50e7ce34..89c2d4e9f65 100644 --- a/quest/m2/emsg.md +++ b/quest/m2/emsg.md @@ -20,8 +20,10 @@ decodes it with its own library, and a new scheme stays forward-compatible. Carry them on a sidecar of the video rendition (the audio rendition when there is none) following the rule in [SEI section](/quest/m2/sei/sei.md): the rendition's group sequence, each box stamped with the wire timestamp of its -presentation time, raw bytes. Export rebuilds the box in the fragment whose -media time contains it. +presentation time, raw bytes; a box before the first `moof` takes the +pre-media placement that rule defines. Export rebuilds the box in the +fragment whose media time contains it, and the pre-media ones before the first +fragment. Test version 0 and version 1 boxes, an emsg before the first moof, several on one fragment, an unknown scheme, a zero duration, and a round trip that is diff --git a/quest/m2/flv-script.md b/quest/m2/flv-script.md index dc49fc43ee3..a8d6c89fcf8 100644 --- a/quest/m2/flv-script.md +++ b/quest/m2/flv-script.md @@ -15,7 +15,8 @@ through the same channel. Carry the tag payload byte-faithfully rather than decoding AMF into a fixed vocabulary, so a custom message type needs no further work, on a sidecar of the video rendition following the rule in [SEI section](/quest/m2/sei/sei.md): -the rendition's group sequence and the tag's timestamp on the wire. +the rendition's group sequence and the tag's timestamp on the wire, with a +tag before the first media tag taking that rule's pre-media placement. Where `onMetaData` duplicates something the catalog already models (dimensions, framerate, bitrate), prefer the value the bitstream actually carries and treat the script tag as opaque data, not a second source of truth for configuration. diff --git a/quest/m2/hls-generation.md b/quest/m2/hls-generation.md index 6106cad1706..dfb5a546caa 100644 --- a/quest/m2/hls-generation.md +++ b/quest/m2/hls-generation.md @@ -18,10 +18,13 @@ inline-parameter-set codec (`avc3`) that restarts at a new resolution keeps its URL while `Rendition::init` caches the old init for the life of the pooled rendition, and the segment window's `push` (`rs/moq-hls/src/export/segments.rs`) resets only the playlist. Version the -init URL with a hash of the rendition config (codec string, dimensions, and -the parameter sets the init carries), so a reconfigure changes the URL and -the cached init is invalidated with it. A hash rather than a counter because -two edges rendering the same broadcast must agree without coordinating. +init URL with a hash of the init segment bytes `Muxer::init` actually +produces, so every input that shapes the init (codec string, dimensions, +parameter sets, the catalog `Cmaf` init, audio sample rate and channel count, +codec description, the effective timescale) is covered without a field list +that can drift, and a reconfigure changes the URL and invalidates the cached +init with it. A hash rather than a counter because two edges rendering the +same broadcast must agree without coordinating. **A publisher restart** reuses segment numbers: `moq-net` splices a re-attached source into the existing broadcast by hop ID so consumers never @@ -36,8 +39,9 @@ playlist renderers in `rs/moq-hls/src/export` emit it in `init.mp4` and segment URLs and the export resets its window when it changes. There is no DASH renderer in the tree, so nothing here promises an MPD. Absent means unversioned URLs, never a wall-clock or a relay-local counter. The cache-header half stays downstream: -the edge restores its long `max-age` once the paths it accepts are the ones the -renderers emit. +the edge restores its long `max-age` only on generation-bearing URLs once the +paths it accepts are the ones the renderers emit, and unversioned URLs keep +`no-store`, since a restart reuses their segment numbers for different bytes. Acceptance: a mid-broadcast reconfigure that cannot be decoded against the previous init gets a new init URL on both edges; a supplied generation appears diff --git a/quest/m2/id3.md b/quest/m2/id3.md index c7bf7c9e410..722c9eb901b 100644 --- a/quest/m2/id3.md +++ b/quest/m2/id3.md @@ -14,7 +14,9 @@ Define the catalog section and frame contract for complete ID3 tags, following the sidecar rule in [SEI section](/quest/m2/sei/sei.md): the ID3 track belongs to the program's video rendition (its audio rendition when there is no video), uses that rendition's group sequence, and stamps each frame with -the wire timestamp of the tag's presentation time. Carry original tag bytes, +the wire timestamp of the tag's presentation time. An ID3-only program has no +rendition to follow, so its track stands alone: one group per tag, stamped from +the tag's own PTS, with the section saying it accompanies no rendition. Carry original tag bytes, and in the section the raw stream-level PMT descriptor loop plus any application or program registration metadata the exporter needs to reproduce the input. Do not parse the tag into a fixed diff --git a/quest/m2/path-patterns/origin.md b/quest/m2/path-patterns/origin.md index afe0aa4f31a..70bd9f5c52f 100644 --- a/quest/m2/path-patterns/origin.md +++ b/quest/m2/path-patterns/origin.md @@ -34,9 +34,11 @@ either hides paths the subscriber may use or leaks the ones it must not, and `request_broadcast` resolves against the route table regardless. The enforcing shape is a pattern-scoped grant on the consumer handle, narrowed at resolution and announce fan-out alike, and changed at runtime through -[relay revalidation](/quest/m2/path-patterns/relay-auth.md); prove the deafen -case (a grant excluding one audio path under a room prefix) in the model-layer -tests here. +[relay revalidation](/quest/m2/path-patterns/relay-auth.md). Narrowing a live +grant must also end the subscriptions it no longer covers, not only refuse new +ones. Prove the deafen case in the model-layer tests here: subscribe under a +room prefix, revalidate with a grant that excludes that audio path, and assert +the existing subscription closes and no further objects arrive. ## Required diff --git a/quest/m2/plan-revalidation-updates.md b/quest/m2/plan-revalidation-updates.md index 5cd9495dd5b..2eaad631b27 100644 --- a/quest/m2/plan-revalidation-updates.md +++ b/quest/m2/plan-revalidation-updates.md @@ -24,7 +24,18 @@ fields the auth API can legitimately change therefore behave inconsistently: announcing under a root the API no longer assigns, but it is a silent hard disconnect for what may be a benign rename. -Decide deliberately, per field: update in place, close, or refuse the change. +Two things are fixed before this decision is made. The +[relay auth](/quest/m2/path-patterns/relay-auth.md) quest already requires a +re-check to compare the full versioned grant, reject unsupported, mixed, +invalid, and out-of-scope grants, and resize a live session with no prefix-only +widening window; this plan inherits that invariant rather than restating a +weaker one. And "the alias becomes the token's `root`" describes today's v0 +behavior only: under v1 grants a literal root aliases or rebases the patterns +and never becomes one, so the canonical alias transformation has to be stated +before deciding whether an alias change closes or updates a session. + +Then decide deliberately, per field: update in place, close, or refuse the +change. Then the implementing quest applies it in `rs/moq-relay/src/auth.rs` with a test per field and documents the contract in `doc/bin/relay/auth.md`. diff --git a/quest/m2/relay-memory.md b/quest/m2/relay-memory.md index 3c9a3508525..50e6ee51d0c 100644 --- a/quest/m2/relay-memory.md +++ b/quest/m2/relay-memory.md @@ -3,7 +3,10 @@ ## Goal A relay's memory scales with what it serves, not with what the mesh knows, -and the numbers behind that claim are current. +and the numbers behind that claim are current. Two costs to keep apart when +measuring: the route table and per-announcement state (`RouteEntry` plus +`ServeState`) scale with announcements and routes, while the served-content +cache a `ServeState` materializes scales with demand. ## Plan diff --git a/quest/m2/sei/sei.md b/quest/m2/sei/sei.md index dfdbc78a5db..3c9c1d8f92e 100644 --- a/quest/m2/sei/sei.md +++ b/quest/m2/sei/sei.md @@ -17,6 +17,12 @@ wire timestamp of the media it accompanies, and carries raw bytes. A 1080p and a 360p rendition carry different SEI, so there is one sidecar per video rendition, and group 7 of the sidecar holds the SEI for group 7 of its video. +Metadata that precedes the first media unit (an `emsg` before the first +`moof`, an FLV script tag before the first media tag) rides the rendition's +first group, ordered before that group's media, stamped with its own +presentation time when it has one and otherwise with the first media frame's, +and an exporter emits it before the first media unit. + Within a group the frame's wire timestamp is the key, so an application syncing to presentation time reads it directly instead of joining against the video track it deliberately did not subscribe to. Several access units can diff --git a/quest/m3/teleop/correlation.md b/quest/m3/teleop/correlation.md index 3bb6f4a5fda..dfd27046ee3 100644 --- a/quest/m3/teleop/correlation.md +++ b/quest/m3/teleop/correlation.md @@ -19,8 +19,12 @@ operator's command to a robot's frame; `timeline.rs`'s `set_wall` is what can. So the contract is: the robot anchors its video and telemetry timelines, the operator anchors its command track, and a recording joins the three on -`wall + pts`. State the assumption beside the API, and report the anchor's -presence so a consumer can tell "unsynced" from "synced". +`wall + pts`, resolving each sample against the latest anchor at or before it +(the catalog `wall` until a timeline record after a discontinuity carries a +new one). State the clock assumption beside the API. Report only whether an +anchor is present; presence says nothing about whether the hosts are synced, +and a join across unsynced hosts looks valid while being wrong, so synchronization +is a declared deployment property, never inferred from the data. This is also the answer to Kyber's headline claim of continuous drift computation onto one unified timeline. Worth answering on the merits: From 419de625455c1d7d73a2e6b98ffcadeee997c409 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 19:04:57 -0700 Subject: [PATCH 20/29] fix(relay): make the web listeners share one socket-capturing acceptor (#3415) Co-authored-by: Claude Opus 5 (1M context) --- ...ng-guards-the-socket-capturing-acceptor.md | 37 --- quest/m1/README.md | 1 - rs/moq-relay/src/internal.rs | 5 +- rs/moq-relay/src/listener.rs | 15 +- rs/moq-relay/src/web.rs | 293 +++++++++++++----- rs/moq-relay/tests/smoke.rs | 8 +- 6 files changed, 231 insertions(+), 128 deletions(-) delete mode 100644 quest/m1/2980-moq-relay-nothing-guards-the-socket-capturing-acceptor.md diff --git a/quest/m1/2980-moq-relay-nothing-guards-the-socket-capturing-acceptor.md b/quest/m1/2980-moq-relay-nothing-guards-the-socket-capturing-acceptor.md deleted file mode 100644 index ccc0fa34c50..00000000000 --- a/quest/m1/2980-moq-relay-nothing-guards-the-socket-capturing-acceptor.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] moq-relay: nothing guards the socket-capturing acceptor being installed on a listener - -## Goal - -Implement and verify the behavior tracked in [#2980](https://github.com/moq-dev/moq/issues/2980) -within the issue's stated scope and boundaries. - -## Plan - -Use the public issue's scope, implementation notes, and acceptance criteria -below as the starting plan. Reconcile paths and assumptions with the current -tree before implementation. - -### Issue context - -Follow-up from #2963, raised independently by two review passes and accepted there rather than blocking the merge. - -#### The gap - -`SocketAcceptor` (plain HTTP) and `MtlsAcceptor` (HTTPS) each capture the connection's descriptor so qmux can report an RTT at SETUP time. `web.rs` tests cover the capture itself and its propagation onto the request, but nothing covers the *installation*: deleting `.acceptor(SocketAcceptor)` from `Web::serve` leaves the whole suite green. - -That is not hypothetical. #2963 originally shipped exactly that bug - only the HTTPS listener installed an acceptor, so every `ws://` session silently advertised no Probe capability and stayed on the fallback jitter buffer. It looked correct in any TLS deployment and did nothing for local development and the demo. - -#### Why it was not fixed there - -Two obvious approaches were considered and rejected: - -- **Install the acceptor inside `listener::server()`** so no call site can forget. `internal.rs` shares that helper and serves only an ops router, so this would duplicate a descriptor on every health-check connection for nothing. -- **Assert it end to end** in `tests/smoke.rs`, which does have a working `ws://` relay harness. `moq_net::ConnectionStats` exposes `estimated_recv_rate` from PROBE but not the PROBE's RTT, and the relay's send-rate estimate is `None` on macOS, so there is no cheap observable that proves the capture reached qmux. - -#### What would close it - -Surfacing the peer-reported PROBE RTT on `ConnectionStats` would make the smoke assertion straightforward, and is plausibly useful to consumers in its own right - an application on the adaptive-jitter path currently cannot see the RTT its buffer is derived from. That is a public API addition and deserves its own design pass rather than being bolted on to satisfy a test. - -## Closes - -- [#2980](https://github.com/moq-dev/moq/issues/2980) - close this issue when the quest finishes diff --git a/quest/m1/README.md b/quest/m1/README.md index 0e51adba618..ba480394630 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -24,7 +24,6 @@ with the current dev tree before starting. - [#2924](/quest/m1/2924-moq-relay-tls-rotation-is-not-atomic-across-thread-per.md) - moq-relay: TLS rotation is not atomic across thread-per-core QUIC workers - [#2964](/quest/m1/2964-quic-workers-dropping-one-split-server-resizes-the.md) - QUIC workers: dropping one split() Server resizes the reuseport group - [#2979](/quest/m1/2979-moq-tokio-does-not-compile-with-no-default-features-and.md) - moq-tokio compiles with any subset of transport features, and nightly checks it per crate -- [#2980](/quest/m1/2980-moq-relay-nothing-guards-the-socket-capturing-acceptor.md) - moq-relay: nothing guards the socket-capturing acceptor being installed on a listener - [#2853](/quest/m1/2853-quiche-with-a-pinned-source-port-can-dial-only-a-broken.md) - quiche with a pinned source port can dial only a broken IPv4 address - [#2624](/quest/m1/2624-moq-native-goaway-redirect-guard-classifies-hosts-by-name.md) - moq-native: GOAWAY redirect guard classifies hosts by name, not by resolved address - [#3086](/quest/m1/3086-refactor-net-make-group-delivery-order-a-handle-and-move.md) - refactor(net): make group delivery order a handle, and move timestamp-based skipping into moq-net diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 46522cd29a8..edc90b06249 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -34,6 +34,7 @@ use axum::{ response::{IntoResponse, Response}, routing::get, }; +use axum_server::accept::DefaultAcceptor; /// Configuration for the internal (ops) listener. #[derive(usage::Args, Clone, Debug, serde::Deserialize, serde::Serialize, Default)] @@ -170,7 +171,9 @@ impl Internal { let listener = moq_tokio::bind::tcp(listen).context("failed to bind internal listener")?; // No blanket "…server failed" context here: the caller (main.rs) adds // that single top-level layer, matching `Web::serve` / `Cluster::run`. - crate::listener::server(listener, self.health)? + // No accept-time work: the ops router never hands a connection to qmux, so + // capturing a descriptor per health check would spend one for nothing. + crate::listener::server(listener, self.health, DefaultAcceptor::new())? .serve(app.into_make_service()) .await?; Ok(()) diff --git a/rs/moq-relay/src/listener.rs b/rs/moq-relay/src/listener.rs index e7ca031dcfc..61f8d7bbbfa 100644 --- a/rs/moq-relay/src/listener.rs +++ b/rs/moq-relay/src/listener.rs @@ -52,12 +52,21 @@ impl Listener { } } -/// An `axum_server` bound to [`Listener`], ready for `.acceptor(..)`/`.serve(..)`. +/// An `axum_server` bound to [`Listener`] with `acceptor` installed, ready for +/// `.serve(..)`. +/// +/// The acceptor is a parameter rather than the builder step `axum_server` offers, +/// so what a listener does at accept time is something each call site states rather +/// than something it can leave out. /// /// The address type is spelled here rather than at each call site: `A::Listener` /// doesn't pin `A` down, so inference cannot get there from the listener alone. -pub(crate) fn server(listener: std::net::TcpListener, health: Health) -> io::Result> { - Ok(axum_server::Server::from_listener(Listener::new(listener, health)?)) +pub(crate) fn server( + listener: std::net::TcpListener, + health: Health, + acceptor: A, +) -> io::Result> { + Ok(axum_server::Server::from_listener(Listener::new(listener, health)?).acceptor(acceptor)) } impl AddrListener for Listener { diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 2a3553d06ce..09513bef0e5 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -276,16 +276,13 @@ impl Web { pub async fn serve(self, app: Router) -> anyhow::Result<()> { let config = self.config; let app = app.fallback(serve_landing).into_make_service(); + let ws = config.resolved_ws(); let http = if let Some(listen) = config.http.listen { // Dual-stack so the cert endpoint + WebSocket fallback answer over IPv4 // too, even on Windows where `[::]` is IPv6-only by default. let listener = moq_tokio::bind::tcp(listen).context("failed to bind HTTP listener")?; - // Same socket capture the HTTPS path gets from `MtlsAcceptor`: without it - // a `ws://` session reaches qmux with no descriptor and reports no RTT. - let server = crate::listener::server(listener, self.health.clone())?.acceptor(SocketAcceptor { - ws: config.resolved_ws(), - }); + let server = crate::listener::server(listener, self.health.clone(), WebAcceptor::plain(ws))?; Some(server.serve(app.clone())) } else { None @@ -301,16 +298,8 @@ impl Web { tokio::spawn(reload_https_config(rustls_config.clone(), cert, key, root)); - // MtlsAcceptor surfaces a verified peer cert as a request extension. - // When no client CA is configured, the inner verifier is `NoClientAuth` - // and `peer_certificates()` always returns None — the wrapper is then - // a near-no-op, but keeping a single path simplifies reload + serve. - let acceptor = MtlsAcceptor { - inner: RustlsAcceptor::new(rustls_config), - ws: config.resolved_ws(), - }; let listener = moq_tokio::bind::tcp(listen).context("failed to bind HTTPS listener")?; - let server = crate::listener::server(listener, self.health.clone())?.acceptor(acceptor); + let server = crate::listener::server(listener, self.health.clone(), WebAcceptor::tls(rustls_config, ws))?; Some(server.serve(app)) } else { None @@ -425,17 +414,73 @@ pub(crate) struct SocketStats(std::convert::Infallible); #[derive(Clone, Debug)] pub struct MtlsPeer; -/// Wraps [`RustlsAcceptor`] so that, after the TLS handshake, we extract the -/// peer cert presence from rustls's `ServerConnection` and attach it to every -/// request on this connection as `Extension>`. +/// Accepts a connection on a public web listener. +/// +/// Captures the socket, then hands the stream to `inner`: [`DefaultAcceptor`] for +/// `http://`, or [`RustlsAcceptor`] for `https://`, which also gives the connection +/// a peer certificate to report. Both listeners wrap this same acceptor, so the +/// capture cannot reach one and miss the other. #[derive(Clone)] -struct MtlsAcceptor { - inner: RustlsAcceptor, - /// As [`SocketAcceptor::ws`]. +struct WebAcceptor { + inner: A, + /// Whether a WebSocket route exists to use the handle. The capture holds a + /// duplicated descriptor for the life of the connection, so doing it when + /// nothing can read it would cost every API, HLS, and health-check connection a + /// second descriptor for nothing. ws: bool, } -/// The connection [`MtlsAcceptor`] accepts: a byte stream, plus a descriptor to +impl WebAcceptor { + /// Accepts plain HTTP, which has no handshake of its own to run. + fn plain(ws: bool) -> Self { + Self { + inner: DefaultAcceptor::new(), + ws, + } + } +} + +impl WebAcceptor> { + /// Accepts HTTPS, running the TLS handshake and surfacing a verified peer + /// certificate as [`MtlsPeer`]. + /// + /// When no client CA is configured the inner verifier is `NoClientAuth` and + /// `peer_certificates()` always returns None, so the marker never appears. + fn tls(config: RustlsConfig, ws: bool) -> Self { + Self { + inner: RustlsAcceptor::new(config), + ws, + } + } +} + +/// The mTLS marker a finished connection carries, if any. +/// +/// Which streams can present a client certificate is a property of the stream type, +/// not of the acceptor: only rustls has one to report, and a plain TCP connection +/// never does. +trait MtlsStream { + /// The marker for this connection's verified peer certificate. + fn mtls_peer(&self) -> Option; +} + +impl MtlsStream for tokio::net::TcpStream { + fn mtls_peer(&self) -> Option { + None + } +} + +impl MtlsStream for TlsStream { + fn mtls_peer(&self) -> Option { + self.get_ref() + .1 + .peer_certificates() + .filter(|certs| !certs.is_empty()) + .map(|_| MtlsPeer) + } +} + +/// The connection [`WebAcceptor`] is handed: a byte stream, plus a descriptor to /// read socket statistics from on platforms that expose them. /// /// The concrete stream is always the `TcpStream` from [`crate::listener::Peer`], so @@ -452,53 +497,17 @@ pub(crate) trait AcceptStream: AsyncRead + AsyncWrite + Unpin + Send + 'static { #[cfg(not(unix))] impl AcceptStream for T {} -/// Captures the socket for a plain-HTTP connection. -/// -/// [`MtlsAcceptor`] does this for HTTPS on its way through the TLS handshake, but -/// the HTTP listener has no acceptor of its own, so a `ws://` session would reach -/// qmux with no descriptor and report no RTT. This is the same capture without the -/// TLS half. -#[derive(Clone, Copy)] -struct SocketAcceptor { - /// Whether a WebSocket route exists to use the handle. The capture holds a - /// duplicated descriptor for the life of the connection, so doing it when - /// nothing can read it would cost every API, HLS, and health-check connection a - /// second descriptor for nothing. - ws: bool, -} - -impl Accept for SocketAcceptor -where - I: AcceptStream, - S: Send + 'static, -{ - type Stream = I; - type Service = SetConnectionExtensions; - type Future = BoxFuture<'static, std::io::Result<(Self::Stream, Self::Service)>>; - - fn accept(&self, stream: I, service: S) -> Self::Future { - let socket = self.ws.then(|| socket_stats(&stream)).flatten(); - async move { - Ok(( - stream, - SetConnectionExtensions { - inner: service, - peer: None, - socket, - }, - )) - } - .boxed() - } -} - -impl Accept for MtlsAcceptor +impl Accept for WebAcceptor where I: AcceptStream, S: Send + 'static, + A: Accept, + A::Stream: MtlsStream + Send + 'static, + A::Service: Send + 'static, + A::Future: Send + 'static, { - type Stream = TlsStream; - type Service = SetConnectionExtensions; + type Stream = A::Stream; + type Service = SetConnectionExtensions; type Future = BoxFuture<'static, std::io::Result<(Self::Stream, Self::Service)>>; fn accept(&self, stream: I, service: S) -> Self::Future { @@ -507,15 +516,10 @@ where let socket = self.ws.then(|| socket_stats(&stream)).flatten(); let inner = self.inner.accept(stream, service); async move { - let (tls, service) = inner.await?; - let peer = tls - .get_ref() - .1 - .peer_certificates() - .filter(|certs| !certs.is_empty()) - .map(|_| MtlsPeer); + let (stream, service) = inner.await?; + let peer = stream.mtls_peer(); Ok(( - tls, + stream, SetConnectionExtensions { inner: service, peer, @@ -980,15 +984,12 @@ mod tests { /// A real accepted socket must reach the request as an extension. /// - /// This is the whole point of the plain-HTTP path: `MtlsAcceptor` captures the - /// descriptor on its way through the TLS handshake, and `SocketAcceptor` has to - /// do the same for `ws://`. Without a capture the handle is `None`, qmux gets no - /// socket, and the session advertises no Probe capability -- silently, which is - /// why this drives an actual `TcpStream` rather than constructing the service by - /// hand. + /// Without a capture the handle is `None`, qmux gets no socket, and the session + /// advertises no Probe capability -- silently, which is why this drives an actual + /// `TcpStream` rather than constructing the service by hand. #[cfg(all(unix, feature = "websocket"))] #[tokio::test] - async fn socket_acceptor_surfaces_the_accepted_socket() { + async fn web_acceptor_surfaces_the_accepted_socket() { use axum::http::Request; use std::convert::Infallible; @@ -1014,7 +1015,7 @@ mod tests { let (accepted, _) = listener.accept().await.unwrap(); let _client = connecting.await.unwrap(); - let (stream, mut service) = Accept::<_, EchoSocket>::accept(&SocketAcceptor { ws: true }, accepted, EchoSocket) + let (stream, mut service) = Accept::<_, EchoSocket>::accept(&WebAcceptor::plain(true), accepted, EchoSocket) .await .unwrap(); @@ -1052,14 +1053,14 @@ mod tests { /// descriptors a relay can spend on connections, for nothing. #[cfg(all(unix, feature = "websocket"))] #[tokio::test] - async fn socket_acceptor_skips_capture_without_websockets() { + async fn web_acceptor_skips_capture_without_websockets() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let connecting = tokio::spawn(async move { tokio::net::TcpStream::connect(addr).await.unwrap() }); let (accepted, _) = listener.accept().await.unwrap(); let _client = connecting.await.unwrap(); - let (stream, service) = Accept::<_, ()>::accept(&SocketAcceptor { ws: false }, accepted, ()) + let (stream, service) = Accept::<_, ()>::accept(&WebAcceptor::plain(false), accepted, ()) .await .unwrap(); @@ -1071,6 +1072,134 @@ mod tests { drop(stream); } + /// Reports whether the connection's socket reached the request. + #[cfg(all(unix, feature = "websocket"))] + async fn report_socket(socket: Option>) -> &'static str { + match socket { + Some(_) => "captured", + None => "missing", + } + } + + /// Two ports the kernel just handed out, released together so neither bind can + /// be handed the other's. + #[cfg(all(unix, feature = "websocket"))] + fn free_ports() -> (u16, u16) { + let http = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let https = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + (http.local_addr().unwrap().port(), https.local_addr().unwrap().port()) + } + + /// Connect to `port`, waiting for [`Web::serve`] to finish binding. + #[cfg(all(unix, feature = "websocket"))] + async fn connect(port: u16) -> tokio::net::TcpStream { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + match tokio::net::TcpStream::connect(("127.0.0.1", port)).await { + Ok(stream) => return stream, + Err(err) if std::time::Instant::now() >= deadline => { + panic!("web listener never came up on port {port}: {err}") + } + Err(_) => tokio::time::sleep(std::time::Duration::from_millis(25)).await, + } + } + } + + /// `GET /socket` over `io`, returning the body the handler produced. + /// + /// Hand-rolled rather than reached through an HTTP client so the same request + /// works over TLS and plain TCP. `Connection: close` is what ends the read. + #[cfg(all(unix, feature = "websocket"))] + async fn get_socket(mut io: S) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + io.write_all(b"GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + + let mut response = String::new(); + // An unclean TLS shutdown is an error even after a complete response, so the + // body decides whether the exchange worked, not the read. + let _ = io.read_to_string(&mut response).await; + response + .split_once("\r\n\r\n") + .unwrap_or_else(|| panic!("no HTTP response body in {response:?}")) + .1 + .to_string() + } + + /// A TLS client trusting only `ca`, so the handshake also proves the relay + /// served the certificate it was configured with. + #[cfg(all(unix, feature = "websocket"))] + fn tls_connector(ca: &std::path::Path) -> tokio_rustls::TlsConnector { + use rustls::pki_types::{CertificateDer, pem::PemObject}; + + let pem = std::fs::read(ca).unwrap(); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(&pem) { + roots.add(cert.unwrap()).unwrap(); + } + + let mut config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + // The listener advertises h2 first, and this speaks HTTP/1.1 by hand. + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + + tokio_rustls::TlsConnector::from(Arc::new(config)) + } + + /// [`Web::serve`] must install the capturing acceptor on every listener it opens. + /// + /// The tests above cover the acceptor; this covers its installation. A listener + /// that skips it hands qmux a session with no descriptor, which stays on the + /// fallback jitter buffer while the other listener still looks correct, so both + /// run here and each is asked whether the capture reached the request. + #[cfg(all(unix, feature = "websocket"))] + #[tokio::test] + async fn serve_captures_the_socket_on_every_listener() { + let dir = TempDir::new().unwrap(); + let (ca, cert, key) = make_certs(&dir); + let (http, https) = free_ports(); + + let mut config = WebConfig::default(); + config.http.listen = Some(format!("127.0.0.1:{http}").parse().unwrap()); + config.https.listen = Some(format!("127.0.0.1:{https}").parse().unwrap()); + config.https.cert = vec![cert.clone()]; + config.https.key = vec![key]; + + // The probed route is the test's own, so auth never runs; it just has to be + // configured with something for `Web` to build. + let mut auth_config = crate::AuthConfig::default(); + auth_config.public = Some(crate::PublicConfig::Detailed(crate::PublicDetailed { + subscribe: vec![String::new()], + ..Default::default() + })); + let auth = Auth::new(auth_config).await.unwrap(); + let cluster = Cluster::new(crate::ClusterConfig::default()).unwrap(); + let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); + + let web = Web::new(auth, cluster, certificates, config); + let serving = tokio::spawn(web.serve(Router::new().route("/socket", get(report_socket)))); + + assert_eq!( + get_socket(connect(http).await).await, + "captured", + "the HTTP listener must install the capturing acceptor" + ); + + let tcp = connect(https).await; + let name = rustls::pki_types::ServerName::try_from("localhost").unwrap(); + let tls = tls_connector(&ca).connect(name, tcp).await.expect("TLS handshake"); + assert_eq!( + get_socket(tls).await, + "captured", + "the HTTPS listener must install the capturing acceptor" + ); + + serving.abort(); + } + /// Confirm `SetConnectionExtensions` injects the marker into request extensions /// when a peer cert was presented, and leaves them untouched otherwise. #[tokio::test] diff --git a/rs/moq-relay/tests/smoke.rs b/rs/moq-relay/tests/smoke.rs index d2e83e248b3..f8988ed9453 100644 --- a/rs/moq-relay/tests/smoke.rs +++ b/rs/moq-relay/tests/smoke.rs @@ -310,10 +310,10 @@ async fn relay_web_serves_merged_routes() { /// The HTTPS listener has to terminate TLS and answer a real request. /// -/// The plain-HTTP tests above go through `axum_server`'s default acceptor, so they -/// say nothing about the TLS stack: `MtlsAcceptor` wraps `RustlsAcceptor`, hot -/// reload swaps the config underneath it, and the listener the relay hands -/// `axum_server` is its own. A compile is not evidence any of that still handshakes. +/// The plain-HTTP tests above run no handshake at all, so they say nothing about +/// the TLS stack: the HTTPS acceptor wraps `RustlsAcceptor`, hot reload swaps the +/// config underneath it, and the listener the relay hands `axum_server` is its own. +/// A compile is not evidence any of that still handshakes. #[tokio::test] async fn relay_https_terminates_tls() { let port = free_tcp_port(); From f6ba494803ac92e01b5bd9ad6affa14ebb8e7bbc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 19:09:02 -0700 Subject: [PATCH 21/29] docs(quest): settle scope narrowing in place, and mark pre-media sidecar placement (#3427) Co-authored-by: Claude Fable 5.1 --- quest/m2/plan-revalidation-updates.md | 7 +++++-- quest/m2/sei/sei.md | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/quest/m2/plan-revalidation-updates.md b/quest/m2/plan-revalidation-updates.md index 2eaad631b27..6877d344b20 100644 --- a/quest/m2/plan-revalidation-updates.md +++ b/quest/m2/plan-revalidation-updates.md @@ -3,8 +3,11 @@ ## Goal A settled contract for which fields a revalidation reply can change on a live -session and which force a reconnect. Scope narrowing is settled (close, and -the client reconnects into the narrower grant); `tier` and `alias` are not. +session and which force a reconnect. Scope narrowing is settled by the +[relay auth](/quest/m2/path-patterns/relay-auth.md) contract: the session +stays up, resized to the narrower grant, and only the subscriptions and +publications the grant no longer covers are closed; `tier` and `alias` are +not settled. Run `/plan-quest`; the settled plan becomes the implementing quest that closes the issue. diff --git a/quest/m2/sei/sei.md b/quest/m2/sei/sei.md index 3c9c1d8f92e..7364e28542b 100644 --- a/quest/m2/sei/sei.md +++ b/quest/m2/sei/sei.md @@ -19,9 +19,12 @@ rendition, and group 7 of the sidecar holds the SEI for group 7 of its video. Metadata that precedes the first media unit (an `emsg` before the first `moof`, an FLV script tag before the first media tag) rides the rendition's -first group, ordered before that group's media, stamped with its own -presentation time when it has one and otherwise with the first media frame's, -and an exporter emits it before the first media unit. +first group, stamped with its own presentation time when it has one and +otherwise with the first media frame's. A timestamp cannot order it before +that frame, since a tag at timestamp zero and the first frame share one, so +the sidecar frame's placement field, the same one that records prefix versus +suffix SEI, carries an explicit before-first-media value, and an exporter +emits those frames before the first media unit. Within a group the frame's wire timestamp is the key, so an application syncing to presentation time reads it directly instead of joining against the From 27c199bacfac070037daa66c8847fd0498337ef7 Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Sat, 5 Sep 2026 04:11:26 +0200 Subject: [PATCH 22/29] fix(moq-video): pick the V4L2 mode nearest the requested resolution (#3355) Co-authored-by: Luke Curley Co-authored-by: Codex --- rs/moq-video/src/capture/v4l2.rs | 270 ++++++++++++++++++++++++++----- 1 file changed, 230 insertions(+), 40 deletions(-) diff --git a/rs/moq-video/src/capture/v4l2.rs b/rs/moq-video/src/capture/v4l2.rs index c0801ce4fb4..e77b1b129a0 100644 --- a/rs/moq-video/src/capture/v4l2.rs +++ b/rs/moq-video/src/capture/v4l2.rs @@ -18,8 +18,8 @@ use zune_jpeg::zune_core::bytestream::ZCursor; use super::channel::FrameChannel; use super::pump::{self, Geometry}; use super::{Config, Stream}; -use crate::Error; use crate::frame::{I420, Surface}; +use crate::{Error, Size}; /// List V4L2 capture nodes using paths that [`open_device`] accepts. pub(super) fn cameras() -> Result, Error> { @@ -98,18 +98,39 @@ const DEFAULT_HEIGHT: u32 = 720; /// Driver buffers to keep in flight; a small ring lets capture overlap encode. const BUFFER_COUNT: u32 = 4; -/// Raw 4:2:2; resampled to I420 with no color-space conversion. -const FOURCC_YUYV: &[u8; 4] = b"YUYV"; -/// Motion-JPEG; decoded per frame. -const FOURCC_MJPG: &[u8; 4] = b"MJPG"; - /// The negotiated source format, chosen once at open. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Source { + /// Raw 4:2:2; resampled to I420 with no color-space conversion. Yuyv, + /// Motion-JPEG; decoded per frame. Mjpeg, } +impl Source { + /// Every format we can convert, cheapest first. The order only breaks ties + /// between modes that fit the requested size equally well. + const ALL: [Self; 2] = [Self::Yuyv, Self::Mjpeg]; + + fn fourcc(self) -> FourCC { + FourCC::new(match self { + Self::Yuyv => b"YUYV", + Self::Mjpeg => b"MJPG", + }) + } + + fn from_fourcc(fourcc: FourCC) -> Option { + Self::ALL.into_iter().find(|source| source.fourcc() == fourcc) + } + + fn cost(self) -> u8 { + match self { + Self::Yuyv => 0, + Self::Mjpeg => 1, + } + } +} + pub(crate) struct Camera { stream: MmapStream<'static>, source: Source, @@ -127,25 +148,10 @@ impl Camera { let width = config.width.unwrap_or(DEFAULT_WIDTH); let height = config.height.unwrap_or(DEFAULT_HEIGHT); - let format = negotiate(&device, width, height)?; - let source = match &format.fourcc.repr { - f if f == FOURCC_YUYV => Source::Yuyv, - f if f == FOURCC_MJPG => Source::Mjpeg, - other => { - return Err(Error::Codec(anyhow::anyhow!( - "camera {name} supports neither YUYV nor MJPEG (negotiated {})", - FourCC::new(other) - ))); - } - }; + let (format, source) = negotiate(&device, &name, Size::new(width, height))?; let (width, height, stride) = (format.width, format.height, format.stride); - // I420 chroma is 2x2 subsampled, so the encoder needs even dimensions. - if !width.is_multiple_of(2) || !height.is_multiple_of(2) { - return Err(Error::Codec(anyhow::anyhow!( - "camera resolution {width}x{height} must be even for H.264 encoding" - ))); - } + Size::new(width, height).validate("camera resolution")?; // Best-effort framerate request; many cameras clamp or ignore it. if let Some(fps) = config.framerate { @@ -192,7 +198,17 @@ impl Camera { let (w, h) = decoder .dimensions() .ok_or_else(|| Error::Codec(anyhow::anyhow!("MJPEG frame had no dimensions")))?; - I420::from_rgb(&rgb, w as u32, h as u32)? + // The stream reports the negotiated size and the encoder is built + // from it, so a frame that decodes to another one can't be published + // as this stream's. + if w as u32 != self.width || h as u32 != self.height { + return Err(Error::Codec(anyhow::anyhow!( + "MJPEG frame is {w}x{h}, not the negotiated {}x{}", + self.width, + self.height + ))); + } + I420::from_rgb(&rgb, self.width, self.height)? } }; Ok(pump::Read::Frame(Surface::I420(i420))) @@ -228,21 +244,195 @@ fn open_error(device: &str, error: std::io::Error) -> Error { } } -/// Negotiate a format we can convert to I420. We ask for YUYV then MJPEG; the -/// driver substitutes its nearest supported pixel format, so accept the first -/// reply that lands on one we handle. -fn negotiate(device: &Device, width: u32, height: u32) -> Result { - let mut last = None; - for want in [FOURCC_YUYV, FOURCC_MJPG] { - let requested = Format::new(width, height, FourCC::new(want)); - let got = Capture::set_format(device, &requested) - .map_err(|e| Error::Codec(anyhow::anyhow!("V4L2 set format: {e}")))?; - if &got.fourcc.repr == FOURCC_YUYV || &got.fourcc.repr == FOURCC_MJPG { - return Ok(got); +/// Negotiate the format we can convert to I420 that lands closest to `want`. +/// +/// V4L2's non-mutating `VIDIOC_TRY_FMT` is optional, so use the required +/// `VIDIOC_S_FMT`. It asks and applies in one step, substituting the driver's +/// nearest supported mode for anything it doesn't have. Each format we handle +/// is applied in turn and scored against the requested geometry, then the +/// winner is applied again to leave the device on it. +/// +/// Taking the first reply instead would pin most laptop webcams to VGA: USB +/// bandwidth doesn't fit uncompressed 4:2:2 above that, so they offer YUYV only +/// at small sizes and reach HD through MJPEG alone. Asking such a camera for +/// YUYV at 1080p gets 640x480 back, which is a valid YUYV mode and nowhere near +/// what the caller asked for. +fn negotiate(device: &Device, name: &str, want: Size) -> Result<(Format, Source), Error> { + negotiate_with(name, want, |format| set_format(device, format)) +} + +fn negotiate_with( + name: &str, + want: Size, + mut apply: impl FnMut(Format) -> Result, +) -> Result<(Format, Source), Error> { + let mut replies = Vec::with_capacity(Source::ALL.len()); + let mut offered = Vec::new(); + let mut probe_error = None; + for candidate in Source::ALL { + let got = match apply(Format::new(want.width, want.height, candidate.fourcc())) { + Ok(got) => got, + Err(error) => { + probe_error = Some(error); + continue; + } + }; + let description = format!("{}x{} {}", got.width, got.height, got.fourcc); + if !offered.contains(&description) { + offered.push(description); + } + if let Some(source) = Source::from_fourcc(got.fourcc) { + replies.push((got, source)); + } + } + + let Some((best, source)) = closest(replies, want) else { + if offered.is_empty() { + let Some(error) = probe_error else { + return Err(Error::Codec(anyhow::anyhow!("camera {name} has no formats to probe"))); + }; + return Err(error); + } + let offered = offered.join(", "); + let wanted = Source::ALL.map(|source| source.fourcc().to_string()).join(", "); + return Err(Error::Codec(anyhow::anyhow!( + "camera {name} has no encodable {wanted} mode (the driver returned {offered})" + ))); + }; + + // A successful probe may have left the device on another candidate. + let applied = apply(Format::new(best.width, best.height, best.fourcc))?; + if applied.fourcc != best.fourcc || applied.width != best.width || applied.height != best.height { + return Err(Error::Codec(anyhow::anyhow!( + "camera {name} would not re-apply the {}x{} {} mode it just negotiated", + best.width, + best.height, + best.fourcc + ))); + } + Ok((applied, source)) +} + +/// The encodable reply nearest the requested geometry, ties going to the cheaper +/// returned format. +fn closest(replies: impl IntoIterator, want: Size) -> Option<(Format, Source)> { + replies + .into_iter() + .filter(|(format, _)| { + Size::new(format.width, format.height) + .validate("camera resolution") + .is_ok() + }) + .min_by_key(|(format, source)| (distance(*format, want), source.cost())) +} + +/// How far a negotiated mode lands from the requested geometry, summed over both +/// dimensions. Zero is an exact match. +fn distance(format: Format, want: Size) -> u64 { + u64::from(format.width.abs_diff(want.width)) + u64::from(format.height.abs_diff(want.height)) +} + +fn set_format(device: &Device, format: Format) -> Result { + Capture::set_format(device, &format).map_err(|e| Error::Codec(anyhow::anyhow!("V4L2 set format: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reply(width: u32, height: u32, source: Source) -> (Format, Source) { + (Format::new(width, height, source.fourcc()), source) + } + + /// The case that motivates scoring at all, taken from a real UVC webcam: + /// YUYV tops out at VGA while MJPEG reaches 720p, so a 720p request has to + /// land on MJPEG even though YUYV is probed first and answers successfully. + #[test] + fn prefers_the_nearer_mode_over_the_cheaper_one() { + let replies = [reply(640, 480, Source::Yuyv), reply(1280, 720, Source::Mjpeg)]; + let (format, source) = closest(replies, Size::new(1280, 720)).expect("a reply is usable"); + assert_eq!(source, Source::Mjpeg); + assert_eq!((format.width, format.height), (1280, 720)); + } + + /// When both formats reach the requested size, the cheaper one wins regardless + /// of probe order: YUYV resamples, MJPEG costs a full JPEG decode per frame. + #[test] + fn breaks_ties_toward_the_cheaper_format() { + let replies = [reply(640, 480, Source::Mjpeg), reply(640, 480, Source::Yuyv)]; + let (_, source) = closest(replies, Size::new(640, 480)).expect("a reply is usable"); + assert_eq!(source, Source::Yuyv); + } + + /// An exact odd mode cannot feed I420, so a nearby even mode has to win rather + /// than letting `Camera::open` reject the selected result. + #[test] + fn ignores_a_nearer_mode_the_pipeline_cannot_encode() { + let replies = [reply(1279, 719, Source::Mjpeg), reply(1280, 720, Source::Yuyv)]; + let (format, source) = closest(replies, Size::new(1279, 719)).expect("an even reply is usable"); + assert_eq!(source, Source::Yuyv); + assert_eq!((format.width, format.height), (1280, 720)); + } + + /// A YUYV-only driver may reject MJPEG instead of substituting its supported + /// mode, which must not discard the valid reply from the first probe. + #[test] + fn keeps_a_valid_mode_when_another_probe_fails() { + let mut calls = 0; + let (format, source) = negotiate_with("camera", Size::new(640, 480), |requested| { + calls += 1; + match calls { + 1 => Ok(Format::new(640, 480, Source::Yuyv.fourcc())), + 2 => Err(Error::Codec(anyhow::anyhow!("MJPEG is unsupported"))), + 3 => Ok(requested), + _ => panic!("unexpected format probe"), + } + }) + .expect("the YUYV reply is usable"); + + assert_eq!(calls, 3); + assert_eq!(source, Source::Yuyv); + assert_eq!((format.width, format.height), (640, 480)); + } + + /// When every candidate is rejected, preserve the last real driver error. + #[test] + fn returns_an_error_when_every_probe_fails() { + let mut calls = 0; + let error = negotiate_with("camera", Size::new(640, 480), |_| { + calls += 1; + Err(Error::Codec(anyhow::anyhow!("probe {calls} failed"))) + }) + .expect_err("no format probe succeeded"); + + assert_eq!(calls, Source::ALL.len()); + assert_eq!(error.to_string(), "probe 2 failed"); + } + + /// Zero and odd dimensions cannot feed I420, so they leave nothing to score. + #[test] + fn no_usable_reply_is_none() { + let replies = [reply(0, 720, Source::Yuyv), reply(1279, 719, Source::Mjpeg)]; + assert!(closest(replies, Size::new(1280, 720)).is_none()); + } + + /// Distance is symmetric in the two dimensions and zero only on an exact hit, + /// so a mode that overshoots is no better than one that undershoots by as much. + #[test] + fn distance_is_zero_only_on_an_exact_match() { + let want = Size::new(1280, 720); + assert_eq!(distance(Format::new(1280, 720, Source::Yuyv.fourcc()), want), 0); + assert_eq!(distance(Format::new(1280, 600, Source::Yuyv.fourcc()), want), 120); + assert_eq!(distance(Format::new(1280, 840, Source::Yuyv.fourcc()), want), 120); + } + + /// Every format we advertise round-trips through its fourcc, which is what + /// lets `negotiate` recognize the driver's substitution. + #[test] + fn fourcc_round_trips() { + for source in Source::ALL { + assert_eq!(Source::from_fourcc(source.fourcc()), Some(source)); } - last = Some(got.fourcc); + assert_eq!(Source::from_fourcc(FourCC::new(b"GREY")), None); } - Err(Error::Codec(anyhow::anyhow!( - "camera supports neither YUYV nor MJPEG (negotiated {last:?})" - ))) } From 23f4b482e01aaec8f6635d4207d77a27c04beb51 Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Sat, 5 Sep 2026 04:38:44 +0200 Subject: [PATCH 23/29] feat(moq-video): add the Android MediaCodec encoder and decoder (#3354) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Luke Curley --- Cargo.lock | 1 + doc/lib/rs/crate/moq-video.md | 14 +- doc/lib/rs/index.md | 2 +- rs/justfile | 21 + rs/moq-video/Cargo.toml | 15 +- rs/moq-video/README.md | 22 +- rs/moq-video/src/decode/backend/mediacodec.rs | 606 +++++++++++++++ rs/moq-video/src/decode/backend/mod.rs | 23 +- rs/moq-video/src/decode/decoder.rs | 4 +- rs/moq-video/src/encode/backend/mediacodec.rs | 687 ++++++++++++++++++ rs/moq-video/src/encode/backend/mod.rs | 11 +- rs/moq-video/src/encode/encoder.rs | 4 +- rs/moq-video/src/frame.rs | 269 +++++++ rs/moq-video/src/lib.rs | 13 +- 14 files changed, 1658 insertions(+), 34 deletions(-) create mode 100644 rs/moq-video/src/decode/backend/mediacodec.rs create mode 100644 rs/moq-video/src/encode/backend/mediacodec.rs diff --git a/Cargo.lock b/Cargo.lock index 644dbc95e5e..719095eade5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5216,6 +5216,7 @@ dependencies = [ "moq-net", "moq-nvenc", "moq-vaapi", + "ndk", "objc2 0.6.4", "objc2-av-foundation", "objc2-core-foundation", diff --git a/doc/lib/rs/crate/moq-video.md b/doc/lib/rs/crate/moq-video.md index 455f36e8cce..4866cb41d29 100644 --- a/doc/lib/rs/crate/moq-video.md +++ b/doc/lib/rs/crate/moq-video.md @@ -26,8 +26,8 @@ Four role modules, symmetric on both ends of the wire: | Module | Does | Platform | | --- | --- | --- | | `capture` | Camera, display, window, or application frames | AVFoundation + ScreenCaptureKit (macOS), V4L2 + X11/portal + PipeWire (Linux), Media Foundation + DXGI/GDI (Windows) | -| `encode` | Raw frames to H.264/H.265, published through `moq-mux` | VideoToolbox, Media Foundation, NVENC, VAAPI, openh264 | -| `decode` | A subscribed track back to raw frames | VideoToolbox, Media Foundation/DXVA, NVDEC, openh264 | +| `encode` | Raw frames to H.264/H.265, published through `moq-mux` | VideoToolbox, Media Foundation, NVENC, VAAPI, MediaCodec, openh264 | +| `decode` | A subscribed track back to raw frames | VideoToolbox, Media Foundation/DXVA, NVDEC, MediaCodec, openh264 | | `render` | A frame drawn on the GPU, handed back as a `wgpu` texture | wgpu, with zero-copy Metal and Vulkan imports | A picture is a `Frame` wherever it crosses the API: a `moq_net::Timestamp` and a @@ -55,7 +55,7 @@ slow path. AV1 is decode-only, via NVDEC. cargo add moq-video ``` -Capture, the Linux hardware codecs, and the GPU renderer are on by default. +Capture, the platform hardware codecs, and the GPU renderer are on by default. PipeWire screen capture is not, since it links `libpipewire-0.3` at build time: ```bash @@ -65,6 +65,7 @@ cargo add moq-video --features pipewire | Feature | Default | Pulls in | | --- | --- | --- | | `capture` | yes | Native device capture (`v4l` and `zune-jpeg` on Linux) | +| `mediacodec` | yes | MediaCodec encode/decode and `AHardwareBuffer` frames on Android API 26+ | | `nvidia` | yes | NVENC encode and NVDEC decode on Linux (`cudarc`, `moq-nvenc`), dlopen'd at runtime | | `vaapi` | yes | Intel/AMD encode on Linux (`moq-vaapi`), dlopen'd at runtime; not yet validated on hardware | | `v4l2` | yes | V4L2 M2M encode and decode on ARM SoCs (Raspberry Pi and friends), validated on a Raspberry Pi 4 | @@ -72,9 +73,9 @@ cargo add moq-video --features pipewire | `pipewire` | no | Wayland screen capture via xdg-desktop-portal and DMA-BUF | `--no-default-features` gives a codec-only build that still encodes and decodes -H.264 with openh264 but omits native capture, the renderer, and the Linux GPU -backends. A relay or language binding that only handles supplied frames needs -none of them. +H.264 with openh264 but omits native capture, the renderer, and platform GPU +backends. The language bindings use that shape, which keeps their Android API +24 floor instead of opting into MediaCodec's API 26 entry points. ## Publishing @@ -148,6 +149,7 @@ rather than a blanket promise: | macOS | `PixelBuffer` (VideoToolbox) | yes | yes, via `CVMetalTextureCache` | | Linux | `Cuda` (NVDEC) | yes, straight into NVENC | decoded CUDA frames: no; packed PipeWire DMA-BUF capture: yes, via Vulkan | | Windows | `Texture` (Media Foundation / DXVA) | yes, through the Direct3D11 video processor | no, downloaded to I420 first | +| Android | `HardwareBuffer` (MediaCodec) | no, the encoder takes NV12 in a ByteBuffer | not here: the buffer imports through EGL in the application's own GL context | `Frame::resize` stays on the GPU through a `VTPixelTransferSession`, CUDA kernel, or Direct3D11 video processor. Call `Frame::resize_with` with diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index b66d595ee08..571789e6b27 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -72,7 +72,7 @@ GPU rendering of decoded frames. **Features:** -- VideoToolbox, Media Foundation, NVENC/NVDEC, VAAPI, with openh264 as the H.264 software fallback +- VideoToolbox, Media Foundation, NVENC/NVDEC, VAAPI, MediaCodec, with openh264 as the H.264 software fallback - Decoded frames stay on the GPU where the platform allows, so transcoding skips the round trip - `wgpu` renderer handing back a texture you present, importing the decoder's surface zero-copy on macOS - Linux hardware libraries `dlopen`'d, so a build links and starts without a GPU diff --git a/rs/justfile b/rs/justfile index d3a0692c007..8fe8e422830 100644 --- a/rs/justfile +++ b/rs/justfile @@ -411,6 +411,27 @@ windows *args: macos *args: {{ cargo_compile }} check --locked -p moq-video -p moq-audio --all-targets --all-features {{ args }} +# Same idea as `windows`/`macos`, for Android: moq-video's MediaCodec encoder and +# decoder, its `Surface::HardwareBuffer` variant, and the `frame::android` module +# are all `#[cfg(target_os = "android")]`, so a host-target check compiles none of +# them and every error in them stays invisible. +# +# Unlike `windows` and `macos` this needs no special host: the Android targets +# cross-compile from the Linux dev shell in about half a minute. It needs an NDK, +# which `cargo-ndk` finds through `ANDROID_NDK_HOME` or the standard SDK layout, +# so it is a manual gate rather than a CI one until the shell provides one. +# +# arm64-v8a alone: it is the only ABI anything here ships, and the code is not +# ABI-specific, so the other two would compile the same source twice more. +# +# `check` rather than a `-D warnings` clippy, matching `windows` and `macos`: +# the capture module's plumbing is dead on a target with no capture backend, so +# a denying lint would fail on code this recipe is not here to police. + +# Compile the Android backends, which the host-target check skips. +android *args: + cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets {{ args }} + # Same idea as `windows`/`macos`, for the browser: moq-wasm's whole crate root is # `#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` # compiles an empty crate and every error in it stays invisible. Only a wasm32 diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index defb2fd6bfe..1b8c059a8d7 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -20,11 +20,15 @@ categories = ["multimedia", "multimedia::video", "multimedia::encoding"] # it links libpipewire at build time. Workspace consumers opt out at the root # manifest (`default-features = false`) and pick per crate, so the relay and the # language bindings never build the device or render stacks. -default = ["capture", "nvidia", "vaapi", "v4l2", "render"] +default = ["capture", "mediacodec", "nvidia", "vaapi", "v4l2", "render"] # Native camera and screen capture. Enabled by default for direct consumers, # but workspace consumers opt in so codec-only bindings avoid device build # dependencies such as V4L2 and libclang. capture = ["dep:v4l", "dep:x11rb", "dep:zune-jpeg"] +# Android MediaCodec encode/decode and its AHardwareBuffer frame surface. Kept +# behind a feature because these NDK entry points require API 26, while the +# language-binding artifacts still support API 24 and disable default features. +mediacodec = ["dep:ndk"] # The NVIDIA codecs on Linux: the NVENC encoder and the NVDEC decoder. One # feature because they are one dependency stack (cudarc + moq-nvenc, plus a # libloading probe for the driver libs) and one code path: NVDEC hands NVENC a @@ -116,6 +120,15 @@ wgpu = { version = "30", optional = true } # RGBA -> I420 color conversion (replaces ffmpeg swscale). yuv = "0.8.14" +[target.'cfg(target_os="android")'.dependencies] +# MediaCodec (hardware H.264/H.265 encode and decode), ImageReader, and +# AHardwareBuffer, all from the NDK that is already on the path of anything +# targeting Android. `default-features = false` drops the raw-window-handle +# integration, which nothing here uses; `api-level-26` is what unlocks +# AMediaCodec_setParameters (live bitrate and keyframe requests) and the +# hardware buffer behind a decoded frame. +ndk = { version = "0.9", optional = true, default-features = false, features = ["media", "api-level-26"] } + [target.'cfg(target_os="linux")'.dependencies] # xdg-desktop-portal ScreenCast negotiation for the `pipewire` feature (pure Rust, # zbus over DBus). The portal owns source selection: the user picks a monitor in a diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index 1a0e67ed0c0..ba41a34752e 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -61,10 +61,10 @@ The codec is chosen via `encode::Codec`. Backends are tried in order (hardware first, then software) and the first that opens wins; `encode::Kind` narrows the choice (`Auto` / `Hardware` / `Software` / a named backend). -| Codec | Software | macOS | Windows | Linux | -|---|---|---|---|---| -| H.264 | openh264 (vendored, static) | VideoToolbox | Media Foundation | NVENC (feature `nvidia`), VAAPI (feature `vaapi`) | -| H.265 | none | VideoToolbox | Media Foundation | NVENC (feature `nvidia`) | +| Codec | Software | macOS | Windows | Linux | Android | +|---|---|---|---|---|---| +| H.264 | openh264 (vendored, static) | VideoToolbox | Media Foundation | NVENC (feature `nvidia`), VAAPI (feature `vaapi`) | MediaCodec (feature `mediacodec`, API 26+) | +| H.265 | none | VideoToolbox | Media Foundation | NVENC (feature `nvidia`) | MediaCodec (feature `mediacodec`, API 26+) | Every backend emits Annex-B with in-band parameter sets (SPS/PPS, plus VPS for H.265), so the matching `moq_mux::codec` importer handles framing and catalog @@ -105,8 +105,8 @@ same device keeps it there (the transcode path), while `into_i420()` downloads it. An encoder that can't take that surface (openh264, or a different device) downloads it through I420 for you. Every frame carries a `Surface`, a `#[non_exhaustive]` enum naming where the pixels live (`PixelBuffer` on macOS, -`Texture` on Windows, `Cuda` on Linux, or CPU `I420`). Match it to take a -zero-copy path for a representation you recognize, and fall back to +`Texture` on Windows, `Cuda` on Linux, `HardwareBuffer` on Android, or CPU +`I420`). Match it to take a zero-copy path for a representation you recognize, and fall back to `Surface::into_i420()`, which always works. On macOS `Surface::into_pixel_buffer()` is the mirror: free for a hardware-decoded frame, an upload for a CPU one. `Surface::into_rgba()` is the portable exit for CPU image and UI toolkits, @@ -114,11 +114,11 @@ returning owned, tightly packed RGBA8 pixels with the surface's color metadata applied. Backends are tried hardware-first, like encode: -| Codec | Software | macOS | Windows | Linux | -|---|---|---|---|---| -| H.264 | openh264 (vendored, static) | VideoToolbox | Media Foundation (DXVA) | NVDEC (feature `nvidia`) | -| H.265 | none | VideoToolbox | Media Foundation (DXVA) | NVDEC (feature `nvidia`) | -| AV1 | none | none | none | NVDEC (feature `nvidia`) | +| Codec | Software | macOS | Windows | Linux | Android | +|---|---|---|---|---|---| +| H.264 | openh264 (vendored, static) | VideoToolbox | Media Foundation (DXVA) | NVDEC (feature `nvidia`) | MediaCodec (feature `mediacodec`, API 26+) | +| H.265 | none | VideoToolbox | Media Foundation (DXVA) | NVDEC (feature `nvidia`) | MediaCodec (feature `mediacodec`, API 26+) | +| AV1 | none | none | none | NVDEC (feature `nvidia`) | MediaCodec (feature `mediacodec`, when the device provides it) | On macOS VideoToolbox decodes H.264 and H.265 on hardware, pulling the parameter sets (SPS/PPS, plus VPS for H.265) out of each keyframe to build the format diff --git a/rs/moq-video/src/decode/backend/mediacodec.rs b/rs/moq-video/src/decode/backend/mediacodec.rs new file mode 100644 index 00000000000..c9b6f295c19 --- /dev/null +++ b/rs/moq-video/src/decode/backend/mediacodec.rs @@ -0,0 +1,606 @@ +//! Hardware H.264 / H.265 / AV1 decode backend via Android MediaCodec +//! (`AMediaCodec`). +//! +//! The inverse of the encode MediaCodec backend, and the one place the two +//! differ in shape: the decoder is configured with an `ImageReader`'s surface as +//! its output rather than with CPU output buffers. A decoded picture is then an +//! `AHardwareBuffer` a GL or Vulkan consumer imports directly, which is what +//! [`Surface::HardwareBuffer`] carries, and the read-back to I420 takes the row +//! and pixel strides off the image instead of guessing at the device's padding. +//! The ByteBuffer output path offers neither. +//! +//! Access units arrive Annex-B with the parameter sets inline ahead of each +//! keyframe (the front end converts avc1 / hvc1 for us), which MediaCodec takes +//! as-is, so no `csd-0` / `csd-1` configuration is needed and nothing here parses +//! an avcC record. +//! +//! The codec is fed access units and hands back output buffers in display order. +//! Releasing one for rendering propagates its presentation time to the image in +//! nanoseconds, so each acquired image carries its own timestamp even if the +//! surface drops an excessive frame. +//! +//! Frames hold slots in the reader's queue, so a consumer that hoards them +//! stalls decoding. That is the same trade the VideoToolbox and NVDEC backends +//! make: draw and drop. + +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use moq_net::Timestamp; +use ndk::hardware_buffer::HardwareBufferUsage; +use ndk::media::image_reader::{AcquireResult, Image, ImageFormat, ImageReader}; +use ndk::media::media_codec::{self, DequeuedInputBufferResult, DequeuedOutputBufferInfoResult, MediaCodecDirection}; +use ndk::media::media_format::MediaFormat; +use ndk::media_error::MediaError; + +use super::{Backend, Codec, Config}; +use crate::frame::{ + Surface, + android::{HardwareBuffer, Reader}, +}; +use crate::{Error, Frame}; + +pub(crate) const NAME: &str = "mediacodec"; + +/// The MIME types MediaCodec names the codecs by. +const MIME_H264: &str = "video/avc"; +const MIME_H265: &str = "video/hevc"; +const MIME_AV1: &str = "video/av01"; + +// `AMediaFormat` keys. The NDK ships them as `AMEDIAFORMAT_KEY_*` string +// constants that the `ndk` crate does not re-export; they are the same strings +// `android.media.MediaFormat` documents. +const KEY_MIME: &str = "mime"; +const KEY_WIDTH: &str = "width"; +const KEY_HEIGHT: &str = "height"; +const KEY_ALLOW_FRAME_DROP: &str = "allow-frame-drop"; +const KEY_LOW_LATENCY: &str = "low-latency"; +const KEY_PRIORITY: &str = "priority"; + +/// `PRIORITY_REALTIME`: this is a live stream, not a file being played back. +const PRIORITY_REALTIME: i32 = 0; + +/// `AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG`, likewise not re-exported by the `ndk` +/// crate. +const FLAG_CODEC_CONFIG: u32 = 2; +const FLAG_END_OF_STREAM: u32 = 4; + +/// The size the reader's queue is created at. +/// +/// Only a default: MediaCodec sets the real geometry on the window once it has +/// parsed the stream, and each image reports what it actually got, which is what +/// a frame's size comes from. Sized for the common case so the queue usually +/// does not have to reallocate. +const DEFAULT_SIZE: (i32, i32) = (1920, 1080); + +/// How many decoded pictures the reader's queue holds. +/// +/// It has to cover the decoder's reference pictures (4 or so for H.264), the +/// consumer's playout buffer, and whatever is in flight between them; anything +/// less and the decoder stalls waiting for a slot that a consumer holding two +/// frames could have freed. +const QUEUE_DEPTH: i32 = 8; + +/// How long to wait for a free input buffer on each round. +const INPUT_TIMEOUT: Duration = Duration::from_millis(10); + +/// Poll for decoded pictures without waiting. [`MediaCodec::decode`] drains on +/// both sides of every submission, so a picture that is not ready yet is +/// collected on the next access unit rather than waited for here. +const OUTPUT_TIMEOUT: Duration = Duration::ZERO; + +/// How long each round of an end-of-stream drain waits for codec output or a +/// rendered image. +const DRAIN_TIMEOUT: Duration = Duration::from_millis(50); + +/// How many rounds a tail drain runs before giving up on a wedged codec. +const DRAIN_ROUNDS: u32 = 20; + +/// How many rounds a submission spends freeing an input buffer before giving up. +const SUBMIT_ROUNDS: u32 = 20; + +pub(crate) struct MediaCodec { + /// Declared first so it is dropped first: the codec writes into the reader's + /// queue, so it has to stop before the reader can go. + codec: media_codec::MediaCodec, + /// Shared with every frame handed out, since deleting the reader invalidates + /// images already acquired from it. + reader: Arc, + /// Woken by the reader when a rendered output becomes available. Rendering to + /// a surface is asynchronous with respect to releasing the codec buffer, so + /// the end-of-stream drain needs this rather than polling or sleeping. + images: Arc, + /// Last image-listener generation already observed by `collect`. + image_generation: u64, + /// Codec output buffers released for rendering but not yet acquired as images. + pending_images: usize, + /// Whether this stream has accepted any input and therefore needs EOS. + fed: bool, + /// Whether the last acquire found every slot held, so the stall is reported + /// once on the way in rather than on every poll. + stalled: bool, +} + +#[derive(Default)] +struct ImageSignal { + generation: Mutex, + ready: Condvar, +} + +impl ImageSignal { + fn notify(&self) { + let mut generation = self.generation.lock().unwrap_or_else(|e| e.into_inner()); + *generation = generation.wrapping_add(1); + self.ready.notify_one(); + } + + fn current(&self) -> u64 { + *self.generation.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn wait(&self, generation: u64, timeout: Duration) -> u64 { + let current = self.generation.lock().unwrap_or_else(|e| e.into_inner()); + let (current, _) = self + .ready + .wait_timeout_while(current, timeout, |current| *current == generation) + .unwrap_or_else(|e| e.into_inner()); + *current + } +} + +// SAFETY: `AMediaCodec` and `AImageReader` are owned handles with no thread +// affinity; the NDK only requires that calls on one of them are serialized, +// which they are because every method here takes `&mut self` and one decode task +// owns the backend outright. `Send` is what lets the boxed trait object satisfy +// `Backend: Send`. +unsafe impl Send for MediaCodec {} + +impl MediaCodec { + /// Open a decoder for `codec`. + /// + /// `config` is accepted for signature parity: MediaCodec decodes at the + /// stream's native size and has no scaler to point + /// [`Config::resize`](crate::decode::Config) at. + pub(crate) fn open(codec: Codec, _config: &Config) -> Result, Error> { + let mime = match codec { + Codec::H264 => MIME_H264, + Codec::H265 => MIME_H265, + Codec::Av1 => MIME_AV1, + }; + + // `GPU_SAMPLED_IMAGE` is what a consumer importing the buffer as a texture + // needs; `CPU_READ_OFTEN` is what makes the read-back in + // `Surface::into_i420` work at all, at the cost of a layout the device may + // otherwise have tiled. + let (width, height) = DEFAULT_SIZE; + let mut reader = ImageReader::new_with_usage( + width, + height, + ImageFormat::YUV_420_888, + HardwareBufferUsage::GPU_SAMPLED_IMAGE | HardwareBufferUsage::CPU_READ_OFTEN, + QUEUE_DEPTH, + ) + .map_err(|e| reader_err("create an ImageReader", e))?; + let images = Arc::new(ImageSignal::default()); + let ready = images.clone(); + reader + .set_image_listener(Box::new(move |_| ready.notify())) + .map_err(|e| reader_err("register the image listener", e))?; + // `HardwareBuffer::buffer` acquires an extra reference for the caller. + // Android requires a removal listener whenever that can happen. The owned + // reference already keeps the allocation alive, so there is no cleanup to + // perform in the notification itself. + reader + .set_buffer_removed_listener(Box::new(|_, _| {})) + .map_err(|e| reader_err("register the buffer removal listener", e))?; + let surface = reader.window().map_err(|e| reader_err("get the reader's window", e))?; + + let decoder = media_codec::MediaCodec::from_decoder_type(mime) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("no MediaCodec decoder for {mime}")))?; + let format = decoder_format(mime, width, height); + decoder + .configure(&format, Some(&surface), MediaCodecDirection::Decoder) + .map_err(|e| codec_err("configure", e))?; + decoder.start().map_err(|e| codec_err("start", e))?; + + tracing::info!(decoder = NAME, codec = codec.label(), "opened video decoder"); + Ok(Box::new(Self { + codec: decoder, + reader: Arc::new(Reader::new(reader)), + images, + image_generation: 0, + pending_images: 0, + fed: false, + stalled: false, + })) + } + + /// Copy one access unit into a free input buffer. + /// + /// Waits for a buffer rather than dropping the access unit when the codec has + /// none free: a decoder that skips one produces a broken picture until the + /// next keyframe, which is a far worse trade than the wait. + fn submit(&mut self, access_unit: &[u8], timestamp: Timestamp, out: &mut Vec) -> Result<(), Error> { + let time = timestamp.as_micros().min(u64::MAX as u128) as u64; + + for _ in 0..SUBMIT_ROUNDS { + let queued = match self + .codec + .dequeue_input_buffer(INPUT_TIMEOUT) + .map_err(|e| codec_err("dequeue an input buffer", e))? + { + DequeuedInputBufferResult::Buffer(mut buffer) => { + let target = buffer.buffer_mut(); + if target.len() < access_unit.len() { + return Err(Error::Codec(anyhow::anyhow!( + "MediaCodec input buffer is {} bytes, needs {} for this access unit", + target.len(), + access_unit.len() + ))); + } + // SAFETY: `target` is at least `access_unit.len()` bytes (checked + // just above), the two are distinct allocations (a codec input buffer + // and the caller's payload), and `MaybeUninit` shares the layout + // of `u8`, so copying initialized bytes over it is what initializes + // it. + unsafe { + std::ptr::copy_nonoverlapping( + access_unit.as_ptr(), + target.as_mut_ptr().cast::(), + access_unit.len(), + ); + } + self.codec + .queue_input_buffer(buffer, 0, access_unit.len(), time, 0) + .map_err(|e| codec_err("queue an input buffer", e))?; + self.fed = true; + true + } + DequeuedInputBufferResult::TryAgainLater => false, + }; + if queued { + return Ok(()); + } + + // Every input buffer is still with the codec; collecting its output is + // what frees one. + self.drain(OUTPUT_TIMEOUT, out)?; + } + + Err(Error::Codec(anyhow::anyhow!( + "MediaCodec never freed an input buffer for an access unit" + ))) + } + + /// Render every decoded picture the codec has ready and collect the images + /// they turn into. + fn drain(&mut self, timeout: Duration, out: &mut Vec) -> Result { + loop { + let ended = match self + .codec + .dequeue_output_buffer(timeout) + .map_err(|e| codec_err("dequeue an output buffer", e))? + { + DequeuedOutputBufferInfoResult::Buffer(buffer) => { + let info = *buffer.info(); + // Never `OutputBuffer::buffer`: a codec configured with a surface has + // no CPU-visible output buffer, and asking for one panics. An empty + // buffer is the codec's own marker rather than a picture, and a + // codec-config one is the parameter sets echoed back. + let render = info.flags() & FLAG_CODEC_CONFIG == 0 && info.size() > 0; + self.codec + .release_output_buffer(buffer, render) + .map_err(|e| codec_err("release an output buffer", e))?; + if render { + self.pending_images += 1; + } + info.flags() & FLAG_END_OF_STREAM != 0 + } + DequeuedOutputBufferInfoResult::TryAgainLater => { + self.collect(out)?; + return Ok(false); + } + // The stream's geometry is read off each image rather than off the + // format, and the parameter sets are inline, so neither of these + // changes anything here. + DequeuedOutputBufferInfoResult::OutputFormatChanged + | DequeuedOutputBufferInfoResult::OutputBuffersChanged => false, + }; + + // Eagerly, because releasing for rendering queues the picture into the + // reader before it returns, so the image is normally there already and + // waiting for the next access unit would add a frame of latency. + self.collect(out)?; + if ended { + return Ok(true); + } + } + } + + /// Acquire every rendered picture the reader has ready. + fn collect(&mut self, out: &mut Vec) -> Result<(), Error> { + loop { + match self + .reader + .acquire_next_image() + .map_err(|e| reader_err("acquire an image", e))? + { + AcquireResult::Image(image) => { + self.pending_images = self.pending_images.saturating_sub(1); + // MediaCodec propagates the output buffer's presentation time to a + // rendered surface in nanoseconds. Read it from the image itself so a + // surface-dropped frame cannot shift every later timestamp. + let nanos = image + .timestamp() + .map_err(|e| reader_err("read an image's timestamp", e))?; + let timestamp = Timestamp::from_nanos(nanos.max(0) as u64)?; + let (left, top, width, height) = image_size(&image)?; + let buffer = HardwareBuffer::new(self.reader.clone(), image, left, top, width, height); + out.push(Frame::new(Surface::HardwareBuffer(buffer), timestamp)); + } + // The picture is on its way but not queued yet, so it comes out on the + // next round rather than being lost. + AcquireResult::NoBufferAvailable => { + self.image_generation = self.images.current(); + self.stalled = false; + break; + } + // Every slot in the queue is held by a consumer, so there is nowhere + // for this picture to go until one is dropped. Decoding is stalled + // behind whoever is hoarding frames, which is worth saying out loud. + AcquireResult::MaxImagesAcquired => { + // Once, on the way into the stall. `collect` runs two or three + // times per access unit, so warning on every poll would log at + // frame rate for as long as the consumer holds its frames. + if !self.stalled { + self.stalled = true; + tracing::warn!( + decoder = NAME, + depth = QUEUE_DEPTH, + "every decoded frame is still held by a consumer; decoding is stalled" + ); + } + break; + } + } + } + Ok(()) + } + + /// Signal end of input and wait until every delayed output buffer is released. + fn drain_tail(&mut self) -> Result, Error> { + let mut out = Vec::new(); + if !self.fed { + return Ok(out); + } + + self.signal_end_of_input(&mut out)?; + let mut ended = false; + for _ in 0..DRAIN_ROUNDS { + if self.drain(DRAIN_TIMEOUT, &mut out)? { + ended = true; + break; + } + } + if !ended { + return Err(Error::Codec(anyhow::anyhow!( + "MediaCodec did not reach end of stream within {:?}", + DRAIN_TIMEOUT * DRAIN_ROUNDS + ))); + } + + // Releasing an output buffer only schedules the surface render. Wait for the + // ImageReader callback so the final pictures are part of this stream rather + // than appearing after the next keyframe. + for _ in 0..DRAIN_ROUNDS { + self.collect(&mut out)?; + if self.pending_images == 0 { + break; + } + self.image_generation = self.images.wait(self.image_generation, DRAIN_TIMEOUT); + } + if self.pending_images > 0 { + tracing::warn!( + decoder = NAME, + dropped = self.pending_images, + "rendered decoder outputs never arrived at the ImageReader" + ); + self.pending_images = 0; + } + + Ok(out) + } + + /// Queue the empty input buffer that marks the end of a ByteBuffer stream. + fn signal_end_of_input(&mut self, out: &mut Vec) -> Result<(), Error> { + for _ in 0..DRAIN_ROUNDS { + let queued = match self + .codec + .dequeue_input_buffer(INPUT_TIMEOUT) + .map_err(|e| codec_err("dequeue an input buffer", e))? + { + DequeuedInputBufferResult::Buffer(buffer) => { + self.codec + .queue_input_buffer(buffer, 0, 0, 0, FLAG_END_OF_STREAM) + .map_err(|e| codec_err("queue end of stream", e))?; + true + } + DequeuedInputBufferResult::TryAgainLater => false, + }; + if queued { + return Ok(()); + } + self.drain(OUTPUT_TIMEOUT, out)?; + } + + Err(Error::Codec(anyhow::anyhow!( + "MediaCodec never freed an input buffer for the end of stream" + ))) + } +} + +impl Backend for MediaCodec { + fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result, Error> { + let mut out = Vec::new(); + // Collect what the codec finished while the caller was elsewhere, which is + // also what frees the input buffer the submission below needs. + self.drain(OUTPUT_TIMEOUT, &mut out)?; + self.submit(&access_unit, timestamp, &mut out)?; + self.drain(OUTPUT_TIMEOUT, &mut out)?; + Ok(out) + } + + fn flush(&mut self) -> Result, Error> { + let out = self.drain_tail()?; + if self.fed { + // EOS leaves the codec refusing input. Synchronous MediaCodec resumes after + // flush without another start call, and the front end requires the next + // access unit to be a keyframe after this method returns. + self.codec.flush().map_err(|e| codec_err("flush", e))?; + self.fed = false; + } + Ok(out) + } + + fn name(&self) -> &str { + NAME + } +} + +/// The `AMediaFormat` describing the stream to decode. +/// +/// The dimensions are a starting guess that the first keyframe's parameter sets +/// correct, since nothing upstream of a backend knows the coded size. The +/// parameter sets ride the bitstream, so there is no `csd-0` / `csd-1` to set. +fn decoder_format(mime: &str, width: i32, height: i32) -> MediaFormat { + let mut format = MediaFormat::new(); + format.set_str(KEY_MIME, mime); + format.set_i32(KEY_WIDTH, width); + format.set_i32(KEY_HEIGHT, height); + // ImageReader is part of the decoded stream rather than a presentation sink. + // Keep backpressure explicit through its acquired-image limit instead of + // letting Android Q+ silently discard a surface frame. Older versions ignore + // this key, and image timestamps still keep later frames correctly paired. + format.set_i32(KEY_ALLOW_FRAME_DROP, 0); + // Ask the decoder to hold as little as it can, which is the difference + // between a live stream and a file. A hint an older device drops. + format.set_i32(KEY_LOW_LATENCY, 1); + format.set_i32(KEY_PRIORITY, PRIORITY_REALTIME); + format +} + +/// The picture's visible size, both dimensions rounded down to even. +/// +/// The crop rectangle rather than the buffer dimensions: a coded picture is +/// padded up to a macroblock multiple, so 1080 lines arrive as a 1088-line +/// buffer, and the crop is what says how much of it is the picture. Taken only +/// when the crop is anchored at the origin, since the read-back walks each plane +/// from there; a decoder that offsets its crop gets the whole buffer rather than +/// a picture shifted by the offset. +fn image_size(image: &Image) -> Result<(u32, u32, u32, u32), Error> { + let width = image.width().map_err(|e| reader_err("read an image's width", e))?; + let height = image.height().map_err(|e| reader_err("read an image's height", e))?; + let crop = image.crop_rect().map_err(|e| reader_err("read an image's crop", e))?; + + let (mut left, mut top, mut w, mut h) = (0, 0, width, height); + let (cropped_w, cropped_h) = (crop.right - crop.left, crop.bottom - crop.top); + if crop.left >= 0 && crop.top >= 0 && crop.right <= width && crop.bottom <= height && cropped_w > 0 && cropped_h > 0 + { + (left, top, w, h) = (crop.left, crop.top, cropped_w, cropped_h); + } + + // 4:2:0 chroma is 2x2, so an odd dimension has no whole chroma sample to go + // with its last row or column. + let (w, h) = (w.max(0) as u32 & !1, h.max(0) as u32 & !1); + if w == 0 || h == 0 { + return Err(Error::Codec(anyhow::anyhow!( + "MediaCodec produced a {width}x{height} image, which is not a picture" + ))); + } + Ok((left as u32, top as u32, w, h)) +} + +/// Wrap an NDK media error from the codec, naming the call that produced it. +fn codec_err(what: &str, error: MediaError) -> Error { + Error::Codec(anyhow::anyhow!("failed to {what} on the MediaCodec decoder: {error}")) +} + +/// Wrap an NDK media error from the reader, naming the call that produced it. +fn reader_err(what: &str, error: MediaError) -> Error { + Error::Codec(anyhow::anyhow!( + "failed to {what} on the decoder's ImageReader: {error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The hardware round trip: encode a picture with the MediaCodec encoder and + /// decode it back, which is the only way to see that the surface output path + /// produces a frame at all. + #[test] + #[ignore = "needs an Android device with MediaCodec H.264 hardware"] + fn decodes_what_the_encoder_produced() { + use crate::encode::{Config as EncodeConfig, Kind as EncodeKind}; + + let size = crate::Size::new(320, 240); + let mut config = EncodeConfig::new(size.width, size.height, 30); + config.kind = EncodeKind::Named(NAME.to_owned()); + let mut encoder = crate::encode::Encoder::new(&config).expect("a MediaCodec encoder"); + + let i420 = crate::I420::new( + size.width, + size.height, + vec![0x80; crate::I420::len(size.width, size.height)], + ) + .unwrap(); + + let mut decoder = MediaCodec::open(Codec::H264, &Config::new()).expect("a MediaCodec decoder"); + let mut frames = Vec::new(); + for index in 0..30u64 { + let timestamp = Timestamp::from_micros(index * 33_333).unwrap(); + let frame = Frame::new(Surface::I420(i420.clone()), timestamp); + encoder.keyframe(); + for encoded in encoder.encode(&frame).unwrap() { + frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + } + } + for encoded in encoder.flush().unwrap() { + frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + } + frames.extend(decoder.flush().unwrap()); + + // A flush drains the previous stream and leaves the same decoder reusable. + let timestamp = Timestamp::from_micros(1_000_000).unwrap(); + let frame = Frame::new(Surface::I420(i420.clone()), timestamp); + encoder.keyframe(); + for encoded in encoder.encode(&frame).unwrap() { + frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + } + for encoded in encoder.flush().unwrap() { + frames.extend(decoder.decode(encoded.payload, encoded.timestamp, true).unwrap()); + } + frames.extend(decoder.flush().unwrap()); + + let frame = frames.first().expect("at least one decoded frame"); + assert_eq!(frame.size(), size); + assert!( + matches!(frame.surface, Surface::HardwareBuffer(_)), + "a decoded picture should stay in its hardware buffer", + ); + + // And it has to survive the read-back, which is the arm every CPU + // consumer takes and the only exercise `download_i420`'s stride walking + // gets. Mid-gray in, mid-gray out. + let frames = frames.into_iter().next().expect("checked above"); + let i420 = frames.surface.into_i420().expect("read back to I420"); + assert_eq!(i420.len(), crate::I420::len(size.width, size.height)); + let luma = &i420[..(size.width * size.height) as usize]; + assert!( + luma.iter().all(|byte| byte.abs_diff(0x80) <= 8), + "the read-back luma plane should still be mid-gray", + ); + } +} diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index e5cb5a5ed50..a9ee12dacc3 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -8,10 +8,10 @@ //! //! [`open`] picks the best backend for a [`Codec`] and [`Config`], trying //! hardware candidates (platform-gated: VideoToolbox on macOS, Media Foundation -//! / DXVA on Windows, NVDEC then V4L2 on Linux) before the openh264 software -//! fallback, exactly like the encode side. Only backends that support the requested codec -//! are considered: there is no software H.265 or AV1 decoder, so those tracks -//! have no fallback below the hardware path. +//! / DXVA on Windows, MediaCodec on Android, NVDEC then V4L2 on Linux) before +//! the openh264 software fallback, exactly like the encode side. Only backends +//! that support the requested codec are considered: there is no software H.265 +//! or AV1 decoder, so those tracks have no fallback below the hardware path. use bytes::Bytes; use moq_net::Timestamp; @@ -30,6 +30,9 @@ mod videotoolbox; #[cfg(target_os = "windows")] mod mediafoundation; +#[cfg(all(target_os = "android", feature = "mediacodec"))] +mod mediacodec; + #[cfg(all(target_os = "linux", feature = "nvidia"))] mod nvdec; @@ -62,9 +65,9 @@ pub(crate) trait Backend: Send { /// Decode one access unit stamped with its presentation `timestamp`. /// `keyframe` marks a random-access frame. Takes an owned [`Bytes`] so a /// backend can split codec units without copying. - /// Backends that decode one-in one-out echo the input timestamp; NVDEC threads - /// timestamps through its parser, so they survive decoder delay and frame - /// reordering. + /// Backends that decode one-in one-out echo the input timestamp; NVDEC and + /// MediaCodec thread timestamps through the codec, so they survive decoder + /// delay and frame reordering. fn decode(&mut self, access_unit: Bytes, timestamp: Timestamp, keyframe: bool) -> Result, Error>; /// Return the pictures the codec still holds once the stream has ended. @@ -103,6 +106,12 @@ const HARDWARE: &[Candidate] = &[ supports: |c| matches!(c, Codec::H264 | Codec::H265), open: mediafoundation::MediaFoundation::open, }, + #[cfg(all(target_os = "android", feature = "mediacodec"))] + Candidate { + name: mediacodec::NAME, + supports: |c| matches!(c, Codec::H264 | Codec::H265 | Codec::Av1), + open: mediacodec::MediaCodec::open, + }, #[cfg(all(target_os = "linux", feature = "nvidia"))] Candidate { name: nvdec::NAME, diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 1832fe3a21f..e4a8e9aec07 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -37,8 +37,8 @@ pub enum Kind { Hardware, /// Software (openh264) only. Software, - /// A specific backend by name, e.g. `"videotoolbox"`, `"nvdec"`, `"v4l2"`, - /// or `"openh264"`. + /// A specific backend by name, e.g. `"videotoolbox"`, `"mediacodec"`, + /// `"nvdec"`, `"v4l2"`, or `"openh264"`. Named(String), } diff --git a/rs/moq-video/src/encode/backend/mediacodec.rs b/rs/moq-video/src/encode/backend/mediacodec.rs new file mode 100644 index 00000000000..2ddecafe2a2 --- /dev/null +++ b/rs/moq-video/src/encode/backend/mediacodec.rs @@ -0,0 +1,687 @@ +//! Hardware H.264 / H.265 backend via Android MediaCodec (`AMediaCodec`). +//! +//! Runs the codec in synchronous ByteBuffer mode: each raw frame is written into +//! a dequeued input buffer as NV12, and the finished access units are drained +//! back out. MediaCodec emits Annex-B, which is what the matching catalog +//! importer wants (`moq_mux` avc3 / hev1 mode), so unlike VideoToolbox there is +//! no AVCC rewrite. The parameter sets arrive once as a codec-config buffer and +//! are prepended to every keyframe that doesn't already carry them, which is +//! what makes each IDR independently decodable. +//! +//! The codec is a queued device: it encodes frame N while frame N+k goes in, so +//! an access unit is stamped with the frame it belongs to, found through the +//! sample time the codec echoes back in its `BufferInfo`, rather than with +//! whatever frame happens to be going in at the time. +//! +//! Two things the NDK only exposes from API 28, above the API 26 this crate +//! builds against: +//! +//! - The input buffer geometry (`AMediaCodec_getInputFormat`), so the NV12 is +//! written tightly packed at the configured resolution. A device whose encoder +//! pads its input rows is the gap. +//! - The name of the codec that was opened (`AMediaCodec_getName`), so a device +//! with no hardware encoder for the requested codec gets the AOSP software one +//! under this name instead of falling through to openh264. MediaCodec hands +//! back the device's preferred encoder, which is the hardware one wherever +//! there is one. The visible consequence is that +//! [`Kind::Hardware`](crate::encode::Kind::Hardware) cannot be enforced here: +//! this backend sits in the hardware table and answers even where the device +//! has only a software encoder. +//! +//! The codec is created, driven, and dropped on the dedicated encode thread (see +//! `encode::sink`), which is what lets the blocking waits here not park a tokio +//! worker. + +use std::collections::VecDeque; +use std::mem::MaybeUninit; +use std::time::Duration; + +use bytes::{BufMut, Bytes, BytesMut}; +use moq_net::Timestamp; +use ndk::media::media_codec::{ + self, BufferInfo, DequeuedInputBufferResult, DequeuedOutputBufferInfoResult, MediaCodecDirection, OutputBuffer, +}; +use ndk::media::media_format::MediaFormat; +use ndk::media_error::MediaError; + +use super::super::encoder::{Codec, Config}; +use super::{Backend, Encoded}; +use crate::{Color, Error, Frame, I420}; + +pub(crate) const NAME: &str = "mediacodec"; + +/// The MIME types MediaCodec names the two codecs by. +const MIME_H264: &str = "video/avc"; +const MIME_H265: &str = "video/hevc"; + +// `AMediaFormat` keys. The NDK ships them as `AMEDIAFORMAT_KEY_*` string +// constants that the `ndk` crate does not re-export; they are the same strings +// `android.media.MediaFormat` documents. +const KEY_MIME: &str = "mime"; +const KEY_WIDTH: &str = "width"; +const KEY_HEIGHT: &str = "height"; +const KEY_BIT_RATE: &str = "bitrate"; +const KEY_BITRATE_MODE: &str = "bitrate-mode"; +const KEY_FRAME_RATE: &str = "frame-rate"; +const KEY_COLOR_FORMAT: &str = "color-format"; +const KEY_COLOR_STANDARD: &str = "color-standard"; +const KEY_COLOR_TRANSFER: &str = "color-transfer"; +const KEY_COLOR_RANGE: &str = "color-range"; +const KEY_I_FRAME_INTERVAL: &str = "i-frame-interval"; +const KEY_LATENCY: &str = "latency"; +const KEY_LOW_LATENCY: &str = "low-latency"; +const KEY_MAX_B_FRAMES: &str = "max-bframes"; +const KEY_PRIORITY: &str = "priority"; +const KEY_REQUEST_SYNC_FRAME: &str = "request-sync"; +const KEY_VIDEO_BITRATE: &str = "video-bitrate"; + +/// `COLOR_FormatYUV420SemiPlanar`: NV12, a full-size luma plane followed by one +/// interleaved chroma plane. The one raw format every MediaCodec encoder +/// accepts. +const COLOR_FORMAT_NV12: i32 = 21; + +/// `BITRATE_MODE_CBR`. A live track is sized by its uplink, not by its content, +/// so a constant rate is what the congestion controller's estimate means. +const BITRATE_MODE_CBR: i32 = 2; + +/// `PRIORITY_REALTIME`: this is a live stream, not a file being transcoded. +const PRIORITY_REALTIME: i32 = 0; + +// `MediaFormat.COLOR_STANDARD_*` / `COLOR_TRANSFER_*` / `COLOR_RANGE_*`, the +// codes MediaCodec translates into the bitstream's VUI. +const COLOR_STANDARD_BT709: i32 = 1; +const COLOR_STANDARD_BT601_NTSC: i32 = 4; +const COLOR_TRANSFER_SDR_VIDEO: i32 = 3; +const COLOR_RANGE_FULL: i32 = 1; +const COLOR_RANGE_LIMITED: i32 = 2; + +// `AMEDIACODEC_BUFFER_FLAG_*`, likewise not re-exported by the `ndk` crate. +const FLAG_KEY_FRAME: u32 = 1; +const FLAG_CODEC_CONFIG: u32 = 2; +const FLAG_END_OF_STREAM: u32 = 4; + +/// How long to wait for a free input buffer before dropping the frame. A codec +/// that has not freed one within about a third of a frame interval at 30 fps is +/// backed up behind its own output, and a live encoder that blocks instead +/// builds latency it can never give back. +const INPUT_TIMEOUT: Duration = Duration::from_millis(10); + +/// Poll for finished output without waiting. [`MediaCodec::encode`] drains on +/// both sides of every submission, so output that isn't ready yet is collected +/// on the next frame; blocking here would spend most of a frame interval waiting +/// for a picture the codec hasn't started. +const OUTPUT_TIMEOUT: Duration = Duration::ZERO; + +/// How long each round of a tail drain waits for the next access unit. +const DRAIN_TIMEOUT: Duration = Duration::from_millis(50); + +/// How many rounds a tail drain runs before giving up on an end-of-stream that +/// never arrives, so a wedged codec fails the group rather than the process. +const DRAIN_ROUNDS: u32 = 20; + +pub(crate) struct MediaCodec { + codec: media_codec::MediaCodec, + /// Which codec's NAL headers to read, for spotting a keyframe that already + /// carries its own parameter sets. + kind: Codec, + width: usize, + height: usize, + framerate: u32, + /// The parameter sets (SPS/PPS, plus VPS for H.265) as the codec-config + /// buffer delivered them: Annex-B, ahead of the first picture. Kept because + /// every keyframe has to carry them for a subscriber joining there. + parameter_sets: Option, + /// Frames handed to the codec that haven't come back out, oldest first: the + /// sample time we gave it paired with the frame's real timestamp. MediaCodec + /// buffers, so its output is matched against this rather than stamped with + /// whatever is being fed at the time. + pending: VecDeque<(i64, Timestamp)>, + /// The last timestamp paired with an output, reused if the codec ever hands + /// back more access units than it was fed frames. Repeating a time is far + /// kinder to a consumer than the jump to zero the alternative would produce. + last_timestamp: Option, + frame_index: i64, + /// Set while an IDR is owed: asked for by the caller, or by a flush that + /// dropped the reference frames the next picture would predict from. + keyframe_pending: bool, + /// True once end of input has been signalled and before the flush that takes + /// it back, since a codec in that state refuses input. + ended: bool, +} + +// SAFETY: `AMediaCodec` is an owned handle with no thread affinity; the NDK only +// requires that calls on one codec are serialized, which they are because every +// method here takes `&mut self` and the encode thread owns the backend outright. +// `Send` is what lets the boxed trait object satisfy `Backend: Send`. +unsafe impl Send for MediaCodec {} + +impl MediaCodec { + pub(crate) fn open(config: &Config) -> Result, Error> { + // backend::open only routes codecs this backend advertises, so the match is + // exhaustive; a new Codec variant won't compile here until it's handled. + let mime = match config.codec { + Codec::H264 => MIME_H264, + Codec::H265 => MIME_H265, + }; + + let codec = media_codec::MediaCodec::from_encoder_type(mime) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("no MediaCodec encoder for {mime}")))?; + let format = encoder_format(config, mime); + codec + .configure(&format, None, MediaCodecDirection::Encoder) + .map_err(|e| codec_err("configure", e))?; + codec.start().map_err(|e| codec_err("start", e))?; + + tracing::info!( + encoder = NAME, + codec = ?config.codec, + width = config.width, + height = config.height, + "opened video encoder" + ); + Ok(Box::new(Self { + codec, + kind: config.codec, + width: config.width as usize, + height: config.height as usize, + framerate: config.framerate.max(1), + parameter_sets: None, + pending: VecDeque::new(), + last_timestamp: None, + frame_index: 0, + keyframe_pending: false, + ended: false, + })) + } + + /// The codec-side clock for the next frame, in microseconds. + /// + /// A monotonic index over the configured framerate rather than the frame's + /// own timestamp: the codec only needs a clock that increases, the real + /// timestamp rides alongside in `pending`, and this is the value the codec + /// echoes back on the matching output. + fn sample_time(&self) -> i64 { + self.frame_index * 1_000_000 / self.framerate as i64 + } + + /// Ask the codec to make the next picture an IDR. + fn request_keyframe(&self) -> Result<(), Error> { + let mut params = MediaFormat::new(); + params.set_i32(KEY_REQUEST_SYNC_FRAME, 0); + self.codec + .set_parameters(params) + .map_err(|e| codec_err("request a sync frame", e)) + } + + /// Write one frame into a free input buffer, dropping it if the codec has + /// none. + fn submit(&mut self, frame: &Frame, keyframe: bool) -> Result<(), Error> { + if keyframe { + // Keep the request pending until a frame is actually accepted. A full + // codec queue drops this input, but the next submitted picture still has + // to open the group with an IDR. + self.keyframe_pending = true; + self.request_keyframe()?; + } + + let i420 = frame.surface.to_i420()?; + let size = I420::len(self.width as u32, self.height as u32); + let sample_time = self.sample_time(); + + let submitted = match self + .codec + .dequeue_input_buffer(INPUT_TIMEOUT) + .map_err(|e| codec_err("dequeue an input buffer", e))? + { + DequeuedInputBufferResult::Buffer(mut buffer) => { + fill_nv12(buffer.buffer_mut(), &i420, self.width, self.height)?; + self.codec + .queue_input_buffer(buffer, 0, size, sample_time as u64, 0) + .map_err(|e| codec_err("queue an input buffer", e))?; + true + } + // Every input buffer is still with the codec, which means it is behind + // on encoding rather than on anything we can hand it. Drop the frame: + // the drain around this call is what frees a buffer for the next one. + DequeuedInputBufferResult::TryAgainLater => false, + }; + + if !submitted { + tracing::debug!(encoder = NAME, "no input buffer available, dropping a frame"); + return Ok(()); + } + + self.pending.push_back((sample_time, frame.timestamp)); + self.frame_index += 1; + self.keyframe_pending = false; + Ok(()) + } + + /// Collect every access unit the codec has ready, waiting `timeout` on each + /// round. Returns whether it reported end of stream, which is the only thing + /// that says the tail is out rather than still coming. + fn drain(&mut self, timeout: Duration, out: &mut Vec) -> Result { + loop { + let taken = match self + .codec + .dequeue_output_buffer(timeout) + .map_err(|e| codec_err("dequeue an output buffer", e))? + { + DequeuedOutputBufferInfoResult::Buffer(buffer) => { + let info = *buffer.info(); + let unit = access_unit(&buffer, &info); + // Released before anything else can fail: the slot belongs to the + // codec, and an early return still holding it would starve encoding. + self.codec + .release_output_buffer(buffer, false) + .map_err(|e| codec_err("release an output buffer", e))?; + Some((info, unit)) + } + DequeuedOutputBufferInfoResult::TryAgainLater => return Ok(false), + // A codec revising its output format or rotating its buffer set says + // nothing about the bitstream: the parameter sets arrive as a + // codec-config buffer either way. + DequeuedOutputBufferInfoResult::OutputFormatChanged + | DequeuedOutputBufferInfoResult::OutputBuffersChanged => None, + }; + let Some((info, unit)) = taken else { continue }; + + let flags = info.flags(); + if flags & FLAG_CODEC_CONFIG != 0 { + self.parameter_sets = Some(unit); + } else if !unit.is_empty() { + let timestamp = + take_timestamp(&mut self.pending, &mut self.last_timestamp, info.presentation_time_us()); + let payload = if flags & FLAG_KEY_FRAME != 0 { + with_parameter_sets(self.parameter_sets.as_ref(), self.kind, unit) + } else { + unit + }; + out.push(Encoded::new(payload, timestamp)); + } + + if flags & FLAG_END_OF_STREAM != 0 { + return Ok(true); + } + } + } + + /// Signal end of input and wait the codec's tail out, returning everything it + /// was still holding. + /// + /// The wait is the point. An encoder that buffers won't part with the frames + /// it holds until it knows no more input is coming, so sweeping whatever + /// happens to be ready already truncates the stream: that is one access unit + /// lost per group, and on a live track the frame does not vanish but + /// reappears ahead of the next group's keyframe. + fn drain_tail(&mut self) -> Result, Error> { + let mut out = Vec::new(); + // Every frame fed has already come back out, so there is no tail to wait + // for and no reason to cycle the codec through end of stream. This is the + // common case: the format asks for a one-frame pipeline. + if self.ended || self.pending.is_empty() { + return Ok(out); + } + + self.signal_end_of_input(&mut out)?; + self.ended = true; + + for _ in 0..DRAIN_ROUNDS { + if self.drain(DRAIN_TIMEOUT, &mut out)? { + // Anything still outstanding was discarded by the codec rather than + // encoded, so it will never be paired with an output. + self.pending.clear(); + return Ok(out); + } + } + + Err(Error::Codec(anyhow::anyhow!( + "MediaCodec did not reach end of stream within {:?}", + DRAIN_TIMEOUT * DRAIN_ROUNDS + ))) + } + + /// Queue the empty buffer that marks the end of input. + /// + /// ByteBuffer mode has no `AMediaCodec_signalEndOfInputStream`: that one is + /// for a codec fed through a surface. + fn signal_end_of_input(&mut self, out: &mut Vec) -> Result<(), Error> { + let sample_time = self.sample_time(); + for _ in 0..DRAIN_ROUNDS { + let queued = match self + .codec + .dequeue_input_buffer(INPUT_TIMEOUT) + .map_err(|e| codec_err("dequeue an input buffer", e))? + { + DequeuedInputBufferResult::Buffer(buffer) => { + self.codec + .queue_input_buffer(buffer, 0, 0, sample_time as u64, FLAG_END_OF_STREAM) + .map_err(|e| codec_err("queue end of stream", e))?; + true + } + DequeuedInputBufferResult::TryAgainLater => false, + }; + if queued { + return Ok(()); + } + // Every input buffer is still with the codec; collecting its output is + // what frees one. + self.drain(OUTPUT_TIMEOUT, out)?; + } + + Err(Error::Codec(anyhow::anyhow!( + "MediaCodec never freed an input buffer for the end of stream" + ))) + } +} + +impl Backend for MediaCodec { + fn encode(&mut self, frame: &Frame, keyframe: bool) -> Result, Error> { + let mut out = Vec::new(); + // Collect whatever the codec finished while the caller was elsewhere, so + // the input buffer asked for below isn't stuck behind an output nobody + // picked up. + self.drain(OUTPUT_TIMEOUT, &mut out)?; + self.submit(frame, keyframe || self.keyframe_pending)?; + self.drain(OUTPUT_TIMEOUT, &mut out)?; + Ok(out) + } + + fn flush(&mut self) -> Result, Error> { + let out = self.drain_tail()?; + if self.ended { + // End of input leaves the codec refusing input, and only a flush takes + // that back. Nothing more is needed in synchronous mode: the NDK + // documents the `AMediaCodec_start` after a flush as an asynchronous-mode + // requirement. + self.codec.flush().map_err(|e| codec_err("flush", e))?; + self.ended = false; + // A flush drops the reference frames, so the next picture has to be an + // IDR. Asked for rather than assumed: not every encoder inserts one. + self.keyframe_pending = true; + } + Ok(out) + } + + fn finish(&mut self) -> Result, Error> { + self.drain_tail() + } + + fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> { + // Settable on a live codec and applied without an IDR, which is what the + // rate control loop wants. The NDK does warn that a parameter change may + // silently fail to apply, and nothing in the API distinguishes that from + // one that took, so a device that ignores this reports success. + let mut params = MediaFormat::new(); + params.set_i32(KEY_VIDEO_BITRATE, clamp_i32(bitrate)); + self.codec + .set_parameters(params) + .map_err(|e| codec_err("set the bitrate", e)) + } + + fn name(&self) -> &str { + NAME + } +} + +/// The `AMediaFormat` describing what we want encoded and how. +fn encoder_format(config: &Config, mime: &str) -> MediaFormat { + let mut format = MediaFormat::new(); + format.set_str(KEY_MIME, mime); + format.set_i32(KEY_WIDTH, config.width as i32); + format.set_i32(KEY_HEIGHT, config.height as i32); + format.set_i32(KEY_COLOR_FORMAT, COLOR_FORMAT_NV12); + format.set_i32(KEY_BIT_RATE, clamp_i32(config.resolved_bitrate())); + format.set_i32(KEY_BITRATE_MODE, BITRATE_MODE_CBR); + format.set_i32(KEY_FRAME_RATE, config.framerate as i32); + format.set_i32(KEY_PRIORITY, PRIORITY_REALTIME); + // MediaCodec takes the keyframe interval in seconds rather than frames, and + // reads it as a float, so a sub-second GOP survives instead of rounding to + // zero (which would mean an IDR on every frame). + format.set_f32(KEY_I_FRAME_INTERVAL, config.gop as f32 / config.framerate.max(1) as f32); + // Ask for the shortest pipeline the device offers: output a frame after + // input, and no B-frames, whose reorder delay a live track has no use for. + // Both are hints an older device drops, which is why `flush` still drains the + // tail properly rather than assuming there is never one. + format.set_i32(KEY_LATENCY, 1); + format.set_i32(KEY_LOW_LATENCY, 1); + format.set_i32(KEY_MAX_B_FRAMES, 0); + + // State the color space so the encoder writes it into the bitstream's VUI and + // a decoder doesn't fall back to guessing it from the frame height. BT.601 + // goes out as the NTSC standard (SMPTE 170M primaries and matrix, both code + // point 6) with the BT.709 transfer curve, which is what every other backend + // emits: the two curves are defined identically and 709 is the only one all + // of them can name. + let color = config.resolved_color(); + let standard = match color { + Color::Bt601Limited | Color::Bt601Full => COLOR_STANDARD_BT601_NTSC, + Color::Bt709Limited | Color::Bt709Full => COLOR_STANDARD_BT709, + }; + format.set_i32(KEY_COLOR_STANDARD, standard); + format.set_i32(KEY_COLOR_TRANSFER, COLOR_TRANSFER_SDR_VIDEO); + format.set_i32( + KEY_COLOR_RANGE, + if color.limited() { + COLOR_RANGE_LIMITED + } else { + COLOR_RANGE_FULL + }, + ); + format +} + +/// Write `i420` into a MediaCodec input buffer as tightly packed NV12. +/// +/// # Errors +/// +/// Fails when the codec's buffer is smaller than one picture, which is the +/// device disagreeing with the geometry it was configured with rather than +/// anything a caller can fix. +fn fill_nv12(buffer: &mut [MaybeUninit], i420: &I420, width: usize, height: usize) -> Result<(), Error> { + let luma_len = width * height; + let needed = luma_len + luma_len / 2; + if buffer.len() < needed { + return Err(Error::Codec(anyhow::anyhow!( + "MediaCodec input buffer is {} bytes, needs {needed} for {width}x{height} NV12", + buffer.len() + ))); + } + + let (luma, chroma) = buffer[..needed].split_at_mut(luma_len); + // Y is already the plane MediaCodec wants, byte for byte. + // SAFETY: `luma` is exactly `luma_len` bytes by the split above, and + // `i420.y()` is `i420.width() * i420.height()`, which equals `luma_len` + // because `Encoder::encode` rejects any frame whose size differs from the + // config this codec was opened with. They are distinct allocations (a codec + // input buffer and our own frame), and `MaybeUninit` shares the layout of + // `u8`, so copying initialized bytes over it is what initializes it. + unsafe { + std::ptr::copy_nonoverlapping(i420.y().as_ptr(), luma.as_mut_ptr().cast::(), luma_len); + } + + // NV12 interleaves the two chroma planes into one. + for ((pair, u), v) in chroma.chunks_exact_mut(2).zip(i420.u()).zip(i420.v()) { + pair[0].write(*u); + pair[1].write(*v); + } + Ok(()) +} + +/// The valid bytes of an output buffer. +/// +/// `BufferInfo` delimits the access unit inside a buffer that is usually larger, +/// and a codec reporting a window outside its own buffer is clamped rather than +/// panicking a slice. An empty one is not read at all: it is a marker rather +/// than a picture (the end-of-stream buffer is one), and asking the NDK for the +/// bytes of a buffer it has none for panics rather than returning an error. +fn access_unit(buffer: &OutputBuffer<'_>, info: &BufferInfo) -> Bytes { + if info.size() <= 0 { + return Bytes::new(); + } + + let bytes = buffer.buffer(); + let start = (info.offset().max(0) as usize).min(bytes.len()); + let end = start.saturating_add(info.size().max(0) as usize).min(bytes.len()); + Bytes::copy_from_slice(&bytes[start..end]) +} + +/// The timestamp of the frame an output belongs to, found by the sample time the +/// codec echoed back and removed from `pending`. +/// +/// Falls back to the oldest frame still outstanding when the sample time matches +/// nothing: the encoder emits access units in the order it was fed (low latency, +/// no reordering), so oldest-first is the right pairing, and dropping the packet +/// or stamping it with the wrong frame would both be worse. +fn take_timestamp( + pending: &mut VecDeque<(i64, Timestamp)>, + last: &mut Option, + sample_time: i64, +) -> Timestamp { + // `remove` rather than `pop_front`: without reordering these coincide, but a + // codec that did reorder would otherwise pair every later packet wrongly. + let matched = match pending.iter().position(|(fed, _)| *fed == sample_time) { + Some(index) => pending.remove(index), + None => { + let oldest = pending.pop_front(); + if oldest.is_some() { + tracing::debug!(sample_time, "encoder output did not match a fed sample time"); + } + oldest + } + }; + + match matched { + Some((_, timestamp)) => { + *last = Some(timestamp); + timestamp + } + // Nothing outstanding: the codec produced more access units than it was fed + // frames, which shouldn't happen. Repeat the last frame's time so the + // stream keeps flowing in order rather than jumping backwards. + None => { + tracing::warn!("encoder produced output with no frame outstanding"); + last.unwrap_or(Timestamp::ZERO) + } + } +} + +/// A keyframe with the parameter sets in front of it, so a subscriber joining +/// there can decode without having seen the codec-config buffer. +/// +/// Left alone when the codec already prepended them itself, which some devices +/// do whether or not they were asked. +fn with_parameter_sets(sets: Option<&Bytes>, codec: Codec, unit: Bytes) -> Bytes { + let Some(sets) = sets else { return unit }; + if opens_with_parameter_set(&unit, codec) { + return unit; + } + let mut out = BytesMut::with_capacity(sets.len() + unit.len()); + out.put_slice(sets); + out.put_slice(&unit); + out.freeze() +} + +/// Whether an Annex-B access unit opens with a parameter set: an H.264 SPS +/// (type 7), or an H.265 VPS or SPS (types 32 and 33). +fn opens_with_parameter_set(unit: &[u8], codec: Codec) -> bool { + let header = match unit { + [0, 0, 0, 1, header, ..] | [0, 0, 1, header, ..] => *header, + _ => return false, + }; + match codec { + Codec::H265 => matches!((header >> 1) & 0x3f, 32 | 33), + _ => header & 0x1f == 7, + } +} + +/// Wrap an NDK media error, naming the call that produced it. +fn codec_err(what: &str, error: MediaError) -> Error { + Error::Codec(anyhow::anyhow!("failed to {what} on the MediaCodec encoder: {error}")) +} + +fn clamp_i32(value: u64) -> i32 { + value.min(i32::MAX as u64) as i32 +} + +#[cfg(test)] +mod tests { + use super::*; + + const SPS: &[u8] = &[0, 0, 0, 1, 0x67, 0x42, 0xc0, 0x1e]; + const IDR: &[u8] = &[0, 0, 0, 1, 0x65, 0x88, 0x84]; + + fn timestamp(micros: u64) -> Timestamp { + Timestamp::from_micros(micros).unwrap() + } + + #[test] + fn a_keyframe_without_parameter_sets_gets_them() { + let sets = Bytes::from_static(SPS); + let unit = Bytes::from_static(IDR); + let out = with_parameter_sets(Some(&sets), Codec::H264, unit); + assert_eq!(&out[..SPS.len()], SPS); + assert_eq!(&out[SPS.len()..], IDR); + } + + #[test] + fn a_keyframe_that_already_carries_them_is_left_alone() { + let sets = Bytes::from_static(SPS); + let mut unit = Vec::from(SPS); + unit.extend_from_slice(IDR); + let unit = Bytes::from(unit); + + let out = with_parameter_sets(Some(&sets), Codec::H264, unit.clone()); + assert_eq!(out, unit); + } + + #[test] + fn h265_parameter_sets_are_recognized_by_their_own_header() { + // A two-byte H.265 header: VPS is type 32, an IDR_W_RADL slice type 19. + let vps = [0, 0, 0, 1, 0x40, 0x01]; + let idr = [0, 0, 0, 1, 0x26, 0x01]; + assert!(opens_with_parameter_set(&vps, Codec::H265)); + assert!(!opens_with_parameter_set(&idr, Codec::H265)); + } + + #[test] + fn output_is_paired_with_the_frame_it_was_encoded_from() { + let mut pending = VecDeque::from(vec![(0, timestamp(1_000)), (33_333, timestamp(34_333))]); + let mut last = None; + + // The codec is a frame behind, so the first output carries the first + // frame's sample time while the second frame is already in. + assert_eq!(take_timestamp(&mut pending, &mut last, 0), timestamp(1_000)); + assert_eq!(take_timestamp(&mut pending, &mut last, 33_333), timestamp(34_333)); + assert!(pending.is_empty()); + } + + #[test] + fn an_unmatched_sample_time_falls_back_to_the_oldest_frame() { + let mut pending = VecDeque::from(vec![(0, timestamp(1_000))]); + let mut last = None; + assert_eq!(take_timestamp(&mut pending, &mut last, 12_345), timestamp(1_000)); + + // Nothing outstanding: repeat rather than jump back to zero. + assert_eq!(take_timestamp(&mut pending, &mut last, 12_345), timestamp(1_000)); + } + + #[test] + #[ignore = "needs an Android device with a MediaCodec H.264 encoder"] + fn encodes_a_keyframe_with_parameter_sets_inline() { + let config = Config::new(320, 240, 30); + let mut backend = MediaCodec::open(&config).expect("a MediaCodec encoder"); + + let size = config.size(); + let i420 = I420::new(size.width, size.height, vec![0x80; I420::len(size.width, size.height)]).unwrap(); + let frame = Frame::new(crate::Surface::I420(i420), timestamp(0)); + + let mut encoded = backend.encode(&frame, true).unwrap(); + if encoded.is_empty() { + encoded = backend.flush().unwrap(); + } + + let annexb: Vec = encoded.iter().flat_map(|frame| frame.payload.iter().copied()).collect(); + moq_mux::codec::h264::config(&annexb).expect("a keyframe carrying its own parameter sets"); + } +} diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index ebcc6324215..65074513659 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -10,7 +10,7 @@ //! Output is stamped with the timestamp of the frame it was encoded from, not //! whichever frame happened to be going in. A backend that flushes each frame //! before returning just echoes the input timestamp; one that buffers (Media -//! Foundation) correlates through the codec's own sample clock. +//! Foundation, MediaCodec) correlates through the codec's own sample clock. //! //! [`open`] picks the best backend for a [`Codec`](super::Codec) + //! [`Kind`](super::Kind): only candidates that support the requested codec are @@ -32,6 +32,9 @@ mod videotoolbox; #[cfg(target_os = "windows")] mod mediafoundation; +#[cfg(all(target_os = "android", feature = "mediacodec"))] +mod mediacodec; + #[cfg(all(target_os = "linux", feature = "nvidia"))] mod nvenc; @@ -102,6 +105,12 @@ const HARDWARE: &[Candidate] = &[ codecs: &[Codec::H264, Codec::H265], open: mediafoundation::MediaFoundation::open, }, + #[cfg(all(target_os = "android", feature = "mediacodec"))] + Candidate { + name: mediacodec::NAME, + codecs: &[Codec::H264, Codec::H265], + open: mediacodec::MediaCodec::open, + }, #[cfg(all(target_os = "linux", feature = "nvidia"))] Candidate { name: nvenc::NAME, diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index 550c9918f39..bfc293e8309 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -40,8 +40,8 @@ pub enum Kind { Hardware, /// Software only (openh264 for H.264). Software, - /// A specific backend by name, e.g. `"videotoolbox"`, `"nvenc"`, `"vaapi"`, - /// `"v4l2"`, or `"openh264"`. + /// A specific backend by name, e.g. `"videotoolbox"`, `"mediacodec"`, + /// `"nvenc"`, `"vaapi"`, `"v4l2"`, or `"openh264"`. Named(String), } diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 3f84e94666e..bd49fa34b75 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -16,6 +16,9 @@ //! - `Surface::DmaBuf` is a Linux DRM allocation, produced by PipeWire capture. //! The Vulkan renderer imports supported packed formats directly, while CPU //! consumers map linear allocations only. +//! - `Surface::HardwareBuffer` is an Android `AHardwareBuffer`, produced by the +//! MediaCodec decoder rendering into an `ImageReader`. A GPU consumer imports +//! it as a GL or Vulkan image; `into_i420` reads the planes back instead. //! - `Surface::I420` is CPU-resident planar I420, for the CPU encode path and //! platforms without a zero-copy capture. //! @@ -375,6 +378,10 @@ pub enum Surface { /// Linux DMA-BUF, exported on access and retained until the last clone drops. #[cfg(all(target_os = "linux", feature = "dmabuf"))] DmaBuf(DmaBuf), + /// Zero-copy GPU buffer (Android `AHardwareBuffer`, 4:2:0 8-bit). Produced + /// only by the MediaCodec decoder. + #[cfg(all(target_os = "android", feature = "mediacodec"))] + HardwareBuffer(android::HardwareBuffer), /// CPU-resident planar I420. I420(I420), } @@ -391,6 +398,8 @@ impl Surface { Surface::Cuda(c) => c.width, #[cfg(all(target_os = "linux", feature = "dmabuf"))] Surface::DmaBuf(d) => d.width, + #[cfg(all(target_os = "android", feature = "mediacodec"))] + Surface::HardwareBuffer(b) => b.width, Surface::I420(i) => i.width, } } @@ -406,6 +415,8 @@ impl Surface { Surface::Cuda(c) => c.height, #[cfg(all(target_os = "linux", feature = "dmabuf"))] Surface::DmaBuf(d) => d.height, + #[cfg(all(target_os = "android", feature = "mediacodec"))] + Surface::HardwareBuffer(b) => b.height, Surface::I420(i) => i.height, } } @@ -578,6 +589,8 @@ impl Surface { Surface::Cuda(_) => None, #[cfg(all(target_os = "linux", feature = "dmabuf"))] Surface::DmaBuf(d) => d.color, + #[cfg(all(target_os = "android", feature = "mediacodec"))] + Surface::HardwareBuffer(_) => None, Surface::I420(i) => i.color(), } } @@ -593,6 +606,8 @@ impl Surface { Surface::Cuda(c) => Ok(Cow::Owned(c.download_i420()?)), #[cfg(all(target_os = "linux", feature = "dmabuf"))] Surface::DmaBuf(d) => Ok(Cow::Owned(d.inner.download_i420()?)), + #[cfg(all(target_os = "android", feature = "mediacodec"))] + Surface::HardwareBuffer(b) => Ok(Cow::Owned(b.download_i420()?)), Surface::I420(i) => Ok(Cow::Borrowed(i)), } } @@ -1059,6 +1074,260 @@ mod cache_tests { } } +#[cfg(all(target_os = "android", feature = "mediacodec"))] +pub mod android { + //! Android graphics surfaces: the [`HardwareBuffer`] behind + //! `Surface::HardwareBuffer`, plus the read-back to CPU I420. + + use std::sync::Arc; + + use ndk::hardware_buffer::HardwareBufferRef; + use ndk::media::image_reader::{Image, ImageReader}; + + use super::I420; + use crate::Error; + + // Plane indices of an `AIMAGE_FORMAT_YUV_420_888` image. The format always + // reports three planes in this order, whether the device laid the picture out + // planar (I420) or semi-planar (NV12/NV21); only the pixel strides differ, and + // `HardwareBuffer::plane` reads those rather than assuming a layout. + const Y: i32 = 0; + const U: i32 = 1; + const V: i32 = 2; + + /// The `ImageReader` a decoded picture came out of. + /// + /// A newtype rather than the `ndk` type directly so the thread-safety + /// assertion lives in one place that the decoder and the frames outliving it + /// both share. + pub(crate) struct Reader(ImageReader); + + // No `unsafe impl` here on purpose. Both holders of a `Reader` carry their + // own (`HardwareBuffer` below, and the decoder backend), so auto-trait + // derivation over this type is never consulted, and asserting `Sync` for it + // would claim concurrent acquires are sound when nothing performs them. + + impl Reader { + pub(crate) fn new(reader: ImageReader) -> Self { + Self(reader) + } + } + + impl std::ops::Deref for Reader { + type Target = ImageReader; + + fn deref(&self) -> &ImageReader { + &self.0 + } + } + + /// A decoded picture still in GPU memory, as an Android `AHardwareBuffer`. + /// + /// The MediaCodec decoder renders into an `ImageReader` and hands out the + /// images it acquires, so the picture reaches a GL or Vulkan consumer without + /// a CPU round trip. [`buffer`](Self::buffer) is that handle; everything else + /// goes through [`Surface::into_i420`](crate::Surface::into_i420), which reads + /// the planes back. + /// + /// A frame occupies a slot in the decoder's fixed-size buffer queue for as long + /// as it lives, and the decoder stalls once the queue is empty. Draw and drop + /// rather than holding a backlog. + pub struct HardwareBuffer { + /// The reader owns the queue each image is a slot in, and deleting it + /// invalidates images already acquired from it, so a frame outliving its + /// decoder has to keep the reader alive too. + _reader: Arc, + image: Image, + pub(crate) left: u32, + pub(crate) top: u32, + pub(crate) width: u32, + pub(crate) height: u32, + } + + // SAFETY: an `AImage` is an owned handle to one buffer-queue slot, so it is not + // tied to the thread that acquired it. Every method here is a read + // (`AImage_getPlane*`, `AImage_getHardwareBuffer`) of an image nothing else can + // reach: the decoder acquires it and hands ownership straight out, and it is + // never re-acquired. The reader alongside it carries its own assertion. `Sync` + // is load-bearing the same way it is for the macOS pixel buffer, since + // moq-transcode fans decoded frames out as `Arc`. + unsafe impl Send for HardwareBuffer {} + unsafe impl Sync for HardwareBuffer {} + + impl HardwareBuffer { + pub(crate) fn new(reader: Arc, image: Image, left: u32, top: u32, width: u32, height: u32) -> Self { + Self { + _reader: reader, + image, + left, + top, + width, + height, + } + } + + /// The visible picture's horizontal offset within the hardware buffer. + pub fn left(&self) -> u32 { + self.left + } + + /// The visible picture's vertical offset within the hardware buffer. + pub fn top(&self) -> u32 { + self.top + } + + /// The picture width in pixels. + pub fn width(&self) -> u32 { + self.width + } + + /// The picture height in pixels. + pub fn height(&self) -> u32 { + self.height + } + + /// The `AHardwareBuffer` holding the pixels, with a reference acquired for + /// the caller, to import as a GL image (`EGL_ANDROID_image_native_buffer`) or + /// a Vulkan one (`VK_ANDROID_external_memory_android_hardware_buffer`). + /// + /// The reference keeps the allocation alive, but only this surface keeps the + /// *picture* in it: dropping the surface returns the slot to the decoder, + /// which writes the next picture over the same memory. Sample the visible + /// rectangle reported by [`left`](Self::left), [`top`](Self::top), + /// [`width`](Self::width), and [`height`](Self::height) while you still hold + /// the surface. + /// + /// # Errors + /// + /// Fails when the image has no hardware buffer behind it, which is what a + /// software `ImageReader` allocation reports. + pub fn buffer(&self) -> Result { + let buffer = self + .image + .hardware_buffer() + .map_err(|e| Error::Codec(anyhow::anyhow!("AImage_getHardwareBuffer failed: {e}")))?; + // The getter borrows: it hands back the image's own handle without + // touching the refcount, so the caller needs a reference of its own. + Ok(buffer.acquire()) + } + + /// Read the picture back to packed I420 (the CPU path). + /// + /// A layout change, not a color conversion, so the samples keep whatever + /// space they arrived in. The device chooses the layout and the strides, and + /// both are read off the image rather than assumed, so a planar device and a + /// semi-planar one produce the same bytes here. + pub(crate) fn download_i420(&self) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let (cw, ch) = (w / 2, h / 2); + + let mut data = vec![0u8; I420::len(self.width, self.height)]; + let (luma, chroma) = data.split_at_mut(w * h); + let (u_dst, v_dst) = chroma.split_at_mut(cw * ch); + + self.plane(Y)? + .gather(luma, self.left as usize, self.top as usize, w, h)?; + self.plane(U)? + .gather(u_dst, self.left as usize / 2, self.top as usize / 2, cw, ch)?; + self.plane(V)? + .gather(v_dst, self.left as usize / 2, self.top as usize / 2, cw, ch)?; + + Ok(I420 { + width: self.width, + height: self.height, + data, + color: None, + }) + } + + /// One plane's bytes and the two strides needed to walk them. + fn plane(&self, index: i32) -> Result, Error> { + let fail = |what: &str, e: ndk::media_error::MediaError| { + Error::Codec(anyhow::anyhow!("AImage_getPlane{what} failed for plane {index}: {e}")) + }; + Ok(Plane { + data: self.image.plane_data(index).map_err(|e| fail("Data", e))?, + row: self.image.plane_row_stride(index).map_err(|e| fail("RowStride", e))? as usize, + pixel: self + .image + .plane_pixel_stride(index) + .map_err(|e| fail("PixelStride", e))? as usize, + }) + } + } + + /// A borrowed image plane: its bytes, its row pitch, and the gap between + /// horizontally adjacent samples (2 where the device interleaved U and V). + struct Plane<'a> { + data: &'a [u8], + row: usize, + pixel: usize, + } + + impl Plane<'_> { + /// Copy a `width` x `height` sample window into `dst`, tightly packed. + /// + /// # Errors + /// + /// Fails when the plane is shorter than its own strides say it should be, + /// rather than reading past the mapping. + fn gather(&self, dst: &mut [u8], left: usize, top: usize, width: usize, height: usize) -> Result<(), Error> { + let (Some(rows), Some(cols)) = (height.checked_sub(1), width.checked_sub(1)) else { + return Ok(()); + }; + let needed = top + .checked_add(rows) + .and_then(|row| row.checked_mul(self.row)) + .and_then(|offset| { + left.checked_add(cols) + .and_then(|col| col.checked_mul(self.pixel)) + .and_then(|col| offset.checked_add(col)) + }) + .and_then(|offset| offset.checked_add(1)) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("image plane strides overflow the address space")))?; + if self.data.len() < needed { + return Err(Error::Codec(anyhow::anyhow!( + "image plane is {} bytes, needs {needed} for {width}x{height} at ({left}, {top}), row stride {}, and pixel stride {}", + self.data.len(), + self.row, + self.pixel + ))); + } + + for (row, out) in dst.chunks_exact_mut(width).enumerate().take(height) { + let offset = (top + row) * self.row + left * self.pixel; + let src = &self.data[offset..]; + if self.pixel == 1 { + out.copy_from_slice(&src[..width]); + } else { + for (col, sample) in out.iter_mut().enumerate() { + *sample = src[col * self.pixel]; + } + } + } + Ok(()) + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn plane_gather_applies_the_crop_origin() { + let data: Vec = (0..24).collect(); + let plane = Plane { + data: &data, + row: 6, + pixel: 2, + }; + let mut out = [0; 4]; + plane.gather(&mut out, 1, 1, 2, 2).unwrap(); + assert_eq!(out, [8, 10, 14, 16]); + } + } +} + #[cfg(target_os = "macos")] pub mod macos { //! macOS CoreVideo surfaces: the [`PixelBuffer`] behind diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index bb714bfd835..4af39dfc7f4 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -62,9 +62,9 @@ //! //! The one deliberate exception is [`Surface`], the enum behind every frame. //! Its variants name platform representations (`CVPixelBuffer`, Direct3D11, -//! CUDA) so you can render or re-encode a frame yourself without a CPU round -//! trip, which means a major bump of one of those platform crates is a breaking -//! change here. It is `#[non_exhaustive]` and every variant has a universal +//! CUDA, `AHardwareBuffer`) so you can render or re-encode a frame yourself +//! without a CPU round trip, which means a major bump of one of those platform +//! crates is a breaking change here. It is `#[non_exhaustive]` and every variant has a universal //! fallback in [`Surface::into_i420`], so matching on it stays portable: take the //! fast path you recognize and let the `_` arm handle the rest. @@ -100,6 +100,13 @@ pub use frame::{DmaBuf, DmaBufExport, DmaBufPlane, DrmFormat}; pub use frame::{Frame, I420, Surface}; pub use size::Size; +/// The NDK bindings [`frame::android::HardwareBuffer::buffer`] hands back, +/// re-exported for the same reason as the Apple and Windows ones: name the +/// exact version this crate links rather than guessing at a matching one, since a +/// hardware buffer from a different `ndk` build is a different type. A major bump +/// here is a breaking change for this crate. +#[cfg(all(target_os = "android", feature = "mediacodec"))] +pub use ndk; /// The CoreFoundation bindings owning the handle [`Surface::into_pixel_buffer`] /// returns, re-exported alongside [`objc2_core_video`] for the same reason. #[cfg(target_os = "macos")] From 3c2f236781e974ec82ad4d6fb2efb60f77e81fe0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 20:20:59 -0700 Subject: [PATCH 24/29] fix(moq-video): make moq-video compile on dev again (#3423) Co-authored-by: Claude Opus 5 (1M context) --- rs/moq-video/src/encode/backend/v4l2.rs | 8 ++++---- rs/moq-video/src/frame.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rs/moq-video/src/encode/backend/v4l2.rs b/rs/moq-video/src/encode/backend/v4l2.rs index 13326068d8c..a63b85106b9 100644 --- a/rs/moq-video/src/encode/backend/v4l2.rs +++ b/rs/moq-video/src/encode/backend/v4l2.rs @@ -145,7 +145,7 @@ impl V4l2 { )?; device.set_control(V4L2_CID_MPEG_VIDEO_H264_LEVEL, h264_level(config)?)?; device.set_control(V4L2_CID_MPEG_VIDEO_GOP_SIZE, config.gop as i32)?; - set_bitrate(&device, config.resolved_bitrate())?; + set_bitrate(&device, config.resolved_bitrate().as_bps())?; // Constant rate is what a live uplink wants: the congestion controller // already owns the rate, and a variable-rate encoder would spend it on the // wrong frames. @@ -829,7 +829,7 @@ fn h264_level(config: &Config) -> Result { let size = config.size(); let per_frame = size.width.div_ceil(16) * size.height.div_ceil(16); let per_second = per_frame as u64 * config.framerate as u64; - let kbps = config.resolved_bitrate().div_ceil(1_000); + let kbps = config.resolved_bitrate().as_bps().div_ceil(1_000); LEVELS .iter() .find(|level| { @@ -916,9 +916,9 @@ mod tests { #[test] fn the_level_clears_the_bitrate() { let mut config = config(1280, 720, 30); - config.bitrate = Some(14_000_000); + config.bitrate = Some(moq_net::bandwidth::Rate::from_bps(14_000_000)); assert_eq!(h264_level(&config).unwrap(), 9); // 3.1 - config.bitrate = Some(14_000_001); + config.bitrate = Some(moq_net::bandwidth::Rate::from_bps(14_000_001)); assert_eq!(h264_level(&config).unwrap(), 10); // 3.2 } diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 3f84e94666e..4771f79ff60 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -216,7 +216,7 @@ impl DmaBufExport { /// Split the descriptor from the lease that keeps the producer's buffer alive, /// for a consumer that has to own the two separately. Only the renderer does. - #[cfg_attr(not(feature = "render"), expect(dead_code))] + #[cfg(feature = "render")] pub(crate) fn into_parts(self) -> (OwnedFd, Arc) { (self.fd, self.inner) } From ee4aa8530f13ece606af74213ccfb31a309f472d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 20:29:03 -0700 Subject: [PATCH 25/29] docs(quest): fold the stream scheduler prototype into the noq scheduler quest (#3430) Co-authored-by: Claude Fable 5.1 --- quest/m1/perf/README.md | 1 - quest/m1/perf/send-order-width.md | 2 +- quest/m1/perf/stream-scheduler.md | 76 ------------------------------- quest/m2/quic/scheduler.md | 36 ++++++++++++++- 4 files changed, 35 insertions(+), 80 deletions(-) delete mode 100644 quest/m1/perf/stream-scheduler.md diff --git a/quest/m1/perf/README.md b/quest/m1/perf/README.md index 1c2760157c3..d3454801cb0 100644 --- a/quest/m1/perf/README.md +++ b/quest/m1/perf/README.md @@ -49,7 +49,6 @@ backend we ship today. - [#3202](/quest/m1/perf/3202-moq-uring-use-fixed-file-slots-for-worker-udp-sockets.md) - moq-uring: use fixed-file slots for worker UDP sockets - [#3204](/quest/m1/perf/3204-moq-uring-register-tx-pool-buffers-for-zero-copy-sends.md) - moq-uring: register TX-pool buffers for zero-copy sends - [Send order width](/quest/m1/perf/send-order-width.md) - a wider transport send order lets a group rank itself instead of taking the queue lock -- [Stream scheduler](/quest/m1/perf/stream-scheduler.md) - round-robin between equal-priority subscriptions, prototyped as a quinn patch - [Priority set_track wakes](/quest/m1/perf/priority-set-track-wakes.md) - a track priority change stops waking groups that end up where they started - [#3203](/quest/m1/perf/3203-moq-uring-add-opt-in-napi-busy-polling.md) - moq-uring: add opt-in NAPI busy polling - [#3205](/quest/m1/perf/3205-moq-uring-register-reusable-io-uring-enter-wait-arguments.md) - moq-uring: register reusable io_uring_enter wait arguments diff --git a/quest/m1/perf/send-order-width.md b/quest/m1/perf/send-order-width.md index ba228f4442a..0a330bb6e7c 100644 --- a/quest/m1/perf/send-order-width.md +++ b/quest/m1/perf/send-order-width.md @@ -71,7 +71,7 @@ That leaves `[track: 8][group: rest]`. Fairness between equal-priority subscriptions belongs in the transport scheduler, which is the only layer that can round-robin, and is out of scope -here. See [Stream scheduler](/quest/m1/perf/stream-scheduler.md); this quest +here. See [Hierarchical stream scheduling](/quest/m2/quic/scheduler.md); this quest deliberately gives up that level to get the queue off the hot path first. Backends that cannot carry a 32-bit scalar compress it themselves rather than diff --git a/quest/m1/perf/stream-scheduler.md b/quest/m1/perf/stream-scheduler.md deleted file mode 100644 index 352a98497e9..00000000000 --- a/quest/m1/perf/stream-scheduler.md +++ /dev/null @@ -1,76 +0,0 @@ -# [L] Prototype a three-level stream scheduler as a quinn patch - -## Goal - -MoQ wants three scheduling levels: strict by track priority, round-robin between -subscriptions at equal track priority, then strict newest-group-first within a -subscription. Nothing available offers all three, and a scalar send order cannot -express the middle one at all, since any total order picks a winner and that -winner takes strict precedence. - -Prototype the real shape against quinn's scheduler, which already has two of the -three levels and the counter the third one needs, rather than designing it in -the abstract. - -## Plan - -What exists today, which is more than it looks: - -- quinn is already strict-priority then round-robin. `PendingStream` is ordered - `(priority, recency, id)` in a max-heap, where `recency` is a monotonically - decreasing counter that requeues a stream behind its equal-priority peers once - it writes, and `TransportConfig::send_fairness` defaults to true. That is - levels one and two. -- quiche is the same two levels: `urgency` strict, `incremental` round-robin - within an urgency. -- W3C [`sendGroup`](https://www.w3.org/TR/webtransport/#sendGroup) is levels two - and three, the other pair: equal allocation between send groups, strict - `sendOrder` within one. Send groups carry no priority relative to each other, - so it cannot also carry track priority. - -So both native backends already give the top two levels, `sendGroup` gives the -bottom two, and nothing gives three. MoQ currently sees none of the round-robin -because it hands every stream a distinct priority, which means no two streams are -ever equal and the fairness never engages. We pay for a queue to defeat a -scheduler already doing half the job. - -The missing piece is one field, not a scheduler. quinn's ordering key wants to -become `(priority, bucket_recency, order, id)`: `priority` strict for track, -buckets round-robin against each other reusing the existing recency counter, -`order` strict within a bucket for group sequence. That is `sendGroup` semantics -plus the priority dimension `sendGroup` lacks. - -That field lives in the QUIC stack, not above it, and moq-uring does not change -that. Its quinn path hands stream selection to `poll_transmit` and its quiche -path to `Connection::send`, so what moq-uring owns is the UDP path, not the -ordering key. Either backend means a fork or an upstream patch; there is no -third option to reach for first. - -Take quinn, and carry it as a patch rather than a fork. Its key is already -`(priority, recency, id)` with the recency counter this needs, so the change is -one field in one struct and a `SendStream` setter to reach it, which is also the -smallest thing to keep rebased and the most likely to be acceptable upstream. -Run it under moq-uring's quinn backend, which is where the relay workload is, -and treat a working prototype as the evidence for the upstream PR or a spec -issue. quiche stays out of scope: reproducing the result there is a second -fork's worth of work, and its `urgency` plus `incremental` already give the same -top two levels, so it tells us nothing new about whether the third one helps. - -Measure against the two points reachable without any of this, both of which are -real options rather than strawmen: a scalar send order of `[track][group]`, which -buys strict priority and newest-first while giving up fairness (see [Send order -width](/quest/m1/perf/send-order-width.md)), and a scalar of `track` alone, which -lets quinn's existing fairness through and gives up newest-first instead. - -Acceptance: a congested session carrying two equal-priority tracks of different -group cadence, audio against video, keeps both progressing rather than draining -one, while a higher-priority track still preempts both and newest-first still -sheds backlog within a track. Compare all three configurations on the same shape. - -## Required - -- [Send order width](/quest/m1/perf/send-order-width.md) - the scalar lands first - -## Related - -- [Priority set_track wakes](/quest/m1/perf/priority-set-track-wakes.md) - dead once the queue goes diff --git a/quest/m2/quic/scheduler.md b/quest/m2/quic/scheduler.md index 13013c0bb59..3a424e0797f 100644 --- a/quest/m2/quic/scheduler.md +++ b/quest/m2/quic/scheduler.md @@ -44,8 +44,39 @@ subscription's `ordered` setting, never another subscription's sequence. Remove the session-wide `lite::PriorityQueue` once every enabled backend has an honest implementation or fallback. -If moq#3320 lands first, fold its proposed -`quest/m1/perf/stream-scheduler.md` into this quest and remove that index entry. +### Prototype first, as a quinn patch + +Nothing available offers all three levels, but quinn already has two of them +and the counter the third needs: `PendingStream` is ordered +`(priority, recency, id)` in a max-heap, where `recency` is a monotonically +decreasing counter that requeues a stream behind its equal-priority peers once +it writes, and `TransportConfig::send_fairness` defaults to true. quiche's +`urgency` plus `incremental` is the same pair, and W3C `sendGroup` is the other +pair (fair between groups, strict `sendOrder` within one, no priority between +groups). MoQ sees none of quinn's fairness today because it hands every stream +a distinct priority, so no two streams are ever equal. + +The missing piece is one field: the ordering key becomes +`(priority, bucket_recency, order, id)`, with `priority` strict for the track, +buckets round-robin against each other on the existing recency counter, and +`order` strict within a bucket for group sequence. That field lives in the +QUIC stack, and moq-uring does not change that: its quinn path hands stream +selection to `poll_transmit`, so what moq-uring owns is the UDP path, not the +key. Carry it as a quinn patch rather than a fork (one field in one struct +plus a `SendStream` setter), run it under moq-uring's quinn backend where the +relay workload is, and treat a working prototype as the evidence for the noq +proposal above. quiche stays out of scope: reproducing it there is a second +fork's worth of work and its top two levels already match. + +Measure against the two configurations reachable without any of this, both +real options: a scalar send order of `[track][group]`, which buys strict +priority and newest-first while giving up fairness (see +[Send order width](/quest/m1/perf/send-order-width.md)), and a scalar of +`track` alone, which lets quinn's fairness through and gives up newest-first. +A congested session carrying two equal-priority tracks of different group +cadence, audio against video, must keep both progressing rather than draining +one, while a higher-priority track still preempts both and newest-first still +sheds backlog within a track. Compare all three on the same shape. It describes the same scheduler outcome, not a second implementation. Tests saturate the sender with differently sized audio and video groups and @@ -56,6 +87,7 @@ the same scenarios through raw QUIC and qmux. ## Required +- [Send order width](/quest/m1/perf/send-order-width.md) - the scalar lands first; the prototype is measured against it - [Establish the noq relationship](/quest/m2/quic/parent.md) - the scheduler is proposed to noq first From f9c3926ad5869456e71083be6718fe34fde04096 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 20:33:23 -0700 Subject: [PATCH 26/29] docs(quest): import the post-grooming issues as quests (#3431) Co-authored-by: Claude Fable 5.1 --- quest/README.md | 4 +- ...et-shedding-drops-discontinuity-markers.md | 37 ++++++++++++++++++ ...dio-forward-gap-stamps-resampled-output.md | 32 ++++++++++++++++ ...net-subscribe-error-sends-hardcoded-404.md | 38 +++++++++++++++++++ ...roadcast-is-undefined-at-initialization.md | 30 +++++++++++++++ .../m0/3361-js-watch-fails-on-firefox-esr.md | 27 +++++++++++++ ...watch-audio-only-stream-does-not-resume.md | 32 ++++++++++++++++ quest/m0/README.md | 6 +++ ...udio-cpal-allocates-on-the-audio-thread.md | 34 +++++++++++++++++ quest/m2/README.md | 1 + 10 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 quest/m0/3291-moq-net-zero-budget-shedding-drops-discontinuity-markers.md create mode 100644 quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md create mode 100644 quest/m0/3359-moq-net-subscribe-error-sends-hardcoded-404.md create mode 100644 quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md create mode 100644 quest/m0/3361-js-watch-fails-on-firefox-esr.md create mode 100644 quest/m0/3363-js-watch-audio-only-stream-does-not-resume.md create mode 100644 quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md diff --git a/quest/README.md b/quest/README.md index 17925a4bc3e..9e1b8d96ce2 100644 --- a/quest/README.md +++ b/quest/README.md @@ -11,8 +11,8 @@ Milestones sort by kind, which tracks urgency here: m0 fixes what is broken, m1 is the dev branch line, m2 grows the surface, m3 explores. The 2026-08 grooming turned every surviving GitHub issue into a quest and migrated the upstream-facing questlines from the downstream moq.pro tree; issues opened -since are imported in periodic grooming passes, and an issue already fixed on -`dev` stays open until `dev` merges. New work joins the milestone matching its +since are imported in periodic grooming passes (the last one on 2026-09-05), +and an issue already fixed on `dev` stays open until `dev` merges. New work joins the milestone matching its kind, at its priority rank. ## Quests diff --git a/quest/m0/3291-moq-net-zero-budget-shedding-drops-discontinuity-markers.md b/quest/m0/3291-moq-net-zero-budget-shedding-drops-discontinuity-markers.md new file mode 100644 index 00000000000..7918bf59240 --- /dev/null +++ b/quest/m0/3291-moq-net-zero-budget-shedding-drops-discontinuity-markers.md @@ -0,0 +1,37 @@ +# [S] moq-net: a zero-budget subscriber still receives discontinuity markers + +## Goal + +On `dev`, a live-edge subscriber (`max_age` budget of zero, the default) +still receives an empty group, so the codec reset it marks reaches the +decoder. Today `is_stale` sheds the empty group the moment a newer group +exists, the container consumer sees a plain sequence gap, `discontinuity()` +never bumps, and the decoder is fed the new epoch without a reset. + +## Plan + +A dev-only regression from the interaction of max-age shedding (#3251) with +the idle-capture markers (#3214): an empty group's reach is bounded by its +successor's start, so its age equals the successor's span and a zero budget +sheds it. With undeclared rewinds going away +([Monotonic timeline](/quest/m1/monotonic-timeline.md)), the declared marker +is the only reset signal there is, so losing it is not an option. + +Exempt a finished empty group from `is_stale`: it costs nothing to deliver +and carries the reset. The alternative, having the container layer infer a +discontinuity from any sequence gap, would fire on every ordinary shed and +reset decoders that need no reset. State the exemption in the moq-lite draft's +expiration text if the wording implies every group ages alike. + +Test: a producer emits group 1, an empty group, then group 2; a zero-budget +subscriber observes `[1, 0, 1]` at the transport layer and the container +consumer's `discontinuity()` bumps once. Fix on `dev`. + +## Closes + +- [#3291](https://github.com/moq-dev/moq/issues/3291) - close this issue when the quest finishes + +## Related + +- [Monotonic timeline](/quest/m1/monotonic-timeline.md) - makes the declared marker the only reset signal +- [#3161](/quest/m1/3161-retention-should-reclaim-idle-open-groups-now-that-expiry.md) - the other open-group lifecycle rule in the same area diff --git a/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md b/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md new file mode 100644 index 00000000000..4194bc3f063 --- /dev/null +++ b/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md @@ -0,0 +1,32 @@ +# [S] moq-audio: a forward timestamp gap stamps resampled output correctly + +## Goal + +Resampled audio that spans a forward gap in the source timeline is stamped +where its first sample actually belongs. Today a 10 ms packet at `t = 0` +followed by a packet at `t = 1s` produces a chunk stamped near `990 ms` whose +first samples come from `t = 0`, placing that audio almost a second late and, +since `Frame::activity` is picked by the frame's stamp, labelling active audio +as withheld. + +## Plan + +`decode::Consumer::starts_at` derives the chunk's start by subtracting the +samples the resampler still holds from the new packet's timestamp, which +assumes the held samples immediately precede it. A forward gap breaks that +and does not bump the container discontinuity counter, so nothing resets the +state. Only reachable when `decode::Config::sample_rate` differs from the +codec rate, since there is no resampler otherwise. + +Track the source timestamp of the samples fed to the resampler alongside the +samples, so `starts_at` reads the held samples' real origin instead of +deriving it. A gap larger than the resampler's hold then produces two +correctly stamped chunks with a hole between them, which is the shape the +`dev` gap quest for the decode path (#2981) defines for gaps. + +Tests: the 10 ms then 1 s case stamps the first chunk at `0`; contiguous +packets are unchanged; the activity label follows the corrected stamp. + +## Closes + +- [#3326](https://github.com/moq-dev/moq/issues/3326) - close this issue when the quest finishes diff --git a/quest/m0/3359-moq-net-subscribe-error-sends-hardcoded-404.md b/quest/m0/3359-moq-net-subscribe-error-sends-hardcoded-404.md new file mode 100644 index 00000000000..de151a7ed35 --- /dev/null +++ b/quest/m0/3359-moq-net-subscribe-error-sends-hardcoded-404.md @@ -0,0 +1,38 @@ +# [S] moq-net: SUBSCRIBE_ERROR carries a registered error code + +## Goal + +An IETF subscribe the publisher cannot serve is rejected with the error code +the negotiated draft registers for that reason, never a literal `404`, and +`Error::NotFound`'s wire value agrees with it. The moq-interop-runner's +`subscribe-error` and `subscribe-before-announce` cases pass without the +peer's compatibility branch, which is being removed upstream. + +## Plan + +Both not-found paths in `rs/moq-net/src/ietf/publisher.rs` call +`reject_subscribe(.., 404, ..)`, and `reject_subscribe` takes a bare `u64` +because SUBSCRIBE_ERROR (`0x05`) has no named code type. `TrackStatusCode` +in `ietf/track.rs` is the wrong registry (TRACK_STATUS). Separately +`Error::NotFound` maps to `13` in `rs/moq-net/src/error.rs`, so one relay +signals not-found as `404` on one path and `13` on the other. + +Add a named SUBSCRIBE_ERROR code type mapped to the registered values, and +make it version-aware in the style of the existing `Encode` / +`Decode` impls, since the registry moved between draft-14 and +draft-19. Use it at both call sites and route `Error::NotFound` through the +same mapping so the two paths agree. This is the SUBSCRIBE_ERROR half of the +per-protocol code mapping that +[#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md) +asks for on stream resets; build one table both draw from rather than two. + +Tests: the rejection code per negotiated version for a missing broadcast and +a missing track, and the interop-runner cases against a strict peer. + +## Closes + +- [#3359](https://github.com/moq-dev/moq/issues/3359) - close this issue when the quest finishes + +## Related + +- [#3001](/quest/m1/3001-ietf-stream-resets-send-moq-lite-error-codes-so-routine.md) - the stream-reset half of the same per-protocol code mapping diff --git a/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md b/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md new file mode 100644 index 00000000000..17cf2fe0e9f --- /dev/null +++ b/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md @@ -0,0 +1,30 @@ +# [S] js/watch: MoqWatch.broadcast exists from construction + +## Goal + +`moqWatch.broadcast` is usable the moment the element exists, as its +declared type `broadcast: Broadcast` promises. On 0.5.2 a framework effect +that reads `moqWatch.broadcast.catalog` right after binding the element hits +`broadcast is undefined`; it worked on 0.3.2. + +## Plan + +Either the field is created in the constructor, so it is never undefined, +or the type stops lying. Prefer the first: `connection` and `backend` are +constructed eagerly, and a `Broadcast` whose inputs are signals can exist +before any URL is set, so make `broadcast` the same. If some part of it +genuinely cannot exist before `connectedCallback`, split that part out and +keep the field itself eager. + +The issue's side question is a real gap too: `catalog` is a getter, not a +signal, and `observedAttributes` carries `catalog-format` but not `catalog`, +so a framework cannot react to catalog changes. Expose the catalog as a +`Getter` on the element and document in `doc/lib/js` how to read a custom +catalog section from it. + +Regression: construct the element, read `broadcast.catalog` before connecting +it, and subscribe to the catalog signal across a catalog update. + +## Closes + +- [#3360](https://github.com/moq-dev/moq/issues/3360) - close this issue when the quest finishes diff --git a/quest/m0/3361-js-watch-fails-on-firefox-esr.md b/quest/m0/3361-js-watch-fails-on-firefox-esr.md new file mode 100644 index 00000000000..8bc42e22903 --- /dev/null +++ b/quest/m0/3361-js-watch-fails-on-firefox-esr.md @@ -0,0 +1,27 @@ +# [S] js/watch: moq-watch fails on Firefox ESR + +## Goal + +`@moq/watch` plays on Firefox 140 ESR, as 0.3.2 did. On 0.5.2 the element +dies in an effect with `TypeError: can't access property "Consumer", (void 0) +is undefined` before playing anything. + +## Plan + +`(void 0).Consumer` is a module namespace read before that module finished +evaluating, which is what an import cycle looks like under an evaluator +that handles cycles differently from Chrome, or a bundle whose output order +Firefox ESR's module loader resolves differently. It can also be a WebCodecs +or WebTransport feature the ESR line lacks, surfacing as an undefined +namespace after a failed dynamic import. Reproduce on Firefox ESR with the +Svelte page from the issue, bisect between 0.3.2 and 0.5.2, and fix the +cause: break the cycle or gate the feature with the same detection +`@moq/watch/support` already does for other capabilities, so the element +reports what is missing rather than throwing from an effect. + +Firefox is not in the browser harness. Add it to `test/wasm`'s Playwright +matrix if the harness can drive it, or record why it cannot. + +## Closes + +- [#3361](https://github.com/moq-dev/moq/issues/3361) - close this issue when the quest finishes diff --git a/quest/m0/3363-js-watch-audio-only-stream-does-not-resume.md b/quest/m0/3363-js-watch-audio-only-stream-does-not-resume.md new file mode 100644 index 00000000000..4023e9a7805 --- /dev/null +++ b/quest/m0/3363-js-watch-audio-only-stream-does-not-resume.md @@ -0,0 +1,32 @@ +# [S] js/watch: an audio-only stream resumes every time the publisher unmutes + +## Goal + +A subscriber on an audio-only broadcast keeps playing each time the publisher +toggles `muted` back to `false`, not only the first time. Today the second and +every later unmute is detected, played, and immediately closed, until the +publisher reloads its page, which buys exactly one more resume. + +## Plan + +Reproduce with `just relay`, a `` on a `hang`, and a `` +started `invisible` and `muted`, toggling `muted` a few times. Two candidate +mechanisms, and the fix depends on which one it is: + +- The publisher side retracts and re-creates the audio rendition or its + track on each toggle, and `js/watch` treats the second appearance of the + same rendition name as the end of the one it was playing (a stale + `closed` handler from the first subscription evicting the second, the + shape [#2985](/quest/m1/2985-js-net-path-keyed-publisher-state-goes-stale-when-a.md) + describes on the publisher side). +- The watch audio backend keys its "already started" state on the rendition + and never re-arms after the track ends, so the resubscribe races its own + teardown. + +Find which by logging track lifetimes on both ends, fix it where the state +goes stale rather than by retrying the subscribe, and add a `js/watch` test +that unmutes three times and asserts audio frames keep arriving after each. + +## Closes + +- [#3363](https://github.com/moq-dev/moq/issues/3363) - close this issue when the quest finishes diff --git a/quest/m0/README.md b/quest/m0/README.md index 54cf3a74f66..767e96647de 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -16,8 +16,14 @@ regression test per Root Cause First. ## Quests - [#2405](/quest/m0/2405-js-net-connect-logs-on-every-connection-at-the-wrong.md) - js/net: connect logs print the JWT in the relay URL +- [#3361](/quest/m0/3361-js-watch-fails-on-firefox-esr.md) - js/watch: `` throws from an effect on Firefox ESR, a regression since 0.3.2 +- [#3360](/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md) - js/watch: `MoqWatch.broadcast` is undefined at construction although its type says otherwise +- [#3363](/quest/m0/3363-js-watch-audio-only-stream-does-not-resume.md) - js/watch: an audio-only stream resumes only the first time the publisher unmutes +- [#3359](/quest/m0/3359-moq-net-subscribe-error-sends-hardcoded-404.md) - moq-net: SUBSCRIBE_ERROR sends a literal 404 instead of the draft's registered code - [Adapter namespace map](/quest/m0/rs-adapter-namespace-map.md) - moq-net: a duplicate PUBLISH_NAMESPACE on draft-14/15 strands the first request, and the map never shrinks - [#3080](/quest/m0/3080-fix-watch-audio-ring-truncate-can-race-the-worklet-reader.md) - watch: an audio ring truncate can race the worklet reader for one quantum +- [#3326](/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md) - moq-audio: a forward timestamp gap stamps resampled output almost a second late +- [#3291](/quest/m0/3291-moq-net-zero-budget-shedding-drops-discontinuity-markers.md) - moq-net (dev): zero-budget shedding drops the empty group that marks a codec reset - [#2833](/quest/m0/2833-moq-export-ts-a-rewound-timeline-stalls-the-si-table.md) - moq export ts: a rewound timeline stalls SI tables, PCR, and pacing until the media clock catches up - [#2799](/quest/m0/2799-moq-video-capture-negotiates-twice-so-a-window-resize.md) - moq-transcode: the ladder follows a source resolution change instead of keeping the one it started with - [Group charge](/quest/m0/group-charge.md) - charge real per-group cost so MOQ_CACHE_CAPACITY bounds real memory diff --git a/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md b/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md new file mode 100644 index 00000000000..c7387e1364e --- /dev/null +++ b/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md @@ -0,0 +1,34 @@ +# [S] moq-audio: no allocator call on the audio thread when cpal reports an error + +## Goal + +A cpal stream error raised from the real-time render thread reaches +`moq-audio` without an allocation or a free on that thread. Today cpal's +`From` runs `format!` on the render thread before our +callback sees the error, and the callback then drops the owned `Error`, +freeing on the same thread; either can block on the allocator lock and cause +a dropout, exactly when playback is already degraded. + +## Plan + +`FailureReporter::report` in `rs/moq-audio/src/playback/driver.rs` is already +allocation-free (atomics and `Thread::unpark`). What remains is upstream and +the hand-off: + +- Raise it with cpal: allocation-free error emission on the RT paths (a + `&'static str` message, or deferring the `format!` to a non-RT thread), + which is where the dominant allocation lives. Carry the upstream PR as + part of this quest and pin the release that includes it. +- Pre-check in `moq-audio` what can be checked before `stream.play()`, so + fewer errors are raised from the render path at all. +- Document the residual constraint beside the driver: the callback owns the + `Error`, so the overflow path still drops it on the callback thread, and no + bounded hand-off removes that entirely. + +Test: an allocation-counting harness around the error callback, the same +shape the capture-callback quest used, proving no allocation or free after +construction on the paths this repo controls. + +## Closes + +- [#3247](https://github.com/moq-dev/moq/issues/3247) - close this issue when the quest finishes diff --git a/quest/m2/README.md b/quest/m2/README.md index 1a266389d04..66aa6d3e1f4 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -67,6 +67,7 @@ can act on. Each still carries its own plan and regression test. - [Cluster flags](/quest/m2/cluster-flags.md) - a discovery mechanism carries its own prerequisites, so an incomplete cluster config cannot be expressed - [Plan: revalidation updates](/quest/m2/plan-revalidation-updates.md) - settle which fields an auth re-check may update on a live session and which force a reconnect - [#3137](/quest/m2/3137-moqsrc-bound-the-pending-rendition-subscriptions-a.md) - moqsrc: bound the pending rendition subscriptions a catalog can open +- [#3247](/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md) - moq-audio: cpal allocates and frees on the audio thread when it reports a stream error - [#3115](/quest/m2/3115-moqsink-the-publication-has-no-generation-so-a-flush.md) - moqsink: a flushing restart after EOS opens a new publication generation - [#709](/quest/m2/709-automatic-letsencrypt-support.md) - the relay provisions and renews its own ACME certificate over HTTP-01, persisted on disk - [Room SDK](/quest/m2/room-sdk.md) - a headless room package: a room is a path prefix, no service, no storage From 72cb79c2d86c303b2a3b0b59e884fe784b86ec20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 20:43:51 -0700 Subject: [PATCH 27/29] docs(quest): apply the Codex findings on the issue import (#3432) Co-authored-by: Claude Fable 5.1 --- ...dio-forward-gap-stamps-resampled-output.md | 32 ----------- ...roadcast-is-undefined-at-initialization.md | 53 ++++++++++++------- quest/m0/README.md | 3 +- ...udio-cpal-allocates-on-the-audio-thread.md | 11 ++-- quest/m2/README.md | 1 + quest/m2/cpal-release.md | 23 ++++++++ 6 files changed, 65 insertions(+), 58 deletions(-) delete mode 100644 quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md create mode 100644 quest/m2/cpal-release.md diff --git a/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md b/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md deleted file mode 100644 index 4194bc3f063..00000000000 --- a/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md +++ /dev/null @@ -1,32 +0,0 @@ -# [S] moq-audio: a forward timestamp gap stamps resampled output correctly - -## Goal - -Resampled audio that spans a forward gap in the source timeline is stamped -where its first sample actually belongs. Today a 10 ms packet at `t = 0` -followed by a packet at `t = 1s` produces a chunk stamped near `990 ms` whose -first samples come from `t = 0`, placing that audio almost a second late and, -since `Frame::activity` is picked by the frame's stamp, labelling active audio -as withheld. - -## Plan - -`decode::Consumer::starts_at` derives the chunk's start by subtracting the -samples the resampler still holds from the new packet's timestamp, which -assumes the held samples immediately precede it. A forward gap breaks that -and does not bump the container discontinuity counter, so nothing resets the -state. Only reachable when `decode::Config::sample_rate` differs from the -codec rate, since there is no resampler otherwise. - -Track the source timestamp of the samples fed to the resampler alongside the -samples, so `starts_at` reads the held samples' real origin instead of -deriving it. A gap larger than the resampler's hold then produces two -correctly stamped chunks with a hole between them, which is the shape the -`dev` gap quest for the decode path (#2981) defines for gaps. - -Tests: the 10 ms then 1 s case stamps the first chunk at `0`; contiguous -packets are unchanged; the activity label follows the corrected stamp. - -## Closes - -- [#3326](https://github.com/moq-dev/moq/issues/3326) - close this issue when the quest finishes diff --git a/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md b/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md index 17cf2fe0e9f..8602b010909 100644 --- a/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md +++ b/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md @@ -1,29 +1,42 @@ -# [S] js/watch: MoqWatch.broadcast exists from construction +# [S] js/watch: MoqWatch.broadcast is undefined when a framework binds the element ## Goal -`moqWatch.broadcast` is usable the moment the element exists, as its -declared type `broadcast: Broadcast` promises. On 0.5.2 a framework effect -that reads `moqWatch.broadcast.catalog` right after binding the element hits -`broadcast is undefined`; it worked on 0.3.2. +A framework that binds `` and reads `moqWatch.broadcast` in its +first effect gets the `Broadcast` the type promises, or a documented reason +it cannot yet. On 0.5.2 the read returns `undefined`; it worked on 0.3.2. ## Plan -Either the field is created in the constructor, so it is never undefined, -or the type stops lying. Prefer the first: `connection` and `backend` are -constructed eagerly, and a `Broadcast` whose inputs are signals can exist -before any URL is set, so make `broadcast` the same. If some part of it -genuinely cannot exist before `connectedCallback`, split that part out and -keep the field itself eager. - -The issue's side question is a real gap too: `catalog` is a getter, not a -signal, and `observedAttributes` carries `catalog-format` but not `catalog`, -so a framework cannot react to catalog changes. Expose the catalog as a -`Getter` on the element and document in `doc/lib/js` how to read a custom -catalog section from it. - -Regression: construct the element, read `broadcast.catalog` before connecting -it, and subscribe to the catalog signal across a catalog update. +The obvious fix is already the implementation: `js/watch/src/element.ts` +constructs `this.broadcast = new Broadcast({..})` synchronously in the +constructor, so the field is never undefined on an upgraded element. What the +report shows is therefore a read before upgrade: Svelte's `bind:this` hands +the framework the raw `HTMLElement` as soon as it is in the DOM, and the +constructor only runs once `customElements.define` has registered the tag. If +the definition lands after the framework's first effect (a module-order +difference between 0.3.2 and 0.5.2 would explain the regression), every field +the class sets is absent until the upgrade. + +- Reproduce with the issue's Svelte page and confirm the timing by logging + `customElements.whenDefined("moq-watch")` against the effect. +- Fix whichever side owns it: if the entrypoint defers registration, register + synchronously on import as 0.3.2 did; if the framework simply reads early, + document `await customElements.whenDefined("moq-watch")` in `doc/lib/js` + and make the element's own `connectedCallback` tolerate late upgrade. +- Regression on the ordering itself, not on the upgrade: the browser + upgrades an existing node when the definition is registered, so a test + that waits on `whenDefined` before reading passes either way. Assert + instead that importing the entrypoint has defined `moq-watch` before the + import resolves (`customElements.get("moq-watch")` is set synchronously + after `import "@moq/watch/element"`), so a framework that mounts after the + import can never observe a pre-upgrade node. + +The issue's side question, reacting to catalog changes from a framework, +needs no new API: `broadcast.out.catalog` is already a read-only `Getter`, +which `demo/web` reacts to with `effect.get(watch.broadcast.out.catalog)`. +Document that in `doc/lib/js` as part of this quest, with a custom section +read through it. ## Closes diff --git a/quest/m0/README.md b/quest/m0/README.md index 767e96647de..337b6ecf732 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -17,12 +17,11 @@ regression test per Root Cause First. - [#2405](/quest/m0/2405-js-net-connect-logs-on-every-connection-at-the-wrong.md) - js/net: connect logs print the JWT in the relay URL - [#3361](/quest/m0/3361-js-watch-fails-on-firefox-esr.md) - js/watch: `` throws from an effect on Firefox ESR, a regression since 0.3.2 -- [#3360](/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md) - js/watch: `MoqWatch.broadcast` is undefined at construction although its type says otherwise +- [#3360](/quest/m0/3360-js-watch-broadcast-is-undefined-at-initialization.md) - js/watch: a framework binding the element reads `broadcast` before the custom element is upgraded - [#3363](/quest/m0/3363-js-watch-audio-only-stream-does-not-resume.md) - js/watch: an audio-only stream resumes only the first time the publisher unmutes - [#3359](/quest/m0/3359-moq-net-subscribe-error-sends-hardcoded-404.md) - moq-net: SUBSCRIBE_ERROR sends a literal 404 instead of the draft's registered code - [Adapter namespace map](/quest/m0/rs-adapter-namespace-map.md) - moq-net: a duplicate PUBLISH_NAMESPACE on draft-14/15 strands the first request, and the map never shrinks - [#3080](/quest/m0/3080-fix-watch-audio-ring-truncate-can-race-the-worklet-reader.md) - watch: an audio ring truncate can race the worklet reader for one quantum -- [#3326](/quest/m0/3326-moq-audio-forward-gap-stamps-resampled-output.md) - moq-audio: a forward timestamp gap stamps resampled output almost a second late - [#3291](/quest/m0/3291-moq-net-zero-budget-shedding-drops-discontinuity-markers.md) - moq-net (dev): zero-budget shedding drops the empty group that marks a codec reset - [#2833](/quest/m0/2833-moq-export-ts-a-rewound-timeline-stalls-the-si-table.md) - moq export ts: a rewound timeline stalls SI tables, PCR, and pacing until the media clock catches up - [#2799](/quest/m0/2799-moq-video-capture-negotiates-twice-so-a-window-resize.md) - moq-transcode: the ladder follows a source resolution change instead of keeping the one it started with diff --git a/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md b/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md index c7387e1364e..f9342e05fad 100644 --- a/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md +++ b/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md @@ -15,10 +15,9 @@ a dropout, exactly when playback is already degraded. allocation-free (atomics and `Thread::unpark`). What remains is upstream and the hand-off: -- Raise it with cpal: allocation-free error emission on the RT paths (a - `&'static str` message, or deferring the `format!` to a non-RT thread), - which is where the dominant allocation lives. Carry the upstream PR as - part of this quest and pin the release that includes it. +- The dominant allocation is upstream; the + [cpal release](/quest/m2/cpal-release.md) gate quest carries that proposal + and the pin bump, and this quest waits on it. - Pre-check in `moq-audio` what can be checked before `stream.play()`, so fewer errors are raised from the render path at all. - Document the residual constraint beside the driver: the callback owns the @@ -29,6 +28,10 @@ Test: an allocation-counting harness around the error callback, the same shape the capture-callback quest used, proving no allocation or free after construction on the paths this repo controls. +## Required + +- [cpal release](/quest/m2/cpal-release.md) - the upstream fix has to ship before the audio thread can be allocation-free + ## Closes - [#3247](https://github.com/moq-dev/moq/issues/3247) - close this issue when the quest finishes diff --git a/quest/m2/README.md b/quest/m2/README.md index 66aa6d3e1f4..5d66c60ef77 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -67,6 +67,7 @@ can act on. Each still carries its own plan and regression test. - [Cluster flags](/quest/m2/cluster-flags.md) - a discovery mechanism carries its own prerequisites, so an incomplete cluster config cannot be expressed - [Plan: revalidation updates](/quest/m2/plan-revalidation-updates.md) - settle which fields an auth re-check may update on a live session and which force a reconnect - [#3137](/quest/m2/3137-moqsrc-bound-the-pending-rendition-subscriptions-a.md) - moqsrc: bound the pending rendition subscriptions a catalog can open +- [cpal release](/quest/m2/cpal-release.md) - a cpal release that emits errors from its real-time paths without allocating - [#3247](/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md) - moq-audio: cpal allocates and frees on the audio thread when it reports a stream error - [#3115](/quest/m2/3115-moqsink-the-publication-has-no-generation-so-a-flush.md) - moqsink: a flushing restart after EOS opens a new publication generation - [#709](/quest/m2/709-automatic-letsencrypt-support.md) - the relay provisions and renews its own ACME certificate over HTTP-01, persisted on disk diff --git a/quest/m2/cpal-release.md b/quest/m2/cpal-release.md new file mode 100644 index 00000000000..d883d62c011 --- /dev/null +++ b/quest/m2/cpal-release.md @@ -0,0 +1,23 @@ +# [XS] cpal release with allocation-free error emission + +## Goal + +A published cpal release emits stream errors from its real-time paths without +allocating, so `moq-audio` can pin it and stop paying an allocator call on +the audio thread when a device fails. + +## Plan + +Propose the change upstream: a `&'static str` message on the RT emission +paths, or deferring the `format!` in `From` (and the other +hosts' equivalents) to a non-RT thread. Record the maintainers' answer here. +When a release carries it, remove the bullet below, bump the pin, and +complete this quest; that unblocks the dependent one. + +## Required + +- A cpal release that emits errors from its real-time paths without allocating + +## Related + +- [#3247](/quest/m2/3247-moq-audio-cpal-allocates-on-the-audio-thread.md) - the moq-audio work that waits on this release From 339810a695bab19a48fd49de8d5822e5eaef6db7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 20:51:40 -0700 Subject: [PATCH 28/29] fix(moq-mux): align TS tests with PCR buffering --- rs/moq-mux/src/container/ts/export.rs | 4 +- rs/moq-mux/src/container/ts/export_test.rs | 99 +++++++++++++--------- 2 files changed, 61 insertions(+), 42 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 30ce0c0f49e..ca8937df90f 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -230,7 +230,7 @@ struct SiTrack { /// so holding a revision to the interval delivers the time seconds late and can /// re-assert an already-sent value (#2934). dirty: bool, - /// Media timestamp of the last emission ([`Export::write_frame`]). + /// Media timestamp of the last emission ([`Export::mux`]). last_emit: Option, /// The same budget the media sources use. SI carries table snapshots at a low /// rate, so the real-time default would take only the newest group and drop @@ -470,7 +470,7 @@ impl Export { } // 1b. Drive the SI subscriptions, folding arrived snapshots into each - // entry's active set (emission happens on cadence in `write_frame`). + // entry's active set (emission happens on cadence in `mux`). // // Deliberately not part of the first-frame gate below: nothing in SI is // something a TS stream cannot begin without (the PAT/PMT are built locally, diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 4a24a04f823..7b5669301d4 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2789,13 +2789,15 @@ async fn late_join_matches_a_running_exporter() { async fn late_join_matches_a_running_exporter_without_video() { let (a, b) = export_twice(false).await; - // The joiner leads with its own PCR and a PAT/PMT-carrying tune-in frame so a receiver - // can start; the running exporter has no reason to repeat those there. From the first - // timestamp after the tune-in frame the two are rendering the same stream. (The joiner's - // leading PCR is stamped at a slot boundary at or before the tune-in frame, so comparing - // strictly after the tune-in excludes both.) - let tune_in = b.iter().find(|f| !is_pcr_frame(f)).expect("a tune-in frame").timestamp; - let from = b.iter().map(|f| f.timestamp).filter(|&t| t > tune_in).min().unwrap(); + // The joiner's first span leads with PAT/PMT, while the running exporter has no + // reason to repeat them there. The span can cover several PCR slices, so compare + // from the next PAT/PMT refresh, which both exporters anchor to the media grid. + let from = b + .iter() + .filter(|frame| count_pid(std::slice::from_ref(*frame), 0) > 0) + .nth(1) + .expect("a periodic PAT/PMT refresh") + .timestamp; assert_only_continuity_differs(&a, &b, from); } @@ -2907,12 +2909,7 @@ async fn repointed_si_entry_resubscribes() { let mut before = BytesMut::new(); write_key(&mut producer, 0); write_key(&mut producer, 1); - for _ in 0..2 { - let frame = tokio::time::timeout(Duration::from_secs(1), exporter.next()) - .await - .expect("a frame before the switch") - .unwrap() - .unwrap(); + for frame in drain_frames(&mut exporter).await { before.extend_from_slice(&frame.payload); } @@ -3160,15 +3157,19 @@ async fn si_revision_does_not_wait_for_the_interval() { let mut rig = si_cadence_rig(0x0014, 0x70, Duration::from_secs(30)).await; rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(tick(1))).unwrap(); - assert_eq!(rig.media(0, 0x0014).await, 1, "a fresh exporter leads with the table"); - assert_eq!(rig.media(1_000, 0x0014).await, 0, "unchanged: held by the 30s floor"); + assert_eq!(rig.media(0, 0x0014).await, 0, "the first span stays buffered"); + assert_eq!( + rig.media(1_000, 0x0014).await, + 1, + "the next span closes the lead emission" + ); assert_eq!(rig.media(2_000, 0x0014).await, 0, "unchanged: still held"); // The clock ticks. Nothing about the 30 s interval has elapsed, but the value // changed: it must go out with the very next frame. rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(tick(2))).unwrap(); - assert_eq!(rig.media(3_000, 0x0014).await, 1, "the revision rides the next frame"); - assert_eq!(rig.media(4_000, 0x0014).await, 0, "emitted once, then floored again"); + assert_eq!(rig.media(3_000, 0x0014).await, 0, "the revision enters the open span"); + assert_eq!(rig.media(4_000, 0x0014).await, 1, "the next span closes the revision"); } /// #2934, the repeat half: an *unchanged* snapshot repeats only once its interval @@ -3182,29 +3183,34 @@ async fn si_repeats_are_floored_from_the_last_emission() { let mut rig = si_cadence_rig(0x0011, 0x42, Duration::from_secs(2)).await; rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(sdt_v1)).unwrap(); - assert_eq!(rig.media(0, 0x0011).await, 1, "lead emission"); - assert_eq!(rig.media(1_000, 0x0011).await, 0, "within the floor"); - assert_eq!(rig.media(2_000, 0x0011).await, 1, "repeat once 2s elapsed"); + assert_eq!(rig.media(0, 0x0011).await, 0, "the first span stays buffered"); + assert_eq!( + rig.media(1_000, 0x0011).await, + 1, + "the next span closes the lead emission" + ); + assert_eq!(rig.media(2_000, 0x0011).await, 0, "the repeat enters the open span"); // A revision lands mid-interval: it waits only for the 1s revision floor // (measured from the 2s repeat), not for the 2s interval or a grid slot. rig.si_track.write_frame(Timestamp::ZERO, Bytes::from(sdt_v2)).unwrap(); assert_eq!( rig.media(2_500, 0x0011).await, - 0, - "a revision honors the revision floor" + 1, + "the prior repeat closes before the revision" ); assert_eq!( rig.media(3_500, 0x0011).await, - 1, - "the revision rides the floor, not the interval" + 0, + "the revision enters the span once its floor lapses" ); // The next repeat is due 2s after that. The absolute grid would have re-sent // at the 4s boundary, 0.5s after the wire last carried the identical sections. - assert_eq!(rig.media(4_000, 0x0011).await, 0, "the old grid slot must not fire"); + assert_eq!(rig.media(4_000, 0x0011).await, 1, "the next span closes the revision"); assert_eq!(rig.media(5_000, 0x0011).await, 0, "within the floor of the revision"); - assert_eq!(rig.media(5_500, 0x0011).await, 1, "repeat 2s after the revision"); + assert_eq!(rig.media(5_500, 0x0011).await, 0, "the repeat enters the open span"); + assert_eq!(rig.media(6_000, 0x0011).await, 1, "the next span closes the repeat"); } /// A reordered (B-frame) timestamp below the emission anchor earns no credit: @@ -3219,12 +3225,12 @@ async fn si_reordered_frames_earn_no_emission_credit() { rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(0))) .unwrap(); - assert_eq!(rig.media(1_000, 0x0011).await, 1, "lead emission"); + assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first span stays buffered"); rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(1))) .unwrap(); - assert_eq!(rig.media(2_500, 0x0011).await, 1, "revision after the floor"); + assert_eq!(rig.media(2_500, 0x0011).await, 1, "the lead emission closes"); // The next revision arrives on a frame stepping back behind the 2.5s anchor, // like a B-frame emitted in decode order: no elapsed time, no emission. @@ -3234,12 +3240,13 @@ async fn si_reordered_frames_earn_no_emission_credit() { assert_eq!(rig.media(2_400, 0x0011).await, 0, "a reordered frame earns no credit"); // The floor measures from the 2.5s anchor: not due at 3.4s, due at 3.5s. - assert_eq!(rig.media(3_400, 0x0011).await, 0, "still inside the floor"); + assert_eq!(rig.media(3_400, 0x0011).await, 1, "only the prior revision closes"); assert_eq!( rig.media(3_500, 0x0011).await, - 1, - "the deferred revision rides the floor" + 0, + "the deferred revision enters the span at the floor" ); + assert_eq!(rig.media(3_600, 0x0011).await, 1, "the next span closes the revision"); } /// The emission anchor never moves backwards, even where a zero-interval entry @@ -3257,10 +3264,10 @@ async fn si_anchor_survives_a_zero_interval_reorder() { rig.si_track .write_frame(Timestamp::ZERO, Bytes::from(section(0))) .unwrap(); - assert_eq!(rig.media(1_000, 0x0011).await, 1, "zero interval rides every frame"); + assert_eq!(rig.media(1_000, 0x0011).await, 0, "the first span stays buffered"); assert_eq!(rig.media(2_500, 0x0011).await, 1, "the anchor advances to 2.5s"); // A reordered frame steps back behind the anchor; zero interval still emits. - assert_eq!(rig.media(2_400, 0x0011).await, 1, "and still rides a reordered frame"); + assert_eq!(rig.media(2_400, 0x0011).await, 0, "the reordered span stays open"); // The catalog raises the interval to 1s. The floor must measure from the 2.5s // anchor, not the reordered 2.4s emission. @@ -3273,8 +3280,13 @@ async fn si_anchor_survives_a_zero_interval_reorder() { .get_mut(&0x42) .unwrap() .interval = Some(Duration::from_secs(1)); - assert_eq!(rig.media(3_400, 0x0011).await, 0, "the reorder span is not credited"); - assert_eq!(rig.media(3_500, 0x0011).await, 1, "due 1s after the true anchor"); + assert_eq!( + rig.media(3_400, 0x0011).await, + 2, + "both zero-interval frames close together" + ); + assert_eq!(rig.media(3_500, 0x0011).await, 0, "the due table enters the open span"); + assert_eq!(rig.media(3_600, 0x0011).await, 1, "the next span closes the due table"); } /// A publisher revising its snapshot before every frame must not drive the mux at @@ -3294,10 +3306,10 @@ async fn si_rapid_revisions_are_rate_bounded() { let frames = rig.media_frames(u64::from(i) * 250).await; let count = count_pid(&frames, 0x0011); emitted += count; - if i == 4 { + if i == 5 { // The 1s floor lapses here; the emission carries the newest revision, - // not the three that were coalesced over. - assert_eq!(count, 1, "the floored emission fires at 1s"); + // not the revisions that arrived after it while its span was buffered. + assert_eq!(count, 1, "the next span closes the 1s emission"); let needle = section(4); assert!( frames @@ -3307,6 +3319,7 @@ async fn si_rapid_revisions_are_rate_bounded() { ); } } + emitted += rig.media(3_250, 0x0011).await; assert_eq!(emitted, 4, "lead plus one per second, not one per revision"); } @@ -3471,7 +3484,10 @@ async fn export_cbr_video() -> Vec { } video.finish().unwrap(); - let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); + let mut exporter = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_max_age(RECORDING_MAX_AGE); drain_frames(&mut exporter).await } @@ -3550,7 +3566,10 @@ async fn pcr_stays_among_the_bytes_across_reordered_tracks() { import.decode(&BytesMut::from(&data[..])).unwrap(); import.finish().unwrap(); - let mut exporter = Export::new(crate::source::announced(&consumer)).await.unwrap(); + let mut exporter = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_max_age(RECORDING_MAX_AGE); let frames = drain_frames(&mut exporter).await; let pcrs = collect_pcrs(&frames); assert!(pcrs.len() > 50, "expected the full feed, got {} PCRs", pcrs.len()); From ac8806fafaffb97f85a0749388a05a37eb740344 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 4 Sep 2026 21:23:57 -0700 Subject: [PATCH 29/29] feat(uring): report per-worker io_uring counters at /metrics (#3408) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: GPT-5.6 Sol --- doc/bin/relay/config.md | 4 + doc/bin/relay/http.md | 60 ++++++++ quest/m1/README.md | 1 - quest/m1/perf/README.md | 7 +- quest/m1/uring-metrics.md | 47 ------ quest/m1/uring-qlog.md | 5 - quest/m1/uring-tcp/README.md | 5 - rs/CLAUDE.md | 2 +- rs/moq-bench/src/connection.rs | 20 ++- rs/moq-relay/src/internal.rs | 273 +++++++++++++++++++++++++++++++-- rs/moq-relay/src/relay.rs | 8 + rs/moq-relay/src/uring.rs | 23 ++- rs/moq-uring/README.md | 9 ++ rs/moq-uring/src/lib.rs | 7 + rs/moq-uring/src/metrics.rs | 151 ++++++++++++++++++ rs/moq-uring/src/park.rs | 7 +- rs/moq-uring/src/shared.rs | 11 +- rs/moq-uring/src/timer.rs | 36 ++++- rs/moq-uring/src/udp.rs | 29 +++- rs/moq-uring/src/worker.rs | 266 ++++++++++++++++++++++++++++++-- 20 files changed, 865 insertions(+), 106 deletions(-) delete mode 100644 quest/m1/uring-metrics.md create mode 100644 rs/moq-uring/src/metrics.rs diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 00632baca99..f51a62eea95 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -104,6 +104,10 @@ io_uring = false The `io-uring` feature is off by default. Prebuilt binaries that ship it say so; building it yourself is `cargo build -p moq-relay --features io-uring`. +Each worker publishes its own runtime counters at `/metrics` on the internal +listener: buffer-pool health, GRO/GSO batching, ring traffic, and scheduling. +See [HTTP Endpoints](/bin/relay/http) for what to watch. + The `[quic]` section applies to this listener too: `max_streams`, `idle_timeout`, `keep_alive`, `congestion_control` and `gso` are honored. `qlog` and `mtu_discovery` are not (the datagram path sends a fixed payload, diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 52e2b94e65c..b82e5650bb0 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -148,6 +148,66 @@ Nothing here reports host CPU, memory, disk, or network; run a node exporter for those. The QUIC listener has no accept counters at all: it multiplexes every session over one UDP socket, so it never calls `accept` and cannot exhaust it. +#### io\_uring worker counters + +A relay running its QUIC workers on io\_uring (`--runtime-io-uring`, see +[Configuration](/bin/relay/config)) also publishes what each worker thread is doing, one +`worker` label per worker in shard order: + +```text +moq_relay_uring_rx_datagrams_total{worker="0"} 918442 +moq_relay_uring_rx_receives_total{worker="0"} 214905 +moq_relay_uring_rx_enobufs_total{worker="0"} 0 +moq_relay_uring_rx_exhausted_total{worker="0"} 0 +moq_relay_uring_tx_datagrams_total{worker="0"} 1243180 +moq_relay_uring_tx_sends_total{worker="0"} 92311 +moq_relay_uring_tx_stalls_total{worker="0"} 0 +moq_relay_uring_submissions_total{worker="0"} 401552 +moq_relay_uring_completions_total{worker="0"} 401544 +moq_relay_uring_enters_total{worker="0"} 118207 +moq_relay_uring_parks_total{worker="0"} 74390 +moq_relay_uring_wakes_total{worker="0"} 6117 +moq_relay_uring_timers_armed_total{worker="0"} 233417 +moq_relay_uring_timers_fired_total{worker="0"} 74102 +moq_relay_uring_timers_cancelled_total{worker="0"} 159299 +moq_relay_uring_timers_active{worker="0"} 16 +``` + +Every worker gets a full row from the moment the port is bound, so a worker that +failed to start or has died shows stuck zeros rather than vanishing from the +scrape. These describe the *runtime*, not the traffic: they are Prometheus-only +and never appear on the `.stats` broadcast, which carries what crosses the relay +rather than how this process is coping with it. + +The ratios are the point, not the raw counts: + +- `rx_datagrams / rx_receives` is the `UDP_GRO` coalescing actually achieved, and + `tx_datagrams / tx_sends` the `UDP_SEGMENT` batching. Both collapse toward 1 as + connections multiply, which is normal; a sudden drop at constant connection + count is not. +- Datagrams over `moq_relay_uring_enters_total` is the syscall amortization the + io\_uring path exists for. If it approaches 1, the runtime is buying nothing + over the tokio workers. + +The two backpressure counters are what to alert on. A rising +`moq_relay_uring_rx_enobufs_total` means the kernel had a datagram and no free +buffer to select, so it never ran the receive. The datagram stays in the socket +queue, so this is not itself packet loss; it becomes loss once the socket buffer +fills while the pool is still dry. It is receive-side backpressure, and the first +thing to look at when throughput sags with no errors anywhere. For actual drops, +read the socket's own counters (`netstat -su`, `RcvbufErrors`). +`moq_relay_uring_rx_exhausted_total` is the same shortage seen from userspace, a +re-arm that found every buffer still held by an unread packet. +`moq_relay_uring_tx_stalls_total` is the send-side equivalent, counting each +time the staging pool becomes drained at its ceiling and a send has to wait. + +`moq_relay_uring_wakes_total` counts the `futex` syscalls other threads had to +make to wake a parked worker, which is the cost of work crossing threads rather +than staying on the one that owns the connection. The timer counters are churn: +a re-arm counts as one cancel plus one arm, so a large +`timers_cancelled / timers_fired` ratio means deadlines are being rewritten far +more often than they expire. + ### GET /nodes Returns this relay's local view of cluster nodes. A node is included when it is diff --git a/quest/m1/README.md b/quest/m1/README.md index 6f9c348c610..12432f66a42 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -16,7 +16,6 @@ with the current dev tree before starting. ## Quests -- [Worker metrics](/quest/m1/uring-metrics.md) - per-worker io_uring counters at `/metrics`, so the runtime's own health is visible - [Stream sessions](/quest/m1/uring-tcp/README.md) - serve WebSocket and HTTP from the io_uring workers, where io_uring pays off most - [qlog](/quest/m1/uring-qlog.md) - io_uring workers write qlog traces instead of refusing the setting - [Perf](/quest/m1/perf/README.md) - eliminate measured hot-path costs across moq-uring, kio, and the moq-net model: copies, locks, clock reads, allocations, syscalls diff --git a/quest/m1/perf/README.md b/quest/m1/perf/README.md index d3454801cb0..fed0713ce82 100644 --- a/quest/m1/perf/README.md +++ b/quest/m1/perf/README.md @@ -30,9 +30,9 @@ quests don't re-litigate them: the egress `Prefetch`) already exists in moq-net; egress is amortized, ingest and the stream-send path are not. -[Worker metrics](/quest/m1/uring-metrics.md) is a soft dependency: it adds -the ring-level counters (enters, park/wake, batch effectiveness) several -quests want as evidence. Use bench CPU/RSS until it lands. The +The relay's `/metrics` endpoint already carries the ring-level counters +(enters, park/wake, batch effectiveness) several quests want as evidence, one +row per io_uring worker. The [noq parity gate](/quest/m2/quic/noq-parity.md) benchmarks noq against the quiche backend; the zero-copy quests here stay independently measured on the backend we ship today. @@ -55,6 +55,5 @@ backend we ship today. ## Related -- [Worker metrics](/quest/m1/uring-metrics.md) - the counters these quests are judged by - [noq parity gate](/quest/m2/quic/noq-parity.md) - the benchmark that decides whether quiche can go, run on these worker primitives diff --git a/quest/m1/uring-metrics.md b/quest/m1/uring-metrics.md deleted file mode 100644 index 999012646f9..00000000000 --- a/quest/m1/uring-metrics.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] Per-worker io_uring counters at /metrics - -## Goal - -The relay's internal ops listener reports what each io_uring worker is doing. -Per-session traffic already flows (the uring accept path attaches -`with_stats`), but nothing anywhere reports the runtime's own health, so the -failure modes the `moq-uring` README calls out are invisible in production and -the perf quests have no live number to be judged against. - -Prometheus only. The counters are runtime internals, not traffic, so they do -not belong on the moq-stats wire. - -## Plan - -`moq-uring` exposes a counter snapshot per worker (through the existing -`Handle`, which already crosses threads), and `moq-relay`'s `render_metrics` -emits them beside the traffic counters and accept health it already renders. -Hand-formatted like the rest of that function; no metrics-registry dependency. - -Four groups, all cheap relaxed atomics on the worker thread: - -- **Buffer-pool health**: provided-buffer exhaustion, `ENOBUFS`, TX-pool - stalls. Receive-side exhaustion *is* the backpressure, so it is the first - thing to look at when throughput sags. -- **Batch effectiveness**: GRO segments per receive, GSO segments per send, - packets per syscall. M4's exit criteria asked that receive batching and send - GSO stay "observable"; this is what makes that true off the bench. -- **Ring traffic**: submissions, completions, `io_uring_enter` calls. The - syscall reduction the runtime exists for, and the metric - [#3199](/quest/m1/perf/3199-moq-uring-remove-sq-indirection-and-per-enter-ring-fd.md) - and [#3200](/quest/m1/perf/3200-moq-uring-batch-completion-wakeups-with-min-timeout.md) - are judged by. -- **Scheduling**: park/wake counts, timer heap depth, timers armed/fired/ - cancelled. Timer churn and futex wakes were open risks in the original epic, - and the clock-read work in - [#3122](/quest/m1/perf/3122-moq-uring-2-5-of-relay-cpu-is-vdso-clock-reads-the-drive.md) - needs the baseline. - -Per cross-package sync, update the `/metrics` documentation in -`doc/bin/relay/http.md`. Cover the renderer with the exposition-format test -that already guards the existing counters. - -## Related - -- [qlog](/quest/m1/uring-qlog.md) - the trace half of io_uring-mode - observability diff --git a/quest/m1/uring-qlog.md b/quest/m1/uring-qlog.md index 3ac99f29aaf..d418ef93251 100644 --- a/quest/m1/uring-qlog.md +++ b/quest/m1/uring-qlog.md @@ -31,8 +31,3 @@ a bespoke writer thread. Either way, document which one shipped and why. Traces are per connection and named from the connection id, matching what the tokio path already produces, so an existing qlog workflow reads both without knowing which runtime wrote them. - -## Related - -- [Worker metrics](/quest/m1/uring-metrics.md) - the other half of io_uring-mode - observability, on the counter side rather than the trace side diff --git a/quest/m1/uring-tcp/README.md b/quest/m1/uring-tcp/README.md index 396d2c63556..09e263f10ae 100644 --- a/quest/m1/uring-tcp/README.md +++ b/quest/m1/uring-tcp/README.md @@ -37,8 +37,3 @@ ablation's number is what justifies the rest of the line. the `hyper::rt` adapters that let axum run on it - [Relay](/quest/m1/uring-tcp/relay.md) - serve the relay's WebSocket and stream listeners from the io_uring workers - -## Related - -- [Worker metrics](/quest/m1/uring-metrics.md) - the counters that make the - ported path observable, and that the ablation reads diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 3bfdc5c0623..955d34ef27a 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -13,7 +13,7 @@ Layered roughly transport -> container/format -> media -> apps/bindings. - `moq-net` (lib): the core wire layer. Negotiates `moq-lite` or IETF `moq-transport`. Owns the Broadcast/Track/Group/Frame model and the Producer/Consumer split (see below). Generic over `web_transport_trait::Session` (no concrete QUIC dep). Each level of the hierarchy is a public role module that owns short names (`broadcast::Consumer`, `track::Producer`, `group::Info`, `frame::Producer`, `origin::Consumer`, `announce::Consumer`); origin + announce share one private implementation surfaced as two curated modules. Traffic counters for those levels live in the `stats` module (`stats::Registry` collects, `stats::Handle` is what a session bumps through); publishing them as broadcasts lives in `moq-stats`. - `moq-tokio` (lib): native connection helpers. `ClientConfig`/`ServerConfig` wrap QUIC backends (Quinn/Quiche/Noq/Iroh), WebTransport, WebSocket, TCP (qmux), Unix sockets, TLS, cert hot-reload, logging, jemalloc. Re-exports `moq_net`, and `moq_sock::bind` as `moq_tokio::bind`. Example: `examples/clock.rs`. - `moq-sock` (lib): socket and thread-per-core listener plumbing shared by `moq-tokio` and `moq-uring`. `bind` (dual-stack UDP/TCP, grown buffers, `SO_REUSEPORT`), `shard` (reuseport group formation with the port lock and bind probe, the cBPF connection-id steering filter, `Shard`, `cid_prefix`), `cpu` (core pinning). The group-formation invariants (bind in index order, never resize) live in its docs; each runtime's worker group is what holds them. -- `moq-uring` (lib, experimental): the Linux thread-per-core io\_uring runtime. `Worker` (one per pinned thread, Linux 6.12 floor, no fallback) owns the ring, a userspace timer heap (`Handle` implements `moq_net::Timers`), a local `!Send` task set, and futex-word parking; `udp::Socket` does multishot provided-buffer receive with `UDP_GRO` and pooled `UDP_SEGMENT` sends; `quic::Endpoint` stacks a sans-IO QUIC stack on that path, serving many connections per socket demuxed by connection id (dials included, ids rotated, unsupported versions negotiated); native peers speak raw QUIC (ALPN-negotiated) while browsers negotiate `h3` and `quic::web::Request` runs the HTTP/3 CONNECT handshake via `web-transport-proto`, with `quic::web::Session` as the runtime's one transport type covering both flavors (`Session::raw` wraps raw connections) and `Handle` as the `moq_net::Runtime`, so `connect_lite`/`accept_lite` run moq-lite sessions on the worker either way. The stack under `quic` is one of three cargo features, with one selected at compile time: `noq` (default, noq-proto over rustls), `quinn` (quinn-proto over rustls), or `quiche` (BoringSSL); `noq` wins if a build asks for several, and with none the `quic` module is left out and the crate is the worker alone. moq-relay picks the same choice with `io-uring` (noq), `io-uring-quinn`, or `io-uring-quiche`, which share the private `_uring` feature that gates the listener itself. `just rs uring` runs all three. Validated by a raw-quiche echo (`tests/echo.rs`), full moq-lite pub/sub sessions including two clients on one server socket (`tests/session.rs`), the endpoint mechanics (`tests/endpoint.rs`), a steered two-worker group (`tests/workers.rs`), and WebTransport interop against `web-transport-quinn` including moq-lite over it (`tests/web.rs`), all kernel-gated and all run against whichever backend is compiled, plus the `echo_quiche` ablation benchmark; `benches/udp_{tokio,uring}.rs` are the older disposable syscall matrices. +- `moq-uring` (lib, experimental): the Linux thread-per-core io\_uring runtime. `Worker` (one per pinned thread, Linux 6.12 floor, no fallback) owns the ring, a userspace timer heap (`Handle` implements `moq_net::Timers`), a local `!Send` task set, and futex-word parking; `udp::Socket` does multishot provided-buffer receive with `UDP_GRO` and pooled `UDP_SEGMENT` sends; `quic::Endpoint` stacks a sans-IO QUIC stack on that path, serving many connections per socket demuxed by connection id (dials included, ids rotated, unsupported versions negotiated); native peers speak raw QUIC (ALPN-negotiated) while browsers negotiate `h3` and `quic::web::Request` runs the HTTP/3 CONNECT handshake via `web-transport-proto`, with `quic::web::Session` as the runtime's one transport type covering both flavors (`Session::raw` wraps raw connections) and `Handle` as the `moq_net::Runtime`, so `connect_lite`/`accept_lite` run moq-lite sessions on the worker either way. `Metrics` is the worker's own health leaving its thread: relaxed counters for the buffer pools, GRO/GSO batching, ring traffic, and scheduling, handed in through `Config::metrics` or read back through `Handle::metrics`, which is what moq-relay publishes per worker at `/metrics`. The stack under `quic` is one of three cargo features, with one selected at compile time: `noq` (default, noq-proto over rustls), `quinn` (quinn-proto over rustls), or `quiche` (BoringSSL); `noq` wins if a build asks for several, and with none the `quic` module is left out and the crate is the worker alone. moq-relay picks the same choice with `io-uring` (noq), `io-uring-quinn`, or `io-uring-quiche`, which share the private `_uring` feature that gates the listener itself. `just rs uring` runs all three. Validated by a raw-quiche echo (`tests/echo.rs`), full moq-lite pub/sub sessions including two clients on one server socket (`tests/session.rs`), the endpoint mechanics (`tests/endpoint.rs`), a steered two-worker group (`tests/workers.rs`), and WebTransport interop against `web-transport-quinn` including moq-lite over it (`tests/web.rs`), all kernel-gated and all run against whichever backend is compiled, plus the `echo_quiche` ablation benchmark; `benches/udp_{tokio,uring}.rs` are the older disposable syscall matrices. - `kio` (lib): "easy async". `Producer`/`Consumer` shared-state channels with `Waiter`-based notification, built on `std::task::Waker`, no runtime dependency. Underpins all the `poll_*` plumbing in moq-net and moq-mux. `src/producer.rs`, `src/consumer.rs`, `src/waiter.rs`. Implement `Pollable` (a `poll(&Waiter)` computation) and wrap it in `Pending` to get a `std::future::Future` (`src/pollable.rs`). Guard discipline: the synchronous methods (`write`, `poll*`) report closure as `Err(Ref)`, a live lock guard; the `async` ones report it as `kio::Closed` instead, since an `Err` held across a later `.await` would stall every other handle. **Container / catalog formats** (standalone specs, mostly no moq-\* deps, reused by moq-mux) diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 6057b517bd0..e60d7307420 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -579,6 +579,12 @@ impl Drop for Gauge<'_> { mod tests { use super::*; + async fn wait_for(counter: &AtomicU64, value: u64) { + while counter.load(Ordering::Relaxed) < value { + tokio::task::yield_now().await; + } + } + fn rolled(fps: u64, frame_size: u64, group_size: u64) -> Rolled { Rolled { broadcasts: 1, @@ -828,6 +834,10 @@ mod tests { let stats = stats.clone(); tokio::spawn(async move { drain(consumer, &stats).await }) }; + // `subscribe(None)` starts at the live frontier. Let the drain consume + // group 0 before opening group 1, or the task may subscribe to group 1 + // and the frame count can never reach two. + wait_for(&stats.frames_recv, 1).await; // Group 1 opens and is picked up by the drain, then the relay gives up on it. // Aborting a group nobody is reading drops its cached frames, so wait for the @@ -836,18 +846,14 @@ mod tests { group .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) .unwrap(); - for _ in 0..100 { - if stats.frames_recv.load(Ordering::Relaxed) >= 2 { - break; - } - tokio::task::yield_now().await; - } - assert_eq!(stats.frames_recv.load(Ordering::Relaxed), 2, "drain is inside group 1"); + wait_for(&stats.frames_recv, 2).await; group.abort(moq_net::Error::Lagged).unwrap(); // Groups 2 and 3 land intact, then the publisher is done. write_group(&mut track); + wait_for(&stats.groups_recv, 2).await; write_group(&mut track); + wait_for(&stats.groups_recv, 3).await; track.finish().unwrap(); broadcast.finish(); diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index edc90b06249..a1b3fd28ef2 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -8,10 +8,13 @@ //! Today it serves: //! - `/metrics` - this node's own traffic counters as Prometheus text //! exposition, plus the accept-loop health of its TCP listeners -//! ([`with_listeners`](Internal::with_listeners)). A distinct plane from both the -//! customer `web` surface and the MoQ `.stats` broadcast: the same atomics, but -//! a different transport and audience (an ops scraper, not a customer or the -//! dashboard/billing aggregators). +//! ([`with_listeners`](Internal::with_listeners)) and the per-worker health of +//! its io_uring runtime ([`with_uring`](Internal::with_uring)). A distinct plane +//! from both the customer `web` surface and the MoQ `.stats` broadcast: the same +//! atomics, but a different transport and audience (an ops scraper, not a +//! customer or the dashboard/billing aggregators). The runtime counters are +//! Prometheus-only on purpose: they describe this process, not the traffic +//! crossing it, so they have no place on the `moq-stats` wire. //! - `/health` - a liveness mirror of the public probe, for internal checks //! that don't want to hit the customer port. //! - `/nodes` - the cluster nodes visible through gossip plus established @@ -36,6 +39,20 @@ use axum::{ }; use axum_server::accept::DefaultAcceptor; +/// One io_uring worker's counters, as [`with_uring`](Internal::with_uring) takes +/// them. +/// +/// Uninhabited when the relay is built without the io_uring listener, so the +/// list is always empty there and the renderer skips itself, rather than +/// `cfg`-ing the two fields, the two struct literals that fill them, the +/// builder and the renderer's signature separately. +#[cfg(all(target_os = "linux", feature = "_uring"))] +type UringWorker = moq_uring::metrics::Metrics; + +/// One io_uring worker's counters. See the io_uring build's alias. +#[cfg(not(all(target_os = "linux", feature = "_uring")))] +type UringWorker = std::convert::Infallible; + /// Configuration for the internal (ops) listener. #[derive(usage::Args, Clone, Debug, serde::Deserialize, serde::Serialize, Default)] #[usage(unknown_flags = "error", args_override_self = false)] @@ -63,6 +80,7 @@ pub struct Internal { nodes: Option, health: moq_tokio::accept::Health, listeners: Vec, + uring: Vec, } #[derive(Clone)] @@ -70,6 +88,7 @@ struct InternalState { stats: moq_net::stats::Registry, nodes: Option, listeners: Vec, + uring: Vec, } impl Internal { @@ -92,6 +111,7 @@ impl Internal { nodes: None, health, listeners, + uring: Vec::new(), } } @@ -127,6 +147,21 @@ impl Internal { self } + /// Report the io_uring QUIC workers' runtime counters at `/metrics`. + /// + /// Takes the counter sets `uring::Workers::metrics` hands out at bind time, + /// in shard order: the position in the iterator is the `worker` label. + /// Register them before the threads start, so a worker that never came up + /// (or has died) still has a series that has stopped moving rather than no + /// series at all. + /// + /// Off the io_uring build the item type is uninhabited, so there is nothing + /// to register and nothing is rendered. + pub fn with_uring(mut self, workers: impl IntoIterator) -> Self { + self.uring.extend(workers); + self + } + /// Attach the relay cluster used to serve the `/nodes` topology snapshot. pub fn with_cluster(mut self, cluster: &crate::Cluster) -> Self { self.nodes = Some(cluster.nodes.clone()); @@ -148,6 +183,7 @@ impl Internal { stats: self.stats.clone(), nodes: self.nodes.clone(), listeners: self.listeners.clone(), + uring: self.uring.clone(), }) } @@ -207,7 +243,7 @@ async fn serve_health() -> Response { /// current cumulative snapshot; a downstream scraper derives rates and live /// counts (`open - closed`). async fn serve_metrics(State(state): State) -> Response { - let body = render_metrics(&state.stats.snapshot(), &state.listeners); + let body = render_metrics(&state.stats.snapshot(), &state.listeners, &state.uring); ([(http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], body).into_response() } @@ -226,7 +262,11 @@ async fn serve_nodes(State(state): State) -> Json String { +fn render_metrics( + snap: &moq_net::stats::Snapshot, + listeners: &[moq_tokio::accept::Health], + uring: &[UringWorker], +) -> String { use std::fmt::Write as _; let traffic = snap.traffic(); @@ -347,6 +387,7 @@ fn render_metrics(snap: &moq_net::stats::Snapshot, listeners: &[moq_tokio::accep } render_accepts(&mut out, listeners); + render_uring(&mut out, uring); out } @@ -395,6 +436,130 @@ fn render_accepts(out: &mut String, listeners: &[moq_tokio::accept::Health]) { } } +/// The io_uring runtime's per-worker counters, one `worker` label per bound +/// worker. +/// +/// These describe the runtime rather than the traffic: how well the receive +/// pool, the batching mechanisms, and the ring itself are holding up on each +/// pinned thread. A worker is a thread that never yields to anything else on the +/// node, so nothing else here can report that one of them has gone sick, and the +/// failure modes are quiet ones: a receive pool running dry leaves datagrams +/// queued in the kernel with no error anywhere, and batching that has collapsed +/// just costs syscalls. +/// +/// The ratios are what to watch, not the raw counts: `rx_datagrams / rx_receives` +/// is the GRO coalescing actually achieved, `tx_datagrams / tx_sends` the GSO +/// batching, and datagrams over `enters` the syscall amortization the runtime +/// exists for. +#[cfg(all(target_os = "linux", feature = "_uring"))] +fn render_uring(out: &mut String, workers: &[UringWorker]) { + use std::fmt::Write as _; + + if workers.is_empty() { + return; + } + let snaps: Vec = workers.iter().map(|worker| worker.snapshot()).collect(); + + // One HELP/TYPE header followed by a row per worker, as `render_metrics` + // does for the traffic counters. + let mut series = |name: &str, kind: &str, help: &str, field: fn(&moq_uring::metrics::Snapshot) -> u64| { + let _ = writeln!(out, "# HELP {name} {help}"); + let _ = writeln!(out, "# TYPE {name} {kind}"); + for (worker, snap) in snaps.iter().enumerate() { + let _ = writeln!(out, "{name}{{worker=\"{worker}\"}} {}", field(snap)); + } + }; + + let mut counter = |name, help, field| series(name, "counter", help, field); + + counter( + "moq_relay_uring_rx_datagrams_total", + "UDP datagrams received by an io_uring worker, counting each UDP_GRO segment.", + |snap| snap.rx_datagrams, + ); + counter( + "moq_relay_uring_rx_receives_total", + "Receive completions that delivered datagrams; datagrams over this is the GRO coalescing achieved.", + |snap| snap.rx_receives, + ); + counter( + "moq_relay_uring_rx_enobufs_total", + "Receives the kernel ended with ENOBUFS: no provided buffer was free, so the receive never ran. Backpressure, not a confirmed loss.", + |snap| snap.rx_enobufs, + ); + counter( + "moq_relay_uring_rx_exhausted_total", + "Receive re-arms that found no free buffer, leaving the socket unarmed until a packet was read.", + |snap| snap.rx_exhausted, + ); + counter( + "moq_relay_uring_tx_datagrams_total", + "UDP datagrams sent by an io_uring worker, counting each UDP_SEGMENT segment.", + |snap| snap.tx_datagrams, + ); + counter( + "moq_relay_uring_tx_sends_total", + "sendmsg operations staged; datagrams over this is the GSO batching achieved.", + |snap| snap.tx_sends, + ); + counter( + "moq_relay_uring_tx_stalls_total", + "Times the send-buffer pool became drained at its ceiling and an acquisition had to wait.", + |snap| snap.tx_stalls, + ); + counter( + "moq_relay_uring_submissions_total", + "Submission queue entries the kernel accepted.", + |snap| snap.submissions, + ); + counter( + "moq_relay_uring_completions_total", + "Completion queue entries dispatched.", + |snap| snap.completions, + ); + counter( + "moq_relay_uring_enters_total", + "io_uring_enter calls; datagrams over this is the syscall amortization.", + |snap| snap.enters, + ); + counter( + "moq_relay_uring_parks_total", + "Times a worker parked in io_uring_enter with nothing left to poll.", + |snap| snap.parks, + ); + counter( + "moq_relay_uring_wakes_total", + "futex wakes another thread had to issue because the worker was parked.", + |snap| snap.wakes, + ); + counter( + "moq_relay_uring_timers_armed_total", + "Timers armed, re-arms included.", + |snap| snap.timers_armed, + ); + counter( + "moq_relay_uring_timers_fired_total", + "Timers that reached their deadline.", + |snap| snap.timers_fired, + ); + counter( + "moq_relay_uring_timers_cancelled_total", + "Timers dropped or re-armed before their deadline.", + |snap| snap.timers_cancelled, + ); + series( + "moq_relay_uring_timers_active", + "gauge", + "Timers currently in a worker's heap.", + |snap| snap.timers_active(), + ); +} + +/// Off the io_uring path there are no workers to describe, and [`UringWorker`] +/// is uninhabited, so the list handed here is always empty. +#[cfg(not(all(target_os = "linux", feature = "_uring")))] +fn render_uring(_out: &mut String, _workers: &[UringWorker]) {} + #[cfg(test)] mod tests { use super::*; @@ -416,7 +581,11 @@ mod tests { #[test] fn absent_listeners_are_not_reported_as_healthy() { let disabled = Internal::new(InternalConfig::default(), moq_net::stats::Registry::disabled()); - let body = render_metrics(&moq_net::stats::Registry::disabled().snapshot(), &disabled.listeners); + let body = render_metrics( + &moq_net::stats::Registry::disabled().snapshot(), + &disabled.listeners, + &[], + ); assert!( !body.contains("listener=\"internal\""), "a disabled internal listener must not publish counters:\n{body}" @@ -426,7 +595,11 @@ mod tests { let stream_only = Internal::new(listening(), moq_net::stats::Registry::disabled()) .with_listeners(None) .with_listeners([moq_tokio::accept::Health::new("tcp")]); - let body = render_metrics(&moq_net::stats::Registry::disabled().snapshot(), &stream_only.listeners); + let body = render_metrics( + &moq_net::stats::Registry::disabled().snapshot(), + &stream_only.listeners, + &[], + ); assert!( body.contains("moq_relay_accept_failures_total{listener=\"tcp\",class=\"exhausted\"} 0"), "the tcp listener must be reported:\n{body}" @@ -450,7 +623,11 @@ mod tests { let web = moq_tokio::accept::Health::new("web"); let internal = Internal::new(listening(), moq_net::stats::Registry::disabled()).with_listeners([web.clone()]); - let body = render_metrics(&moq_net::stats::Registry::disabled().snapshot(), &internal.listeners); + let body = render_metrics( + &moq_net::stats::Registry::disabled().snapshot(), + &internal.listeners, + &[], + ); for listener in ["internal", "web"] { for class in ["connection", "exhausted", "unknown"] { @@ -473,7 +650,11 @@ mod tests { moq_tokio::accept::Failure::Exhausted ); let _ = web.failed(&emfile); - let body = render_metrics(&moq_net::stats::Registry::disabled().snapshot(), &internal.listeners); + let body = render_metrics( + &moq_net::stats::Registry::disabled().snapshot(), + &internal.listeners, + &[], + ); assert!(body.contains("moq_relay_accept_failures_total{listener=\"web\",class=\"exhausted\"} 1")); let prefix = "moq_relay_accept_stalled_seconds{listener=\"web\"} "; @@ -486,6 +667,74 @@ mod tests { assert!(stalled > 0.0, "a stalled listener must not report zero seconds"); } + /// Every io_uring worker gets a full row of series from zero, for the same + /// reason the accept counters do: `rate()` over a series that does not exist + /// yet is empty rather than zero, so an alert on a worker whose receive pool + /// has started running dry is unwritable until it has already happened. A + /// worker that never started has to publish stuck zeros, not nothing. + #[cfg(all(target_os = "linux", feature = "_uring"))] + #[test] + fn uring_metrics_list_every_worker_from_zero() { + let workers = vec![ + moq_uring::metrics::Metrics::default(), + moq_uring::metrics::Metrics::default(), + ]; + let internal = Internal::new(listening(), moq_net::stats::Registry::disabled()).with_uring(workers); + let body = render_metrics( + &moq_net::stats::Registry::disabled().snapshot(), + &internal.listeners, + &internal.uring, + ); + + // Every counter the module documents, for every worker, plus the derived + // gauge. A name that drifts out of the renderer silently deletes a + // dashboard, so the list is spelled out rather than derived from it. + let counters = [ + "rx_datagrams", + "rx_receives", + "rx_enobufs", + "rx_exhausted", + "tx_datagrams", + "tx_sends", + "tx_stalls", + "submissions", + "completions", + "enters", + "parks", + "wakes", + "timers_armed", + "timers_fired", + "timers_cancelled", + ]; + for counter in counters { + let name = format!("moq_relay_uring_{counter}_total"); + assert!( + body.contains(&format!("# TYPE {name} counter")), + "missing type for {name}" + ); + for worker in 0..2 { + let line = format!("{name}{{worker=\"{worker}\"}} 0"); + assert!(body.contains(&line), "missing {line} in:\n{body}"); + } + } + assert!(body.contains("# TYPE moq_relay_uring_timers_active gauge")); + assert!(body.contains("moq_relay_uring_timers_active{worker=\"1\"} 0")); + } + + /// A relay with no io_uring workers publishes no worker series at all, + /// rather than a `worker="0"` row that is permanently zero because nothing + /// is behind it. + #[test] + fn absent_uring_workers_are_not_reported() { + let internal = Internal::new(listening(), moq_net::stats::Registry::disabled()); + let body = render_metrics( + &moq_net::stats::Registry::disabled().snapshot(), + &internal.listeners, + &internal.uring, + ); + assert!(!body.contains("moq_relay_uring_"), "unexpected worker series:\n{body}"); + } + /// `serve` hosts the router it is HANDED, so an embedder's extra ops routes /// answer on the same internal listener as the built-in ones. That is what /// lets an embedder extend this surface without binding a socket of its own @@ -553,6 +802,7 @@ mod tests { stats: moq_net::stats::Registry::disabled(), nodes: None, listeners: Vec::new(), + uring: Vec::new(), }; let Json(snapshot) = serve_nodes(State(state)).await; @@ -568,6 +818,7 @@ mod tests { stats: moq_net::stats::Registry::disabled(), nodes: Some(nodes), listeners: Vec::new(), + uring: Vec::new(), }; let Json(snapshot) = serve_nodes(State(state)).await; @@ -634,7 +885,7 @@ mod tests { group.finish().unwrap(); } - let body = render_metrics(&stats.snapshot(), &[]); + let body = render_metrics(&stats.snapshot(), &[], &[]); assert!( body.contains("# TYPE moq_relay_bytes_total counter"), diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 839c5fe2d58..72fbbc28421 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -225,6 +225,14 @@ impl Relay { .with_cluster(&cluster) .with_listeners(web.accept_health()) .with_listeners(server.accept_health()); + // Bound but not yet serving: registering here (rather than after the + // threads start) is what gives every worker a series from the first + // scrape, including one that is about to fail setup. + #[cfg(all(target_os = "linux", feature = "_uring"))] + let internal = match uring.as_ref() { + Some(uring) => internal.with_uring(uring.metrics()), + None => internal, + }; match addr { Some(addr) => tracing::info!(%addr, "listening"), diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 00a57dccbc2..bce78cdc0c9 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -90,6 +90,12 @@ pub struct Workers { server: moq_uring::quic::server::Config, /// What the workers serve, fingerprinted once, for `/certificate.sha256`. certificates: moq_tokio::tls::Certificates, + /// One counter set per worker, in shard order. Created here rather than on + /// the worker thread so `/metrics` can publish every worker's series from + /// the first scrape, before the threads exist: a series that only appears + /// once a worker has done something reads as a healthy node until it is too + /// late to notice. + metrics: Vec, udp: moq_uring::udp::Config, pin: bool, alpns: Arc>, @@ -253,6 +259,7 @@ impl Workers { tracing::info!(workers = count, %addr, "bound io_uring QUIC workers"); Ok(Self { + metrics: (0..count).map(|_| moq_uring::metrics::Metrics::default()).collect(), members, addr, server, @@ -274,6 +281,14 @@ impl Workers { self.addr } + /// Every worker's counters, in shard order, for an ops surface to scrape. + /// + /// Available from [`bind`](Self::bind), so a scrape covers a worker that has + /// not started (or has died) instead of dropping its series. + pub fn metrics(&self) -> Vec { + self.metrics.clone() + } + /// The certificate every worker presents, for publishing its SHA-256 /// fingerprint. /// @@ -313,6 +328,7 @@ impl Workers { let spawn = Spawn { member, core, + metrics: self.metrics[index as usize].clone(), server: self.server.clone(), udp: self.udp.clone(), serve: serve.clone(), @@ -436,6 +452,8 @@ struct Spawn { member: Member, /// The core to pin to, or `None` when pinning is off or unavailable. core: Option, + /// This worker's counter set, shared with whoever scrapes `/metrics`. + metrics: moq_uring::metrics::Metrics, server: moq_uring::quic::server::Config, udp: moq_uring::udp::Config, serve: Serve, @@ -482,6 +500,7 @@ fn serve_worker(spawn: Spawn) -> bool { let Spawn { member, core, + metrics, server, udp, serve, @@ -499,7 +518,9 @@ fn serve_worker(spawn: Spawn) -> bool { } let setup = (|| -> anyhow::Result<(moq_uring::Worker, moq_uring::quic::Endpoint)> { - let worker = moq_uring::Worker::new(Default::default()).context("io_uring setup failed")?; + let mut config = moq_uring::Config::default(); + config.metrics = metrics; + let worker = moq_uring::Worker::new(config).context("io_uring setup failed")?; let handle = worker.handle(); let socket = handle .udp(member.socket, udp) diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index e5a866ddbd6..9a6d2df64d3 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -33,6 +33,15 @@ the UDP sockets bound through it. `connect_lite`/`accept_lite` run moq-lite sessions on the worker either way, with stream and close codes mapped through the HTTP/3 error space in web mode. +- **Metrics**: a set of relaxed atomic counters per worker, read from any + thread through `metrics::Metrics::snapshot`. Buffer-pool health (`ENOBUFS`, + provided-buffer exhaustion, TX-pool stalls), batch effectiveness (datagrams + per receive and per send), ring traffic (submissions, completions, + `io_uring_enter` calls), and scheduling (parks, remote futex wakes, timer + churn). Pass a `metrics::Metrics` to `Config::metrics` to hold a copy on the + thread that spawned the worker, or read the worker's own with + `Handle::metrics`. `moq-relay` publishes them at `/metrics` on its internal + listener. - **Steering**: an endpoint whose socket sits in a `moq-sock` steered `SO_REUSEPORT` group sets `endpoint::Config::shard`, and every issued connection id leads with the group's steering byte, so the kernel keeps a diff --git a/rs/moq-uring/src/lib.rs b/rs/moq-uring/src/lib.rs index a2efdb0a4f3..b7627cfc17c 100644 --- a/rs/moq-uring/src/lib.rs +++ b/rs/moq-uring/src/lib.rs @@ -21,6 +21,12 @@ //! `quinn`, or `quiche` feature; the module is the same either way, and a build with //! neither leaves it out. //! +//! [`metrics::Metrics`] is how the worker's own health leaves its thread: +//! relaxed counters for the buffer pools, the batching mechanisms, the ring, +//! and the scheduler, snapshotted from anywhere. Hand one to +//! [`Config::metrics`] to keep a copy where the worker was spawned, or read the +//! worker's own through [`Handle::metrics`]. +//! //! Requires Linux 6.12; [`Worker::new`] refuses older kernels with a legible //! error instead of degrading. The crate compiles to nothing off Linux. // Off Linux the crate compiles to nothing, so these doc links have no target. @@ -28,6 +34,7 @@ #![cfg(target_os = "linux")] mod error; +pub mod metrics; mod park; #[cfg(any(feature = "noq", feature = "quiche", feature = "quinn"))] pub mod quic; diff --git a/rs/moq-uring/src/metrics.rs b/rs/moq-uring/src/metrics.rs new file mode 100644 index 00000000000..020249eca95 --- /dev/null +++ b/rs/moq-uring/src/metrics.rs @@ -0,0 +1,151 @@ +//! Per-worker counters: buffer-pool health, batch effectiveness, ring traffic, +//! and scheduling. +//! +//! A worker is a thread that never yields to anything an ops surface can see, +//! so these are how its health leaves the thread. Every write is a relaxed +//! atomic add on the worker's own thread and every read is a relaxed load from +//! whichever thread scrapes, which makes a [`Snapshot`] a cheap, slightly +//! skewed reading rather than a consistent instant. Rates and ratios are what +//! these are for; two counters in one snapshot may be a few operations apart. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// One cumulative counter. +#[derive(Default)] +pub(crate) struct Counter(AtomicU64); + +impl Counter { + pub(crate) fn add(&self, count: u64) { + // Relaxed: the writer is one thread and readers want a magnitude, not + // an ordering against the operation being counted. + self.0.fetch_add(count, Ordering::Relaxed); + } + + fn get(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } +} + +/// The counters themselves, shared by the worker, its sockets, its timer heap, +/// and its park word. +#[derive(Default)] +pub(crate) struct Counters { + pub rx_datagrams: Counter, + pub rx_receives: Counter, + pub rx_enobufs: Counter, + pub rx_exhausted: Counter, + pub tx_datagrams: Counter, + pub tx_sends: Counter, + pub tx_stalls: Counter, + pub submissions: Counter, + pub completions: Counter, + pub enters: Counter, + pub parks: Counter, + pub wakes: Counter, + pub timers_armed: Counter, + pub timers_fired: Counter, + pub timers_cancelled: Counter, +} + +/// A worker's counters, readable from any thread. +/// +/// Hand one to [`crate::Config::metrics`] to keep a copy the process can scrape +/// while the worker runs, or take the worker's own through +/// [`crate::Handle::metrics`]. Clones share one set of counters, so give each +/// worker its own. +#[derive(Clone, Default)] +pub struct Metrics(Arc); + +impl Metrics { + pub(crate) fn counters(&self) -> &Arc { + &self.0 + } + + pub(crate) fn from_counters(counters: Arc) -> Self { + Self(counters) + } + + /// Read every counter. + pub fn snapshot(&self) -> Snapshot { + Snapshot { + rx_datagrams: self.0.rx_datagrams.get(), + rx_receives: self.0.rx_receives.get(), + rx_enobufs: self.0.rx_enobufs.get(), + rx_exhausted: self.0.rx_exhausted.get(), + tx_datagrams: self.0.tx_datagrams.get(), + tx_sends: self.0.tx_sends.get(), + tx_stalls: self.0.tx_stalls.get(), + submissions: self.0.submissions.get(), + completions: self.0.completions.get(), + enters: self.0.enters.get(), + parks: self.0.parks.get(), + wakes: self.0.wakes.get(), + timers_armed: self.0.timers_armed.get(), + timers_fired: self.0.timers_fired.get(), + timers_cancelled: self.0.timers_cancelled.get(), + } + } +} + +impl std::fmt::Debug for Metrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.snapshot().fmt(f) + } +} + +/// A reading of one worker's counters, all cumulative since it started. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct Snapshot { + /// UDP datagrams received, counting each `UDP_GRO` coalesced segment. + pub rx_datagrams: u64, + /// Receive completions that delivered datagrams. `rx_datagrams` over this + /// is the GRO coalescing actually achieved. + pub rx_receives: u64, + /// Receives the kernel ended with `ENOBUFS`: the provided-buffer ring was + /// empty, so no buffer could be selected and the receive was never + /// performed. The datagram stays in the socket queue, so this is + /// receive-side backpressure rather than a confirmed loss; sustained, it + /// becomes one, once the socket buffer fills. The first thing to look at + /// when throughput sags. + pub rx_enobufs: u64, + /// Re-arms that found no free receive buffer at all, so the socket was left + /// unarmed until a packet released one. The pool is at its ceiling and + /// every buffer is held by an unread packet. + pub rx_exhausted: u64, + /// UDP datagrams sent, counting each `UDP_SEGMENT` segment of a GSO train. + pub tx_datagrams: u64, + /// `sendmsg` operations staged. `tx_datagrams` over this is the GSO + /// batching actually achieved. + pub tx_sends: u64, + /// Times the send-buffer pool became drained at its ceiling and an + /// acquisition had to wait. Send-side backpressure. + pub tx_stalls: u64, + /// Submission queue entries the kernel accepted. + pub submissions: u64, + /// Completion queue entries dispatched. + pub completions: u64, + /// `io_uring_enter` calls. Datagrams over this is the syscall amortization + /// the runtime exists for. + pub enters: u64, + /// Times the worker parked in `io_uring_enter` with nothing left to poll. + pub parks: u64, + /// `futex` wakes another thread had to issue because the worker was parked. + pub wakes: u64, + /// Timers armed, re-arms included (a re-arm is a cancel plus an arm). + pub timers_armed: u64, + /// Timers that reached their deadline. + pub timers_fired: u64, + /// Timers dropped or re-armed before their deadline. + pub timers_cancelled: u64, +} + +impl Snapshot { + /// Timers currently in the heap: armed, less those fired and cancelled. + pub fn timers_active(&self) -> u64 { + self.timers_armed + .saturating_sub(self.timers_fired) + .saturating_sub(self.timers_cancelled) + } +} diff --git a/rs/moq-uring/src/park.rs b/rs/moq-uring/src/park.rs index fd3fb172b6f..a833d218a7e 100644 --- a/rs/moq-uring/src/park.rs +++ b/rs/moq-uring/src/park.rs @@ -12,6 +12,8 @@ use std::sync::{ }; use std::task::{Wake, Waker}; +use crate::metrics::Counters; + /// The worker is polling; a wake only needs to prevent the next park. pub(crate) const RUNNING: u32 = 0; /// A wake arrived; the next park attempt consumes it and re-polls instead. @@ -29,12 +31,14 @@ pub(crate) const FUTEX_BITSET_MATCH_ANY: u64 = u32::MAX as u64; /// mints, so a wake from any thread lands here. pub(crate) struct Unpark { pub(crate) word: AtomicU32, + metrics: Arc, } impl Unpark { - pub(crate) fn new() -> Arc { + pub(crate) fn new(metrics: Arc) -> Arc { Arc::new(Self { word: AtomicU32::new(RUNNING), + metrics, }) } @@ -42,6 +46,7 @@ impl Unpark { /// kick the futex so the armed `FUTEX_WAIT` completes. pub(crate) fn unpark(&self) { if self.word.swap(NOTIFIED, Ordering::AcqRel) == PARKED { + self.metrics.wakes.add(1); // SAFETY: waking a futex reads no user memory beyond the address. unsafe { libc::syscall( diff --git a/rs/moq-uring/src/shared.rs b/rs/moq-uring/src/shared.rs index 7990065cb81..8e7ab750566 100644 --- a/rs/moq-uring/src/shared.rs +++ b/rs/moq-uring/src/shared.rs @@ -14,6 +14,7 @@ use std::time::Instant; use io_uring::{IoUring, opcode, squeue}; +use crate::metrics::Counters; use crate::park::Unpark; use crate::{timer, udp}; @@ -58,6 +59,9 @@ pub(crate) struct Shared { /// Tasks handed to [`crate::Handle::spawn`], drained by the worker loop. pub spawns: RefCell>, pub unpark: std::sync::Arc, + /// This worker's counters, also held by its sockets, timer heap, and park + /// word. + pub metrics: std::sync::Arc, /// The next provided-buffer group id; one per socket. pub next_bgid: Cell, /// Set by [`crate::Worker`]'s drop: handles outlive the loop that would @@ -103,7 +107,12 @@ impl Shared { return Ok(()); } } - if let Err(err) = ring.submit() { + self.metrics.enters.add(1); + let submitted = ring.submit(); + if let Ok(count) = &submitted { + self.metrics.submissions.add(*count as u64); + } + if let Err(err) = submitted { // A deadline-bounded teardown can safely retry an interrupted // enter; ordinary callers surface it to their worker. if deadline.is_some() && err.raw_os_error() == Some(libc::EINTR) { diff --git a/rs/moq-uring/src/timer.rs b/rs/moq-uring/src/timer.rs index 850ee8cddcf..e093e5fe592 100644 --- a/rs/moq-uring/src/timer.rs +++ b/rs/moq-uring/src/timer.rs @@ -9,14 +9,17 @@ use std::cell::{Cell, RefCell}; use std::collections::BTreeMap; use std::rc::Rc; +use std::sync::Arc; use std::task::Poll; use std::time::Instant; +use crate::metrics::Counters; + /// The ordered set of armed timers, keyed by (deadline, tiebreak). -#[derive(Default)] pub(crate) struct Heap { queue: BTreeMap<(Instant, u64), Rc>, seq: u64, + metrics: Arc, } /// One timer's state, shared between its [`Timer`] handle and the heap. @@ -29,6 +32,14 @@ struct Slot { } impl Heap { + pub fn new(metrics: Arc) -> Self { + Self { + queue: BTreeMap::new(), + seq: 0, + metrics, + } + } + /// Fire everything due at `now`. Returns whether anything fired. pub fn fire(&mut self, now: Instant) -> bool { let mut fired = false; @@ -40,6 +51,7 @@ impl Heap { slot.key.set(None); slot.elapsed.set(true); slot.waiters.borrow_mut().wake(); + self.metrics.timers_fired.add(1); fired = true; } fired @@ -54,11 +66,23 @@ impl Heap { self.seq += 1; let key = (at, self.seq); self.queue.insert(key, slot); + self.metrics.timers_armed.add(1); key } - fn remove(&mut self, key: (Instant, u64)) { - self.queue.remove(&key); + /// Drop an armed timer before its deadline. A re-arm goes through here + /// too, so the counter reads as timer churn rather than as loss. + fn cancel(&mut self, key: (Instant, u64)) { + if self.queue.remove(&key).is_some() { + self.metrics.timers_cancelled.add(1); + } + } + + /// Drop an armed timer a poll found already due. + fn fire_one(&mut self, key: (Instant, u64)) { + if self.queue.remove(&key).is_some() { + self.metrics.timers_fired.add(1); + } } } @@ -90,7 +114,7 @@ impl moq_net::runtime::Timer for Timer { fn set(&mut self, at: Option) { let mut heap = self.heap.borrow_mut(); if let Some(key) = self.slot.key.take() { - heap.remove(key); + heap.cancel(key); } self.slot.elapsed.set(false); if let Some(at) = at { @@ -112,7 +136,7 @@ impl moq_net::runtime::Timer for Timer { if at <= Instant::now() { // Fire eagerly rather than waiting for the sweep, and drop the // heap entry so the sweep doesn't wake anyone spuriously. - self.heap.borrow_mut().remove(self.slot.key.take().expect("armed")); + self.heap.borrow_mut().fire_one(self.slot.key.take().expect("armed")); self.slot.elapsed.set(true); return Poll::Ready(()); } @@ -124,7 +148,7 @@ impl moq_net::runtime::Timer for Timer { impl Drop for Timer { fn drop(&mut self) { if let Some(key) = self.slot.key.take() { - self.heap.borrow_mut().remove(key); + self.heap.borrow_mut().cancel(key); } } } diff --git a/rs/moq-uring/src/udp.rs b/rs/moq-uring/src/udp.rs index c7a9330281e..6068c86a9fd 100644 --- a/rs/moq-uring/src/udp.rs +++ b/rs/moq-uring/src/udp.rs @@ -41,6 +41,7 @@ use std::task::Poll; use io_uring::{cqueue, opcode, types}; use crate::Error; +use crate::metrics::Counters; use crate::shared::{Cqe, Op, Shared}; /// Space reserved for received control messages (`UDP_GRO` needs one int). @@ -304,6 +305,8 @@ struct Tx { bufs: Vec, free: Vec, waiters: kio::WaiterList, + /// Whether an acquisition is already waiting for the drained pool. + stalled: bool, /// Terminal failure, surfaced by `poll_acquire`. error: Option, } @@ -332,6 +335,9 @@ impl TxSlot { pub(crate) struct SockShared { io: UdpSocket, worker: Weak, + /// The worker's counters, held directly rather than reached through + /// `worker`, so counting a datagram is not a `Weak::upgrade`. + metrics: std::sync::Arc, config: Config, bgid: u16, closed: Cell, @@ -369,6 +375,7 @@ impl SockShared { let mut tx = self.tx.borrow_mut(); debug_assert_eq!(tx.bufs[id as usize].in_flight, 0); tx.free.push(id); + tx.stalled = false; tx.waiters.wake(); } @@ -383,6 +390,7 @@ impl SockShared { slot.in_flight -= 1; if slot.in_flight == 0 { tx.free.push(id); + tx.stalled = false; tx.waiters.wake(); } } @@ -520,12 +528,14 @@ impl Socket { bufs: (0..tx_count).map(|_| TxSlot::new(config.tx_buffer_len)).collect(), free: (0..tx_count).collect(), waiters: kio::WaiterList::new(), + stalled: false, error: None, }; let sock = Rc::new(SockShared { io, worker: Rc::downgrade(shared), + metrics: shared.metrics.clone(), config, bgid, closed: Cell::new(false), @@ -617,6 +627,10 @@ impl Socket { armed: false, })); } + if !tx.stalled { + tx.stalled = true; + self.shared.metrics.tx_stalls.add(1); + } waiter.register(&mut tx.waiters); Poll::Pending } @@ -794,6 +808,7 @@ impl TxBuf { send_one(&shared, &staging, index, unsafe { base.add(offset) }, chunk, to, None)?; } } + sock.metrics.tx_datagrams.add(segments as u64); Ok(()) } } @@ -923,6 +938,7 @@ fn send_one( shared.ops.borrow_mut().remove(key as usize); return Err(err); } + staging.sock.metrics.tx_sends.add(1); Ok(()) } @@ -949,6 +965,7 @@ pub(crate) fn arm_recv(shared: &Rc, sock: &Rc) { // receive would die on ENOBUFS immediately and re-arming here would // spin. if !rx.bufs.iter().any(|buf| !buf.kernel_done) { + sock.metrics.rx_exhausted.add(1); return; } let key = shared.insert(Op::Recv { @@ -969,6 +986,7 @@ pub(crate) fn arm_recv(shared: &Rc, sock: &Rc) { else { // Every buffer is borrowed and the pool is at its ceiling; a // release re-arms us. + sock.metrics.rx_exhausted.add(1); return; }; rx.bufs[bid as usize].claimed = true; @@ -1037,7 +1055,10 @@ pub(crate) fn on_recv( match code { // The receive pool is exhausted. Record it: by the time the re-arm // looks, a recycled buffer may hide that the kernel ran dry. - libc::ENOBUFS => sock.rx.borrow_mut().starved = true, + libc::ENOBUFS => { + sock.metrics.rx_enobufs.add(1); + sock.rx.borrow_mut().starved = true; + } // Socket teardown; nothing to surface. libc::ECANCELED => return, _ => { @@ -1055,6 +1076,12 @@ pub(crate) fn on_recv( }; match received { Ok((_, Some(queued))) => { + sock.metrics.rx_receives.add(1); + // A zero stride would be a kernel that reported a `UDP_GRO` size of + // zero; count the receive as one datagram rather than dividing by it. + sock.metrics + .rx_datagrams + .add(queued.len.div_ceil(queued.stride.max(1)) as u64); let mut rx = sock.rx.borrow_mut(); rx.bufs[queued.bid as usize].outstanding += 1; rx.queue.push_back(queued); diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index cb5046981b4..43c2302da4e 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -8,6 +8,7 @@ use std::time::{Duration, Instant}; use io_uring::{EnterFlags, IoUring, opcode, types}; +use crate::metrics::Metrics; use crate::park::{FUTEX_BITSET_MATCH_ANY, FUTEX2_PRIVATE, FUTEX2_SIZE_U32, PARKED, RUNNING, Unpark}; use crate::shared::{Cqe, Op, Shared, Task}; use crate::{Error, timer, udp}; @@ -38,12 +39,31 @@ const TEARDOWN_TIMEOUT: Duration = Duration::from_millis(3200); /// Worker construction knobs. /// -/// Currently empty: the worker sizes its ring internally, with a completion -/// queue that comfortably covers the per-socket pool ceilings in -/// [`udp::Config`]. The struct exists so future knobs stay additive. -#[derive(Clone, Debug, Default)] +/// The worker sizes its ring internally, with a completion queue that +/// comfortably covers the per-socket pool ceilings in [`udp::Config`]. +#[derive(Debug, Default)] #[non_exhaustive] -pub struct Config {} +pub struct Config { + /// Where the worker accumulates its counters. + /// + /// Default gives it a fresh set, still readable through + /// [`Handle::metrics`]. Pass one in to hold a copy on the thread that + /// spawned the worker, which is how an ops surface scrapes a worker it + /// cannot otherwise reach. A [`Metrics`] clone shares one set of counters, + /// so give each worker its own. Cloning this config starts a fresh set for + /// the cloned worker. + pub metrics: Metrics, +} + +impl Clone for Config { + fn clone(&self) -> Self { + // A cloned construction plan targets a new worker, so its counters must + // not be folded into the source worker's per-worker series. + Self { + metrics: Metrics::default(), + } + } +} /// A thread-pinned io_uring executor: the ring, a timer heap, and a local /// (`!Send`) task set, driven by a caller-owned loop. @@ -79,7 +99,8 @@ impl Worker { /// park timeout, and batched minimum waits with one code path; there is /// deliberately no fallback (use the tokio stack instead). pub fn new(config: Config) -> Result { - let Config {} = config; + let Config { metrics } = config; + let metrics = metrics.counters().clone(); let ring = IoUring::builder() .setup_single_issuer() .setup_defer_taskrun() @@ -113,9 +134,10 @@ impl Worker { shared: Rc::new(Shared { ring: RefCell::new(ring), ops: RefCell::new(slab::Slab::new()), - timers: Rc::new(RefCell::new(timer::Heap::default())), + timers: Rc::new(RefCell::new(timer::Heap::new(metrics.clone()))), spawns: RefCell::new(Vec::new()), - unpark: Unpark::new(), + unpark: Unpark::new(metrics.clone()), + metrics, next_bgid: std::cell::Cell::new(0), stopped: std::cell::Cell::new(false), spill: RefCell::new(std::collections::VecDeque::new()), @@ -197,6 +219,7 @@ impl Worker { flags: entry.flags(), })); } + self.shared.metrics.completions.add(self.cqes.len() as u64); if self.cqes.is_empty() || !self.dispatch_batch(deadline, Instant::now) { return Ok(()); } @@ -223,9 +246,13 @@ impl Worker { if ring.submission().is_empty() { return Ok(()); } + self.shared.metrics.enters.add(1); match ring.submit() { // A partial submit leaves the rest staged for the next pump. - Ok(_) => Ok(()), + Ok(count) => { + self.shared.metrics.submissions.add(count as u64); + Ok(()) + } // A signal interrupted the enter before it consumed anything. The // next worker turn retries the same staged SQEs. Err(err) if err.raw_os_error() == Some(libc::EINTR) => Ok(()), @@ -243,13 +270,14 @@ impl Worker { if ring.submission().is_empty() { return Ok(()); } + self.shared.metrics.enters.add(1); match ring.submit() { // Keep submitting after partial progress. Returning zero while SQEs // remain would otherwise spin forever. Ok(0) => { return Err(std::io::Error::other("io_uring teardown submission made no progress").into()); } - Ok(_) => {} + Ok(count) => self.shared.metrics.submissions.add(count as u64), Err(err) => retry_teardown_submit(&mut interruptions, err)?, } } @@ -275,6 +303,7 @@ impl Worker { let wait = remaining.min(std::time::Duration::from_millis(50)); let ts = types::Timespec::from(wait); let args = types::SubmitArgs::new().timespec(&ts); + self.shared.metrics.enters.add(1); let _ = ring.submitter().submit_with_args(1, &args); } } @@ -366,12 +395,14 @@ impl Worker { } let deadline = self.shared.timers.borrow().next(); + self.shared.metrics.parks.add(1); + self.shared.metrics.enters.add(1); let result = { let mut ring = self.shared.ring.borrow_mut(); let to_submit = ring.submission().len() as u32; let submitter = ring.submitter(); match deadline { - None => submitter.submit_and_wait(1).map(drop), + None => submitter.submit_and_wait(1), Some(at) => { // Zero timeout SQEs: the earliest userspace deadline rides // the enter call as an absolute CLOCK_MONOTONIC timeout. @@ -380,14 +411,17 @@ impl Worker { let flags = EnterFlags::GETEVENTS | EnterFlags::EXT_ARG | EnterFlags::ABS_TIMER; // SAFETY: `args` (and the timespec it references) outlive // the call, and EXT_ARG matches its type. - unsafe { submitter.enter(to_submit, 1, flags.bits(), Some(&args)) }.map(drop) + unsafe { submitter.enter(to_submit, 1, flags.bits(), Some(&args)) } } } }; unpark.word.store(RUNNING, Ordering::Release); match result { - Ok(()) => Ok(()), + Ok(count) => { + self.shared.metrics.submissions.add(count as u64); + Ok(()) + } Err(err) if matches!( err.raw_os_error(), @@ -459,6 +493,11 @@ impl Clone for Handle { } impl Handle { + /// This worker's counters, readable from any thread. + pub fn metrics(&self) -> Metrics { + Metrics::from_counters(self.shared.metrics.clone()) + } + /// Run a `!Send` future on this worker until completion. /// /// If the worker has already been dropped the future is dropped instead of @@ -572,7 +611,11 @@ mod tests { /// Kernel-gated: `None` (with a loud skip) below the 6.12 floor, so these /// tests pass vacuously on older CI kernels and run everywhere else. fn worker() -> Option { - match Worker::new(Config::default()) { + worker_with(Config::default()) + } + + fn worker_with(config: Config) -> Option { + match Worker::new(config) { Ok(worker) => Some(worker), Err(Error::Unsupported(reason)) => { eprintln!("skipping io_uring test: {reason}"); @@ -582,6 +625,16 @@ mod tests { } } + #[test] + fn cloned_config_has_fresh_metrics() { + let config = Config::default(); + let clone = config.clone(); + assert!(!std::sync::Arc::ptr_eq( + config.metrics.counters(), + clone.metrics.counters() + )); + } + #[test] fn ready_future() { let Some(mut worker) = worker() else { return }; @@ -980,9 +1033,189 @@ mod tests { drop(worker); } + /// The counters an ops scrape reads have to move for real work, and a + /// handed-in [`Metrics`] has to be the same set the worker writes: reading + /// zeros off a worker that is busy is indistinguishable from a healthy idle + /// one, which is the failure this whole surface exists to prevent. + #[test] + fn metrics_record_ring_and_socket_activity() { + let metrics = Metrics::default(); + let config = Config { + metrics: metrics.clone(), + ..Default::default() + }; + let Some(mut worker) = worker_with(config) else { return }; + let handle = worker.handle(); + let sock = handle + .udp( + std::net::UdpSocket::bind("127.0.0.1:0").expect("bind"), + udp::Config::default(), + ) + .expect("socket"); + let to = sock.local_addr().expect("addr"); + + // One GSO train of four datagrams: one `sendmsg`, four packets. + let Poll::Ready(Ok(mut tx)) = sock.poll_acquire(&kio::Waiter::noop()) else { + panic!("no tx buffer"); + }; + tx[..4 * 1200].fill(7); + tx.send(4 * 1200, to, 1200).expect("send"); + + // Drive the worker until the loopback delivers, parking on a timer each + // turn so the park and timer counters see traffic too. + let deadline = Instant::now() + Duration::from_secs(5); + let mut received = 0; + while received == 0 && Instant::now() < deadline { + let handle = handle.clone(); + worker + .block_on(async move { + Deadline::after(&handle, Duration::from_millis(10)).wait().await; + }) + .unwrap(); + while let Poll::Ready(packet) = sock.poll_recv(&kio::Waiter::noop()) { + let packet = packet.expect("receive path failed"); + received += packet.payload().len(); + } + } + assert!(received > 0, "the loopback never delivered the send"); + + let snap = metrics.snapshot(); + assert_eq!(snap.tx_sends, 1, "one GSO train is one sendmsg: {snap:?}"); + assert_eq!(snap.tx_datagrams, 4, "four segments: {snap:?}"); + assert!(snap.rx_receives > 0, "no receive completions: {snap:?}"); + assert!( + snap.rx_datagrams >= snap.rx_receives, + "fewer datagrams than receives: {snap:?}" + ); + assert!(snap.submissions > 0, "nothing was submitted: {snap:?}"); + assert!(snap.completions > 0, "nothing completed: {snap:?}"); + assert!(snap.enters > 0, "the ring was never entered: {snap:?}"); + assert!(snap.parks > 0, "the worker never parked: {snap:?}"); + assert!(snap.timers_fired > 0, "the park deadlines never fired: {snap:?}"); + // The worker's own handle reads the same counters as the one passed in, + // rather than a private set the scraper would never see. + let own = handle.metrics().snapshot(); + assert_eq!((own.tx_sends, own.tx_datagrams), (snap.tx_sends, snap.tx_datagrams)); + } + + /// The backpressure counters are the first thing to look at when throughput + /// sags, so both ends of it have to be recorded: a send that found the pool + /// drained, and a receive that could not be re-armed for want of a buffer. + #[test] + fn metrics_record_pool_backpressure() { + let metrics = Metrics::default(); + let config = Config { + metrics: metrics.clone(), + ..Default::default() + }; + let Some(mut worker) = worker_with(config) else { return }; + let handle = worker.handle(); + // One buffer each way, so the pools are at their ceiling immediately. + // Oneshot receives claim a whole buffer, which is what lets a held + // packet leave the socket unarmed. + let sock = handle + .udp( + std::net::UdpSocket::bind("127.0.0.1:0").expect("bind"), + udp::Config { + gro: false, + gso: false, + multishot: false, + rx_buffers_max: 1, + rx_buffer_len: 2048, + tx_buffers_max: 1, + tx_buffer_len: 2048, + }, + ) + .expect("socket"); + let to = sock.local_addr().expect("addr"); + + let Poll::Ready(Ok(tx)) = sock.poll_acquire(&kio::Waiter::noop()) else { + panic!("no tx buffer"); + }; + // The pool is one buffer deep and that one is checked out. + assert!(sock.poll_acquire(&kio::Waiter::noop()).is_pending()); + assert!(sock.poll_acquire(&kio::Waiter::noop()).is_pending()); + assert_eq!(metrics.snapshot().tx_stalls, 1); + tx.send(1200, to, 1200).expect("send"); + + // Hold the received packet: its buffer is the pool, so the re-arm has + // nowhere to receive into. + let deadline = Instant::now() + Duration::from_secs(5); + let mut held = None; + while held.is_none() && Instant::now() < deadline { + let handle = handle.clone(); + worker + .block_on(async move { + Deadline::after(&handle, Duration::from_millis(10)).wait().await; + }) + .unwrap(); + if let Poll::Ready(packet) = sock.poll_recv(&kio::Waiter::noop()) { + held = Some(packet.expect("receive path failed")); + } + } + assert!(held.is_some(), "the loopback never delivered the send"); + assert!( + metrics.snapshot().rx_exhausted > 0, + "a re-arm with every buffer held went unreported: {:?}", + metrics.snapshot() + ); + + // A completed send ends the first stall. Draining the pool again starts + // exactly one new episode, however often its waiter is polled. + let Poll::Ready(Ok(_tx)) = sock.poll_acquire(&kio::Waiter::noop()) else { + panic!("completed tx buffer was not released"); + }; + assert!(sock.poll_acquire(&kio::Waiter::noop()).is_pending()); + assert!(sock.poll_acquire(&kio::Waiter::noop()).is_pending()); + assert_eq!(metrics.snapshot().tx_stalls, 2); + } + + /// Timer churn is the thing #3122 needs a baseline for, so an arm, a + /// re-arm, and a drop each have to land in a different counter, and the + /// derived heap depth has to come back to zero. + #[test] + fn metrics_count_timer_churn() { + use moq_net::runtime::Timer as _; + let metrics = Metrics::default(); + let config = Config { + metrics: metrics.clone(), + ..Default::default() + }; + let Some(worker) = worker_with(config) else { return }; + let handle = worker.handle(); + let mut timer = handle.timer(); + + timer.set(Some(Instant::now() + Duration::from_secs(60))); + assert_eq!(metrics.snapshot().timers_active(), 1); + + // A re-arm is a cancel plus an arm, which is exactly the churn signal. + timer.set(Some(Instant::now() + Duration::from_secs(60))); + let snap = metrics.snapshot(); + assert_eq!((snap.timers_armed, snap.timers_cancelled, snap.timers_fired), (2, 1, 0)); + assert_eq!(snap.timers_active(), 1); + + // An eager poll past the deadline fires rather than cancels. + timer.set(Some(Instant::now() - Duration::from_millis(1))); + assert!(timer.poll(&kio::Waiter::noop()).is_ready()); + let snap = metrics.snapshot(); + assert_eq!((snap.timers_armed, snap.timers_cancelled, snap.timers_fired), (3, 2, 1)); + assert_eq!(snap.timers_active(), 0); + + // A dropped armed timer is a cancel, and the heap empties again. + timer.set(Some(Instant::now() + Duration::from_secs(60))); + assert_eq!(metrics.snapshot().timers_active(), 1); + drop(timer); + assert_eq!(metrics.snapshot().timers_active(), 0); + } + #[test] fn remote_wake_unparks() { - let Some(mut worker) = worker() else { return }; + let metrics = Metrics::default(); + let config = Config { + metrics: metrics.clone(), + ..Default::default() + }; + let Some(mut worker) = worker_with(config) else { return }; let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let thread_flag = flag.clone(); @@ -1009,5 +1242,8 @@ mod tests { .unwrap(); assert!(start.elapsed() >= Duration::from_millis(50)); thread.join().unwrap(); + // The futex syscall the wake had to make is the expensive half, and the + // only counter written from off the worker's thread. + assert!(metrics.snapshot().wakes > 0, "the remote wake went unreported"); } }