diff --git a/Cargo.lock b/Cargo.lock index b1f41440492..325314944ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4626,9 +4626,11 @@ version = "0.5.11" dependencies = [ "async-trait", "axum", + "bench-harness", "bytes", "futures", "hang", + "http-body", "humantime", "kio", "m3u8-rs", diff --git a/Cargo.toml b/Cargo.toml index 4d2fcff63a6..676da251713 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ hang = { version = "0.21.11", path = "rs/hang" } hashbrown = { version = "0.17.1", default-features = false } hashlink = "0.12.2" hex = "0.4" +http-body = "1.0.1" humantime = "2.3" humantime-serde = "1.1.1" hyper = "1.11" diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 3c862db0c6c..33f088adec9 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -56,7 +56,7 @@ What that means for you: - **The publisher declares the interval.** Only a broadcast whose catalog carries `media.join.interval` gets playlists. Rust publishers built on `moq-mux` (the CLI, the RTMP, SRT, and WebRTC gateways, and the native bindings) declare it: an encoder that knows its GOP passes it as a hint, and an ingest without one measures each track over its first second or group, whichever is longer, then declares the longest. `@moq/publish` declares its keyframe interval. - **Every rendition shares the grid.** A group spans from its first frame's timestamp to the next group's and belongs to the first grid line it contains, or, containing none, the line it starts after. Segment N therefore starts at the group containing N times the segment length, every rendition numbers its segments alike, and each cuts its boundaries at its own keyframes. A grid line no group starts a segment at, because a GOP ran past it or the content paused, is listed as an `EXT-X-GAP` so the numbering never skips: on a 2 s grid, a GOP from 0 to 7 s is segment 0, segments 1 and 2 are gaps, and the next GOP starts segment 3. Timestamps that go back start a discontinuity on that rendition alone; `EXT-X-PROGRAM-DATE-TIME`, written after every discontinuity when the catalog has a clock, lets players resync. -- **Idle renditions cost nothing**, and a segment is servable for as long as the relay's [cache](/bin/relay/config#cache) retains it. Keep `--window` (the playlist length, default 16 s) within that retention and within the media's max age (30 s by default), after which a superseded segment's groups are stale. +- **Contribution media starts on selection.** The HTTP multivariant playlist reads each root `contributions` entry's catalog and lists its audio and video beside the broadcast's own. A contribution shares the broadcast's timeline: its renditions are segmented on the broadcast's join interval and clock, whatever its own catalog declares, so publish it on the same timestamps. One not yet routed is listed once its broadcast is announced. Reading its catalog starts no encoding: only a request to one of its rendition URLs subscribes to that rendition's media. Demand lasts through active responses and, after the last one finishes, for three target durations or the playlist window plus one target duration, whichever is longer (20 s by default); a later request starts it again. The broadcast's own renditions stay subscribed. A segment is servable for as long as the relay's [cache](/bin/relay/config#cache) retains it. Keep `--window` (the playlist length, default 16 s) within that retention and within the media's max age (30 s by default), after which a superseded segment's groups are stale. - **Live joins cost the window.** The exporter subscribes 30 s behind the live edge and lists at most 256 segments. One joining partway through a segment lists nothing before the first segment it sees start. - **Segments are timed by the grid.** A segment is listed once a later one starts or the track ends. It starts at its first group's timestamp, or at its line when that group starts after it (as one resuming from a pause does), or a gap at its line, and lasts until the next segment starts, so a segment's bytes never change with a catalog estimate. A GOP longer than the segment is listed at its first line, so its duration understates its media. - **The target duration is twice the segment.** Every media playlist keeps one `EXT-X-TARGETDURATION` while its rendition stays in the catalog, even across a reconfigure: twice the segment length, rounded up to whole seconds (twice the interval for a GOP of a second or more, 2 s for 20 ms audio groups), since the GOP containing a grid line starts at most one interval before it, and a gap ends under two segment lengths after its line. Only a final GOP longer than that is listed past it, against RFC 8216's rule: hls.js plays it, but Apple's `mediastreamvalidator` flags it, so keep keyframe intervals within the declared interval. @@ -65,6 +65,15 @@ What that means for you: - **URLs name the stream.** Every URL the multivariant playlist and DASH manifest list sits under `@`, the stream's instance, so every edge and every reload derive the same URLs and a CDN in front of export caches them. A restarted publisher gets URLs of its own, whose numbering starts over. The multivariant playlist and manifest switch to it at once, while the old stream's playlists end at the segments they listed before the switch, even if its publisher keeps sending; a player already on it plays those out and then has to reload the multivariant playlist. A stream that reached export over moq-lite-06 or older, or moq-transport, has an instance its session minted, so only that export serves its URLs. - **Only a clean finish ends a playlist.** `EXT-X-ENDLIST` and a `static` DASH manifest mean the media tracks finished, or, on a replaced stream's media playlists, that another publisher took the broadcast over. A rendition whose media track fails instead, such as a publisher dropping its session, answers HTTP 500 with `Cache-Control: no-store`, so a truncated recording never looks complete, and leaves the multivariant playlist and DASH manifest so players switch to a sibling. With none left, the DASH manifest answers the error too. +Contribution URLs nest under the served broadcast's (see +[`Contribution`](https://docs.rs/moq-hls/latest/moq_hls/export/struct.Contribution.html)), so +access middleware sees the parent broadcast's path, while the server resolves each +[contribution](/draft/moq-hang#contributions) through its own scoped origin and leaves out +one it may not read. They follow contribution restarts unless the entry pins an `instance`; +an unavailable contribution leaves the multivariant playlist until it returns. DASH and +`--output` include only the broadcast's own renditions; record a contribution as its own +broadcast. + A [broadcast archive](/concept/archives) is served the same way, with no second stored copy: record a broadcast, republish it, and export the republished broadcast. Through this project's publishers and relays, diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 52aae272f54..c8cf35e9acd 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -244,7 +244,10 @@ Each key is a contribution ID. Each entry MUST contain a string `broadcast`, resolved against the catalog's broadcast as in {{field-broadcast}}, and MAY contain an `instance` as in {{field-instance}}. A consumer MUST reject a catalog with a malformed entry or a reference that escapes above the root. -Listing an entry MUST NOT subscribe to the contribution: a consumer opens its catalog only when it selects the contribution or one of its renditions, so a contribution produced on demand starts only for a viewer that wants it. +Listing an entry MUST NOT subscribe to the contribution: a consumer opens its catalog only when it selects the contribution or one of its renditions. +An HLS master selects every contribution it lists, reading their catalogs to describe their renditions, but MUST NOT subscribe to their media until requested by a player. +It segments a contribution's renditions on the listing catalog's `media` ({{media}}), whatever the contribution's own catalog declares. +A publisher producing media on demand MUST answer its catalog without starting media production. A selected contribution resolves like a rendition's reference, under the same rules for `instance` and restarts ({{field-instance}}). A contribution produced on demand has no stream until it resolves, so a publisher omits its `instance` until then. @@ -1125,6 +1128,7 @@ This document has no IANA actions. - Added the optional video `reorder` field: the most frames that can precede one in decode order and follow it in presentation, fixed for the track once declared. A consumer that authors decode times derives them from it and the current group alone, refusing deeper reordering; without it, a consumer infers them from timestamps. - Added an optional `instance` beside every rendition and data track `broadcast` reference ({{field-instance}}). A consumer reads each referenced path from the first broadcast it resolves for a catalog instance; a present `instance` fails a rendition that resolves another stream, and is best effort against an Instance a receiver minted. - Added the optional root `contributions` map ({{contributions}}): whole broadcasts named by a relative `broadcast` and an optional `instance`, opened only on selection, whose rendition IDs stay their own. +- An HLS master selects the contributions it lists by reading their catalogs, without demanding media, and segments them on the listing catalog's `media`; an on-demand publisher answers its catalog without starting media production. - Added optional `stats` and `echo` catalog sections, publisher/viewer snapshots keyed by rendition ID, and the `.echo` feedback broadcast convention. - An Opus frame timestamp counts the OpusHead pre-skip: a consumer presents each decoded sample the pre-skip earlier than its frame timestamp places it. diff --git a/js/hang/src/catalog/contribution.ts b/js/hang/src/catalog/contribution.ts index 6e0f7a858e4..68368a85793 100644 --- a/js/hang/src/catalog/contribution.ts +++ b/js/hang/src/catalog/contribution.ts @@ -5,8 +5,9 @@ import { InstanceSchema, RelativeBroadcastSchema } from "./path"; * A whole broadcast whose own catalog describes its renditions, listed without opening it. * * Listing one subscribes to nothing: a consumer opens the contribution's catalog only when it - * selects one of its renditions. Its rendition IDs belong to that broadcast, never the listing - * catalog. + * selects it, including an HLS master listing its renditions. An on-demand publisher answers + * the catalog without starting media. Its rendition IDs belong to that broadcast, never the + * listing catalog. */ export const ContributionSchema = z.object({ // The contribution's broadcast, relative to the broadcast serving this catalog. diff --git a/quest/m0/README.md b/quest/m0/README.md index 7126f4b487f..c7febcfd5ea 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -36,5 +36,4 @@ edge, which embeds moq-relay; it moved there on 2026-09-28. - [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - two moq-net tests stop failing when kio's waiters remember 16 lists, and the limit becomes a tuning knob worth 3.9% per delivered group - [Backport the relay and auth hardening to release](/quest/m0/backport-release-hardening.md) - `release`'s relay refuses dot-segment paths, serves its HTTP routes with main's response hardening, and withdraws advertisements a peer starves of send credit; its auth client refuses redirects - [Agent build output can't fill the desktop's disk](/quest/m0/agent-disk-budget.md) - finished agent checkouts reclaim their mbx targets automatically, so parallel agent builds stop filling the root disk -- [HLS lists contributions](/quest/m0/hls-contributions.md) - an HLS master lists each catalog contribution's renditions, and a contribution's media is subscribed only while players fetch it - [@moq/net's WebSocket receive windows match Rust](/quest/m0/qmux-receive-windows.md) - WebSocket JS throughput stops being capped below Rust's diff --git a/quest/m0/hls-contributions.md b/quest/m0/hls-contributions.md deleted file mode 100644 index 8efa9596907..00000000000 --- a/quest/m0/hls-contributions.md +++ /dev/null @@ -1,95 +0,0 @@ -# [L] HLS lists a broadcast's contributions without starting their media - -## Goal - -A moq-hls master playlist lists the audio and video renditions of each -contribution in the broadcast's catalog (the root `contributions` map) beside -the broadcast's own. It reads a contribution's catalog to list them, and -subscribes to a contribution rendition's media tracks only once a player -requests that rendition, so an on-demand contribution, such as a transcode -ladder, encodes only the rungs players fetch. Once no player fetches a -contribution rendition, its media subscription ends. - -## Plan - -**Reading the catalog is the listing (decided).** A master playlist variant -needs a `BANDWIDTH` and codecs, so moq-hls cannot list a contribution's -renditions without its catalog. The maintainer decided on 2026-11-02 that the -master reads each contribution's catalog when it is served: only the -contribution's media waits for a player's media playlist request, and a -catalog read starts no encoding. The master opens each contribution's catalog -while the broadcast is being served and follows it, but no media track. The -hang draft's Contributions section says a consumer opens a contribution's -catalog only when it selects it; an HLS master selects every contribution it -lists, so add that to the draft, and that a publisher producing media on -demand answers the catalog without starting media. `moq-transcode` already -does: only rung track demand starts an encode. - -**Follow each contribution's instances.** A contribution is its own broadcast -with its own catalog, so follow it the way the server follows a broadcast it -serves, not the way a rendition binds a sibling. `Source::bind` pins a path's -first answer and never resolves it again after a replacement -(`rs/moq-mux/src/source.rs`), and a rendition whose bound sibling ends ends -with it (`Media` in `rs/moq-hls/src/export/rendition.rs`). So resolve each -contribution entry with `Source::follow` (`moq_mux::follow::Instances`), -through the server's own origin, and build a `moq_mux::Source` for each -instance it hands out, so the instance's catalog and media reads stay pinned -to that instance. A new instance replaces the contribution's renditions and an -ended one drops them from the master until the next one starts. An entry -naming an `instance` follows only that stream, as the draft's -`instance` rules require. Unresolved, a contribution is listed once it -resolves; denied by the origin's scope, it is left out of the master and its -URLs answer `403`. Do not add a second resolver. - -**Segments line up with the broadcast's.** Players switch variants by -segment number, so a contribution's renditions share the broadcast's grid: -its catalog must declare the same `media.join.interval` on the same timeline. -A contribution declaring another interval or none is not HLS-renderable with -the broadcast and is left out of the master, as a rendition that cannot be -exported is today. - -**Lazy media.** A contribution rendition holds no media subscription until a -request reaches its URLs. The first one subscribes to that rendition's tracks -only, and its media playlist answers once it lists a segment. The -subscription ends once no request to the rendition is in progress and none has -finished for longer than a player reloading at the target duration would allow -(RFC 8216 6.3.4); a slow response, such as a first media playlist waiting on an -on-demand encode, keeps it. Pick the multiple and say why in the code, and a -later request starts it again. -Today's renditions keep their eager subscription: the broadcast's own media is -published anyway, so only contributions change. - -**URLs and names.** A contribution's rendition IDs belong to its own catalog, -so its URLs nest under the contribution ID inside the broadcast's instance -(for example `@{instance}/contribution/{id}/{kind}/{rendition}/media.m3u8`; -the implementation settles the spelling), and audio `NAME`s stay unique within -their group. A path-based gate in front of the server sees the broadcast's -path, so a viewer allowed the broadcast can play its contributions, as with a -sibling reference. Video variants join the broadcast's audio groups. - -**Out of scope, with reasons.** The DASH manifest keeps listing only the -broadcast's own renditions: a `$Time$` timeline needs every listed -representation's media, which defeats the lazy start. The recorder's -`renditions::Consumer` keeps yielding only the broadcast's own renditions: a -contribution is its own broadcast with its own recording. - -**Tests.** A master lists the broadcast's and a contribution's renditions -while the contribution's media tracks have no demand. A media playlist request -starts demand on that rendition's tracks only, a request still in progress -past the idle window keeps it, idle release ends it, and a later request starts -it again. A contribution that is unannounced at first is -listed once it resolves. A restarted contribution is followed onto its new instance, and one naming an -`instance` is not. A denied one is -left out and its URLs answer `403`. A mismatched or missing interval leaves the -contribution out. A contribution rendition sharing a name with one of the -broadcast's own gets distinct URLs and names. Unit tests mock time for the idle -release. - -Update `doc/bin/hls.md` and the hang draft's Contributions section. - -Public API: additive in moq-hls (the new URL routes). Wire: none; the draft -gains the HLS listing rule and the on-demand publisher's catalog answer. - -## Related - -- [The HLS server follows each path once](/quest/m2/hls-follow-once.md) - also changes how the server resolves a path diff --git a/quest/m1/catalog-include.md b/quest/m1/catalog-include.md index f76e5422d75..c34fa5c2529 100644 --- a/quest/m1/catalog-include.md +++ b/quest/m1/catalog-include.md @@ -30,12 +30,12 @@ the catalog root. Its why I wanted to call the section `include` originally." - The field arrived in [moq-dev/moq#7312](https://github.com/moq-dev/moq/pull/7312) (2026-11-01) and isn't on `release`, so no compatibility alias is needed and the catalog `version` stays. -- Update the [HLS contributions](/quest/m0/hls-contributions.md) quest, or its - code if #7836 has landed, to the new name. +- Update the HLS listing's code, landed in + [moq-dev/moq#7836](https://github.com/moq-dev/moq/pull/7836), to the new name. Public API: breaking but unreleased (Rust `hang`, `@moq/hang`, `@moq/watch`). Wire: the catalog JSON key changes; unreleased. ## Related -- [HLS lists contributions](/quest/m0/hls-contributions.md) - the HLS consumer settled with the shared-timeline rule +- [HLS lists contributions](https://github.com/moq-dev/moq/pull/7836) - the HLS consumer settled with the shared-timeline rule diff --git a/quest/m2/follow-refused-retry.md b/quest/m2/follow-refused-retry.md index c867c37b705..b01a7118d74 100644 --- a/quest/m2/follow-refused-retry.md +++ b/quest/m2/follow-refused-retry.md @@ -54,10 +54,6 @@ again. Public API: none. Wire: none. -## Required - -- [HLS lists a broadcast's contributions without starting their media](/quest/m0/hls-contributions.md) - its PR, [moq-dev/moq#7836](https://github.com/moq-dev/moq/pull/7836), adds the refused follower this extends - ## Related - [Listing and follow](/quest/m1/api-audit/route/events.md) - `follow` takes over instance changes from announce events diff --git a/rs/hang/src/catalog/contribution.rs b/rs/hang/src/catalog/contribution.rs index 6368914ef57..823c4a546a5 100644 --- a/rs/hang/src/catalog/contribution.rs +++ b/rs/hang/src/catalog/contribution.rs @@ -3,8 +3,9 @@ use serde::{Deserialize, Serialize}; /// A whole broadcast whose own catalog describes its renditions, listed without opening it. /// /// Listing one subscribes to nothing: a consumer opens the contribution's catalog only when it -/// selects one of its renditions. Its rendition IDs belong to that broadcast, never the listing -/// catalog. +/// selects it, including an HLS master listing its renditions. An on-demand publisher answers +/// the catalog without starting media. Its rendition IDs belong to that broadcast, never the +/// listing catalog. /// /// Marked `#[non_exhaustive]` so optional fields can be added later. External callers build one /// with [`Contribution::new`]. diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index b729f140184..bd0242b988f 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -14,13 +14,14 @@ categories = ["multimedia", "network-programming", "web-programming"] [features] default = ["server"] -server = ["dep:axum"] +server = ["dep:axum", "dep:http-body"] [dependencies] # Only needed by the HTTP export server (gated by `server`). axum = { workspace = true, optional = true } bytes = { workspace = true } hang = { workspace = true } +http-body = { workspace = true, optional = true } humantime = { workspace = true } kio = { workspace = true, features = ["runtime"] } m3u8-rs = "6" @@ -41,6 +42,7 @@ url = { workspace = true } [dev-dependencies] async-trait = { workspace = true } +criterion = { package = "bench-harness", path = "../bench-harness" } futures = { workspace = true } moq-archive = { path = "../moq-archive" } moq-json = { path = "../moq-json" } @@ -52,5 +54,10 @@ tower = { version = "0.5", features = ["util"] } tracing-test = { workspace = true } workspace-hack = { path = "../workspace-hack" } +[[bench]] +name = "contributions" +harness = false +required-features = ["server"] + [lints] workspace = true diff --git a/rs/moq-hls/MIGRATE.md b/rs/moq-hls/MIGRATE.md index 5df54fa42c5..a7dc51e8f70 100644 --- a/rs/moq-hls/MIGRATE.md +++ b/rs/moq-hls/MIGRATE.md @@ -4,6 +4,19 @@ How to update your code for each breaking change, newest first. ## Unreleased +### HTTP HLS lists contributions without starting their media + +`Server` follows the root catalog's contributions and lists their renditions from their +catalogs, on the root catalog's join interval and clock. Contribution media starts +only when requested and stops once responses finish and it idles for the longer of three +target durations and the playlist window plus one target duration. +Their URLs include both instances: +`@{instance}/contribution/{id}/{kind}/{rendition}/@{contribution-instance}/...`. +`Route::contribution` identifies that contribution; `Resource::instance` still names the +parent broadcast's instance. Embedders using `Server::respond` need no routing change. +DASH, `Broadcaster::new`, and the recorder cursor continue serving the broadcast's own +renditions only. + ### `Broadcaster::new` returns a cloneable handle and its `Driver` `Broadcaster::new(source, config)` returns `(Broadcaster, Driver)` instead of `Arc`. diff --git a/rs/moq-hls/benches/contributions.rs b/rs/moq-hls/benches/contributions.rs new file mode 100644 index 00000000000..c8c22bdb77b --- /dev/null +++ b/rs/moq-hls/benches/contributions.rs @@ -0,0 +1,104 @@ +//! Master rendering and one contribution lookup, swept over contributions x renditions. +//! The catalogs have no media, so setup also verifies that listing does not need an encode. + +use std::hint::black_box; +use std::time::Duration; + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use hang::catalog::{Contribution, Join, VideoCodec, VideoConfig}; +use moq_hls::{Server, export::Route}; +use moq_mux::catalog::hang::Catalog; + +fn publish_catalog( + broadcast: &moq_net::origin::Publication, + catalog: &Catalog, +) -> moq_json::snapshot::Producer { + let track = broadcast + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut producer = moq_json::snapshot::Producer::new(track, Default::default()); + producer + .update(moq_net::Timed::from(catalog).at(moq_net::Timestamp::ZERO)) + .unwrap(); + producer +} + +fn contributions(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("hls/contributions"); + for count in [1, 8, 64] { + for renditions in [1, 4, 16] { + let (server, broadcaster, route, _publications, _catalogs, _origin) = runtime.block_on(async { + let origin = moq_tokio::origin::spawn(); + let mut root = Catalog::default(); + root.media.join = Join::new(Duration::from_secs(2)); + let mut publications = Vec::new(); + let mut catalogs = Vec::new(); + let mut first = None; + for index in 0..count { + let id = format!("c{index}"); + let instance = moq_net::Instance::random(); + first.get_or_insert(instance.clone()); + let broadcast = origin.create_broadcast(format!("room/{id}")).unwrap(); + let mut catalog = Catalog::default(); + catalog.media = root.media; + for rendition in 0..renditions { + let mut video = VideoConfig::new(VideoCodec::VP8); + video.coded_width = Some(320); + video.coded_height = Some(240); + catalog.video.renditions.insert(format!("v{rendition}"), video); + } + catalogs.push(publish_catalog(&broadcast, &catalog)); + broadcast + .announce(moq_net::origin::Route::default().with_instance(instance, 0)) + .unwrap(); + publications.push(broadcast); + root.contributions.insert( + id.clone(), + Contribution::new(moq_net::path::Relative::new(&format!("./{id}")).unwrap()), + ); + } + let parent = moq_net::Instance::random(); + let broadcast = origin.create_broadcast("room/live").unwrap(); + catalogs.push(publish_catalog(&broadcast, &root)); + broadcast + .announce(moq_net::origin::Route::default().with_instance(parent.clone(), 0)) + .unwrap(); + publications.push(broadcast); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let broadcaster = server.broadcaster("room/live").await.unwrap(); + kio::clock::timeout(Duration::from_secs(5), async { + while broadcaster.master_playlist(None).matches("#EXT-X-STREAM-INF:").count() != count * renditions + { + kio::clock::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("every catalog is listed without media"); + let route = Route::parse(&format!( + "/room/live/@{parent}/contribution/c0/video/missing/@{}/media.m3u8", + first.unwrap() + )) + .unwrap(); + (server, broadcaster, route, publications, catalogs, origin) + }); + let shape = format!("{count}c_{renditions}r"); + group.bench_function(BenchmarkId::new("master", &shape), |b| { + b.iter(|| black_box(broadcaster.master_playlist(None))) + }); + group.bench_function(BenchmarkId::new("lookup", &shape), |b| { + b.iter(|| black_box(runtime.block_on(server.respond(&route, None)))) + }); + // Let cancelled driver tasks release their subscriptions before the next shape. + drop((server, broadcaster, _publications, _catalogs, _origin)); + runtime.block_on(tokio::task::yield_now()); + } + } + group.finish(); +} + +criterion_group!(benches, contributions); +criterion_main!(benches); diff --git a/rs/moq-hls/src/export/contributions.rs b/rs/moq-hls/src/export/contributions.rs new file mode 100644 index 00000000000..13ee050ea1c --- /dev/null +++ b/rs/moq-hls/src/export/contributions.rs @@ -0,0 +1,364 @@ +//! Catalog-only contribution discovery for the HTTP HLS master. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::task::Poll; +#[cfg(feature = "server")] +use std::task::ready; +use std::time::Duration; + +use hang::catalog::{Contribution, Media}; +use moq_mux::{catalog::hang::Catalog, follow}; + +#[cfg(feature = "server")] +use super::Rendition; +use super::{Kind, driver, master, renditions}; + +#[derive(Clone)] +pub(super) struct Producer { + state: kio::Producer, + context: Context, +} + +#[derive(Clone)] +struct Context { + origin: moq_net::origin::Consumer, + window: Duration, + spawner: driver::Spawner, + /// The root catalog's `media`. A contribution is included in the root broadcast, so its + /// renditions share the root's timeline and segment on the root's grid. + root: kio::Producer, +} + +#[derive(Default)] +struct State { + entries: BTreeMap, +} + +struct Entry { + reference: Contribution, + current: kio::Producer, + _owner: Option, +} + +enum Current { + Pending, + Offline, + Online(Arc), + Failed( + #[cfg_attr( + not(feature = "server"), + allow(dead_code, reason = "only the `server` feature answers a lookup") + )] + crate::Error, + ), +} + +struct Instance { + id: String, + renditions: renditions::Producer, + _owner: driver::Owner, +} + +impl Drop for Instance { + fn drop(&mut self) { + self.renditions.cut(); + self.renditions.clear(); + } +} + +impl Producer { + pub fn new(origin: moq_net::origin::Consumer, window: Duration, spawner: driver::Spawner) -> Self { + Self { + state: kio::Producer::default(), + context: Context { + origin, + window, + spawner, + root: kio::Producer::default(), + }, + } + } + + pub fn sync(&self, source: &moq_mux::Source, catalog: &Catalog) { + let Ok(mut state) = self.state.write() else { return }; + // Republished estimates leave the root media as is, so only a change wakes the children. + if *self.context.root.read() != catalog.media + && let Ok(mut root) = self.context.root.write() + { + *root = catalog.media; + } + state + .entries + .retain(|id, entry| catalog.contributions.get(id) == Some(&entry.reference)); + for (id, reference) in &catalog.contributions { + if state.entries.contains_key(id) || !master::is_path_segment(id) { + continue; + } + let current = kio::Producer::new(Current::Pending); + let owner = match source.follow(Some(&reference.broadcast), reference.instance.as_ref()) { + Ok(instances) => Some(self.context.spawner.spawn(follow_instances( + instances, + current.clone(), + self.context.clone(), + ))), + Err(err) => { + *current.write().unwrap_or_else(|_| unreachable!("new entry is open")) = + Current::Failed(err.into()); + None + } + }; + state.entries.insert( + id.clone(), + Entry { + reference: reference.clone(), + current, + _owner: owner, + }, + ); + } + } + + /// Stop following every contribution for good. Closing the state under the same lock + /// makes this terminal: a [`sync`](Self::sync) racing past its catalog await finds the + /// state closed and cannot re-add entries or spawn follow tasks. + pub fn clear(&self) { + if let Ok(mut state) = self.state.write() { + state.entries.clear(); + state.close(); + } + let _ = self.context.root.close(); + } + + pub fn poll_advertised(&self, waiter: &kio::Waiter) -> Poll<()> { + self.state + .poll_ref(waiter, |state| { + for entry in state.entries.values() { + let ready = entry.current.poll_ref(waiter, |current| { + if let Current::Online(instance) = &**current { + // The set's changes wake this; each rendition's binding wakes it as it resolves. + if instance.renditions.poll_advertised(waiter).is_pending() { + return Poll::Pending; + } + let listable = instance.renditions.snapshot().iter().any(|rendition| { + rendition.is_enabled() + && rendition.poll_resolved(waiter) == Poll::Ready(true) + && rendition.poll_refused(waiter).is_pending() + }); + if listable { + return Poll::Ready(()); + } + } + Poll::Pending + }); + if matches!(ready, Poll::Ready(Ok(()))) { + return Poll::Ready(()); + } + } + Poll::Pending + }) + .map(|_| ()) + } + + pub fn variants( + &self, + parent: &str, + query: Option<&str>, + ) -> (Vec, Vec) { + let state = self.state.read(); + let mut video = Vec::new(); + let mut audio = Vec::new(); + for (id, entry) in &state.entries { + let current = entry.current.read(); + let Current::Online(instance) = &*current else { continue }; + for rendition in instance.renditions.snapshot() { + // A rendition whose binding is still resolving may yet be denied, so it waits; one + // whose binding failed has no media to serve. + let noop = kio::Waiter::noop(); + if !rendition.is_enabled() + || rendition.poll_resolved(&noop) != Poll::Ready(true) + || rendition.poll_refused(&noop).is_ready() + { + continue; + } + let uri = master::contribution_uri(parent, id, &instance.id, &rendition, query); + match rendition.kind { + Kind::Video => video.push(master::VideoVariant { + uri, + bandwidth: rendition.bandwidth(), + width: rendition.width, + height: rendition.height, + codec: rendition.codec.clone(), + }), + Kind::Audio => audio.push(master::AudioVariant { + // Qualify the ID with its catalog; the renderer resolves any remaining NAME collision. + name: format!("{id}/{}", rendition.name), + uri, + bandwidth: rendition.bandwidth(), + codec: rendition.codec.clone(), + }), + } + } + } + (video, audio) + } + + #[cfg(feature = "server")] + pub async fn rendition( + &self, + at: &super::route::Contribution, + kind: Kind, + name: &str, + ) -> crate::Result>> { + kio::wait(|waiter| self.poll_rendition(waiter, at, kind, name)).await + } + + #[cfg(feature = "server")] + fn poll_rendition( + &self, + waiter: &kio::Waiter, + at: &super::route::Contribution, + kind: Kind, + name: &str, + ) -> Poll>>> { + let state = self.state.read(); + let Some(entry) = state.entries.get(&at.id) else { + return Poll::Ready(Ok(None)); + }; + match entry.current.poll_ref(waiter, |current| { + let instance = match &**current { + Current::Pending => return Poll::Pending, + Current::Failed(err) => return Poll::Ready(Err(err.clone())), + Current::Offline => return Poll::Ready(Ok(None)), + Current::Online(instance) if instance.id != at.instance => return Poll::Ready(Ok(None)), + Current::Online(instance) => instance, + }; + Poll::Ready(Ok(ready!(instance.renditions.poll_get(waiter, kind, name)))) + }) { + Poll::Ready(Ok(result)) => Poll::Ready(result), + Poll::Ready(Err(_)) => Poll::Ready(Ok(None)), + Poll::Pending => Poll::Pending, + } + } +} + +async fn follow_instances(mut instances: follow::Instances, current: kio::Producer, context: Context) { + loop { + let next = instances.next().await; + let next = match next { + Ok(follow::Event::Start(broadcast)) => { + let id = broadcast + .info() + .instance + .clone() + .unwrap_or_else(moq_net::Instance::random) + .to_string(); + let source = moq_mux::Source::new(context.origin.clone(), broadcast); + let renditions = renditions::Producer::lazy(context.window, context.spawner.clone()); + let owner = context.spawner.lead(super::watch_catalog( + source, + renditions.clone(), + super::Role::Contribution(context.root.consume()), + )); + Current::Online(Arc::new(Instance { + id, + renditions, + _owner: owner, + })) + } + Ok(follow::Event::End) => Current::Offline, + // An authorization refusal is final: the contribution stays out and its URLs answer 403. + Err(err @ moq_mux::Error::Moq(moq_net::Error::Unauthorized)) => { + tracing::debug!(%err, "contribution refused"); + let Ok(mut state) = current.write() else { return }; + *state = Current::Failed(err.into()); + return; + } + Err(moq_mux::Error::Moq(moq_net::Error::Closed)) => { + let Ok(mut state) = current.write() else { return }; + *state = Current::Offline; + return; + } + // Not routed yet, or refused otherwise: wait for the path's next announcement, which + // the follower hands out, however long it takes. Until then nothing is listed. + Err(err) => { + tracing::debug!(%err, "contribution not routed; waiting for its next announcement"); + match &*current.read() { + Current::Pending => Current::Offline, + _ => continue, + } + } + }; + let Ok(mut state) = current.write() else { return }; + *state = next; + } +} + +#[cfg(all(test, feature = "server"))] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn a_sync_after_clear_adds_nothing() { + let origin = moq_tokio::origin::spawn(); + let _parent = origin.publish("room/live", Default::default()).unwrap(); + let source = crate::resolved(&origin, "room/live").await; + let mut catalog = Catalog::default(); + catalog.contributions.insert( + "child".into(), + Contribution::new(moq_net::path::Relative::new("./child").unwrap()), + ); + let (spawner, _driver) = driver::Spawner::new(); + let contributions = Producer::new(origin.consume(), Duration::from_secs(16), spawner); + contributions.clear(); + // A watcher already past its catalog await syncs after the clear. + contributions.sync(&source, &catalog); + assert!(contributions.state.read().entries.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn a_cold_lookup_waits_for_the_first_follow_event() { + let origin = moq_tokio::origin::spawn(); + let _parent = origin.publish("room/live", Default::default()).unwrap(); + let child_id = moq_net::Instance::random(); + let child = origin + .publish( + "room/child", + moq_net::origin::Route::default().with_instance(child_id.clone(), 0), + ) + .unwrap(); + let source = crate::resolved(&origin, "room/live").await; + let mut catalog = Catalog::default(); + catalog.media.join = hang::catalog::Join::new(Duration::from_secs(2)); + let mut video = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + video.coded_width = Some(320); + video.coded_height = Some(240); + catalog.video.renditions.insert("v".into(), video); + let track = child + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut json = moq_json::snapshot::Producer::new(track, Default::default()); + json.update(moq_net::Timed::from(&catalog).at(moq_net::Timestamp::ZERO)) + .unwrap(); + catalog.video.renditions.clear(); + catalog.contributions.insert( + "child".into(), + Contribution::new(moq_net::path::Relative::new("./child").unwrap()), + ); + let (spawner, driver) = driver::Spawner::new(); + let contributions = Producer::new(origin.consume(), Duration::from_secs(16), spawner); + contributions.sync(&source, &catalog); + let at = super::super::Contribution { + id: "child".into(), + instance: child_id.to_string(), + }; + let mut lookup = Box::pin(contributions.rendition(&at, Kind::Video, "v")); + // The catalog has been read, but the driver has not yet polled its follow task. + assert!(futures::poll!(lookup.as_mut()).is_pending()); + let running = tokio::spawn(driver.run()); + let rendition = lookup.await.unwrap().unwrap(); + assert_eq!(rendition.name, "v"); + drop((rendition, contributions)); + running.await.unwrap(); + } +} diff --git a/rs/moq-hls/src/export/driver.rs b/rs/moq-hls/src/export/driver.rs index 84832495719..262786421fb 100644 --- a/rs/moq-hls/src/export/driver.rs +++ b/rs/moq-hls/src/export/driver.rs @@ -14,10 +14,11 @@ use std::task::Poll; /// Dropping it early stops all of that work. #[must_use = "the broadcaster does nothing until its driver runs"] pub struct Driver { - /// The catalog watcher, polled before the other tasks in every pass: a reconfigure must - /// retire its rendition before that rendition's watcher reads groups published after it. - lead: Option, - inbox: kio::Queue, + /// The catalog watchers, the broadcast's and each contribution's, polled before the other + /// tasks in every pass: a reconfigure must retire its rendition before that rendition's + /// watcher reads groups published after it. + leads: kio::Tasks, + inbox: kio::Queue<(Lane, Job)>, tasks: kio::Tasks, /// The inbox closed: every owner that could start a task has dropped. closed: bool, @@ -26,29 +27,26 @@ pub struct Driver { impl Driver { /// Advance every task, ready once nothing is left to run. pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { - if let Some(lead) = &mut self.lead - && kio::Task::poll(lead, waiter).is_ready() - { - self.lead = None; - } while !self.closed { match self.inbox.poll_pop(waiter) { - Poll::Ready(Ok(job)) => self.tasks.push(job), + Poll::Ready(Ok((Lane::Lead, job))) => self.leads.push(job), + Poll::Ready(Ok((Lane::Other, job))) => self.tasks.push(job), Poll::Ready(Err(kio::Closed)) => self.closed = true, Poll::Pending => break, } } + let leads = self.leads.poll(waiter); match self.tasks.poll(waiter) { - Poll::Ready(()) if self.closed && self.lead.is_none() => Poll::Ready(()), + Poll::Ready(()) if self.closed && leads.is_ready() => Poll::Ready(()), _ => Poll::Pending, } } - /// Run `task` ahead of every other task (see [`lead`](Self::lead)), until it finishes or the - /// returned [`Owner`] drops. + /// Run `task` ahead of every other task (see [`leads`](Self::leads)), until it finishes or + /// the returned [`Owner`] drops. pub(crate) fn lead(&mut self, task: impl Future + Send + 'static) -> Owner { let (job, owner) = Job::new(task); - self.lead = Some(job); + self.leads.push(job); owner } @@ -73,7 +71,13 @@ pub(crate) struct Spawner { inbox: Arc, } -struct Inbox(kio::Queue); +/// Whether a task runs among the driver's leads or after them. +enum Lane { + Lead, + Other, +} + +struct Inbox(kio::Queue<(Lane, Job)>); impl Drop for Inbox { fn drop(&mut self) { @@ -86,7 +90,7 @@ impl Spawner { pub fn new() -> (Self, Driver) { let inbox = kio::Queue::new(); let driver = Driver { - lead: None, + leads: kio::Tasks::new(), inbox: inbox.clone(), tasks: kio::Tasks::new(), closed: false, @@ -101,9 +105,19 @@ impl Spawner { /// Run `task` on the driver until it finishes or the returned [`Owner`] drops. pub fn spawn(&self, task: impl Future + Send + 'static) -> Owner { + self.start(Lane::Other, task) + } + + /// [`spawn`](Self::spawn) `task` among the driver's leads, polled before every other task, + /// as a catalog watcher must be. + pub fn lead(&self, task: impl Future + Send + 'static) -> Owner { + self.start(Lane::Lead, task) + } + + fn start(&self, lane: Lane, task: impl Future + Send + 'static) -> Owner { let (job, owner) = Job::new(task); // Refused once the driver is gone, which drops the task unrun. - let _ = self.inbox.0.try_push(job); + let _ = self.inbox.0.try_push((lane, job)); owner } } @@ -143,6 +157,25 @@ impl kio::Task for Job { mod tests { use super::*; + /// A lead a spawner starts, such as a contribution's catalog watcher, runs before the tasks + /// started ahead of it. + #[test] + fn spawned_leads_run_first() { + let (spawner, mut driver) = Spawner::new(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let record = |name: &'static str| { + let order = order.clone(); + async move { + order.lock().unwrap().push(name); + std::future::pending::<()>().await; + } + }; + let _other = spawner.spawn(record("other")); + let _lead = spawner.lead(record("lead")); + assert!(driver.poll(&kio::Waiter::noop()).is_pending()); + assert_eq!(*order.lock().unwrap(), ["lead", "other"]); + } + // The driver finishes once every spawner and task is gone, and an owner's drop ends its task // without waiting for the task to finish on its own. #[test] diff --git a/rs/moq-hls/src/export/master.rs b/rs/moq-hls/src/export/master.rs index 937bc1add30..0aa107bef89 100644 --- a/rs/moq-hls/src/export/master.rs +++ b/rs/moq-hls/src/export/master.rs @@ -46,6 +46,27 @@ pub(crate) fn rendition_uri(instance: &str, kind: Kind, name: &str, query: Optio uri } +pub(crate) fn contribution_uri( + parent: &str, + id: &str, + instance: &str, + rendition: &super::Rendition, + query: Option<&str>, +) -> String { + let mut uri = format!( + "@{}/contribution/{}/{}/{}/@{}/media.m3u8", + utf8_percent_encode(parent, PATH_SEGMENT), + utf8_percent_encode(id, PATH_SEGMENT), + rendition.kind.as_str(), + utf8_percent_encode(&rendition.name, PATH_SEGMENT), + utf8_percent_encode(instance, PATH_SEGMENT) + ); + if let Some(query) = query { + let _ = write!(uri, "?{query}"); + } + uri +} + fn quoted_string(value: &str) -> String { let mut quoted = String::with_capacity(value.len()); for character in value.chars() { @@ -163,9 +184,13 @@ pub fn render(video: &[VideoVariant], audio: &[AudioVariant]) -> String { let audio_groups = group_audio(audio); for group in &audio_groups { + let mut names = std::collections::BTreeSet::new(); for (index, variant) in group.variants.iter().enumerate() { let default = if index == 0 { "YES" } else { "NO" }; - let name = quoted_string(&variant.name); + let mut name = quoted_string(&variant.name); + while !names.insert(name.clone()) { + name.push('~'); + } let _ = writeln!( out, "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"{}\",NAME=\"{}\",DEFAULT={default},AUTOSELECT=YES,URI=\"{}\"", @@ -315,6 +340,7 @@ mod tests { Route::parse(url.path()).unwrap(), Route { broadcast: "live".into(), + contribution: None, resource: Resource::Media { instance: instance.into(), kind: Kind::Video, @@ -338,6 +364,26 @@ mod tests { assert!(!out.contains("\n#EXT-X-INJECT")); } + #[test] + fn audio_names_stay_unique_after_quoting_and_catalog_qualification() { + let out = render( + &[], + &[ + audio("guest/main", 128_000, "opus"), + audio("guest/main", 128_000, "opus"), + audio("a\"", 128_000, "opus"), + audio("a%22", 128_000, "opus"), + ], + ); + let names: Vec<_> = out + .lines() + .filter(|line| line.starts_with("#EXT-X-MEDIA:")) + .map(|line| line.split("NAME=\"").nth(1).unwrap().split('"').next().unwrap()) + .collect(); + assert_eq!(names.len(), 4); + assert_eq!(names.iter().collect::>().len(), 4); + } + #[test] fn codecs_and_uri_lines_cannot_inject() { let mut hd = video("hd", None, None); diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 0b2cbeaf19e..c3df8030db7 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -32,6 +32,7 @@ #[cfg(test)] mod archive_tests; +mod contributions; mod driver; mod grid; pub mod master; @@ -51,8 +52,10 @@ use rand::RngExt; pub use driver::Driver; pub(crate) use playlist::render_media; +#[cfg(feature = "server")] +pub(crate) use rendition::Demand; pub use rendition::{Kind, Rendition}; -pub use route::{Resource, Route}; +pub use route::{Contribution, Resource, Route}; /// Backoff bounds for the initial catalog subscription. /// @@ -112,6 +115,7 @@ pub struct Broadcaster { instance: Arc, /// The current rendition set, reconciled from the catalog by the watcher task. renditions: renditions::Producer, + contributions: Option, /// Shared by every clone; its drop with the last one ends the catalog watcher. _guard: Arc, } @@ -119,6 +123,7 @@ pub struct Broadcaster { /// What the last [`Broadcaster`] clone releases. struct Guard { renditions: renditions::Producer, + contributions: Option, /// Ends the catalog watcher when dropped, or when [`replaced`](Broadcaster::replaced) takes it. watcher: Mutex>, } @@ -130,22 +135,29 @@ impl Drop for Guard { // media flowing, so its final segments are not truncated. Each rendition stops // following its media once no recording cursor holds it. self.renditions.clear(); + if let Some(contributions) = &self.contributions { + contributions.clear(); + } } } impl Broadcaster { - /// Start tracking the renditions of `source`'s catalog broadcast, once the returned + /// Start tracking the own renditions of `source`'s catalog broadcast, once the returned /// [`Driver`] runs. /// /// Fails with [`Error::UnsupportedCpu`](crate::Error::UnsupportedCpu) on a CPU its /// cryptography cannot run on. pub fn new(source: moq_mux::Source, config: Config) -> crate::Result<(Self, Driver)> { rind::cpu::check().map_err(crate::Error::unsupported_cpu)?; - Ok(Self::start(source, config)) + Ok(Self::start(source, config, None)) } /// [`new`](Self::new) for a caller that already checked the CPU. - pub(crate) fn start(source: moq_mux::Source, config: Config) -> (Self, Driver) { + pub(crate) fn start( + source: moq_mux::Source, + config: Config, + origin: Option, + ) -> (Self, Driver) { let broadcast = source.broadcast(); // An instance names the stream on every edge, so each serves it under the same URLs. let instance = broadcast @@ -154,18 +166,25 @@ impl Broadcaster { .clone() .unwrap_or_else(moq_net::Instance::random); let (spawner, mut driver) = driver::Spawner::new(); + let contributions = origin.map(|origin| contributions::Producer::new(origin, config.window, spawner.clone())); let renditions = renditions::Producer::new(config.window, spawner); // The watcher owns its own producer clone; dropping the `Broadcaster` ends it so the // standing catalog subscription stops when nobody's serving from this broadcaster. - let watcher = driver.lead(watch_catalog(source, renditions.clone())); + let watcher = driver.lead(watch_catalog( + source, + renditions.clone(), + Role::Root(contributions.clone().map(Box::new)), + )); let broadcaster = Self { broadcast, instance: Arc::from(instance.as_str()), _guard: Arc::new(Guard { renditions: renditions.clone(), + contributions: contributions.clone(), watcher: Mutex::new(Some(watcher)), }), renditions, + contributions, }; (broadcaster, driver) } @@ -248,7 +267,20 @@ impl Broadcaster { allow(dead_code, reason = "only the `server` feature uses it") )] pub(crate) async fn advertised(&self) { - self.renditions.advertised().await; + kio::wait(|waiter| { + let own = self.renditions.poll_advertised(waiter); + let contribution = self + .contributions + .as_ref() + .is_some_and(|contributions| contributions.poll_advertised(waiter).is_ready()); + // Contributions alone answer only while no own rendition could still be listed. + if own.is_ready() || (!self.renditions.listable() && contribution) { + std::task::Poll::Ready(()) + } else { + std::task::Poll::Pending + } + }) + .await; } /// Version every segment URL with `generation`, a label for the publisher run the @@ -283,7 +315,11 @@ impl Broadcaster { /// holding different tokens; a credential in `Config` would embed one viewer's in /// another's playlist. /// - /// Lists only enabled renditions a player can start, i.e. ones listing at least one segment, + /// A broadcaster obtained from `Server` also lists contribution renditions from their + /// catalogs, starting their media only on request. `Broadcaster::new` exports the own + /// renditions only, so a recorder never demands contributions. + /// + /// Lists only enabled own renditions a player can start, i.e. ones listing at least one segment, /// so an early render may list audio alone, or nothing at all. A disabled rendition (catalog `enabled: false`) is left out until /// it re-enables, while its media playlist keeps serving. pub fn master_playlist(&self, query: Option<&str>) -> String { @@ -315,9 +351,28 @@ impl Broadcaster { }), } } + if let Some(contributions) = &self.contributions { + let (extra_video, extra_audio) = contributions.variants(&self.instance, query); + video.extend(extra_video); + audio.extend(extra_audio); + } (video, audio) } + #[cfg(feature = "server")] + pub(crate) async fn contribution( + &self, + at: &Contribution, + kind: Kind, + name: &str, + ) -> crate::Result>> { + self.renditions.catalogued().await; + match &self.contributions { + Some(contributions) => contributions.rendition(at, kind, name).await, + None => Ok(None), + } + } + /// Render the DASH manifest (MPD) from the current renditions' segments, or `None` while nothing is listable yet (every `SegmentTimeline` /// would be empty, which players reject). /// @@ -449,6 +504,9 @@ impl Broadcaster { let watcher = self._guard.watcher.lock().unwrap().take(); drop(watcher); self.renditions.cut(); + if let Some(contributions) = &self.contributions { + contributions.clear(); + } } /// Whether the current catalog contains no servable renditions (serve path). @@ -461,7 +519,16 @@ impl Broadcaster { } } -async fn watch_catalog(source: moq_mux::Source, renditions: renditions::Producer) { +/// What a catalog watcher reconciles besides its own rendition set. +enum Role { + /// A root broadcast, following its catalog's contributions when an HTTP server serves it. + Root(Option>), + /// A contribution included in a root broadcast. Its renditions take the root catalog's + /// `media`, so they share the root's timeline and grid whatever the contribution declares. + Contribution(kio::Consumer), +} + +async fn watch_catalog(source: moq_mux::Source, renditions: renditions::Producer, role: Role) { // The source's catalog broadcast, so the catalog, the media subscriptions, and the closed // check below all refer to the one broadcast it was built from. let broadcast = &source.broadcast(); @@ -484,17 +551,50 @@ async fn watch_catalog(source: moq_mux::Source, renditions: renditions::Producer } }; + // A contribution's latest catalog, re-synced when the root's media changes. + let mut latest: Option = None; let result = loop { - match kio::wait(|waiter| consumer.poll_next(waiter)).await { - Ok(Some(catalog)) => renditions.sync(&source, &catalog), - Ok(None) => break Ok(()), - Err(err) => { + let next = kio::wait(|waiter| { + if let std::task::Poll::Ready(next) = consumer.poll_next(waiter) { + return std::task::Poll::Ready(Some(next)); + } + if let (Role::Contribution(root), Some(catalog)) = (&role, &latest) + && let std::task::Poll::Ready(Ok(())) = root.poll(waiter, |media| match **media != catalog.media { + true => std::task::Poll::Ready(()), + false => std::task::Poll::Pending, + }) { + return std::task::Poll::Ready(None); + } + std::task::Poll::Pending + }) + .await; + let mut catalog = match next { + Some(Ok(Some(catalog))) => catalog, + Some(Ok(None)) => break Ok(()), + Some(Err(err)) => { tracing::warn!(%err, "broadcast catalog stream ended with error"); break Err(err.into()); } + None => latest.take().expect("a root media change follows a catalog"), + }; + match &role { + Role::Root(Some(contributions)) => contributions.sync(&source, &catalog), + Role::Root(None) => {} + Role::Contribution(root) => catalog.media = *root.read(), + } + renditions.sync(&source, &catalog); + if let Role::Contribution(_) = role { + latest = Some(catalog); } }; + let failed = result.is_err(); renditions.finish(result); + // The root catalog lists the contributions: once it fails, stop following and advertising + // them too. A clean end freezes the final catalog, which keeps its entries, as the own + // renditions keep their final set. + if failed && let Role::Root(Some(contributions)) = &role { + contributions.clear(); + } } #[cfg(test)] diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 113c00015d4..49e6686c3bf 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -120,6 +120,7 @@ enum Config { /// The broadcast serving this rendition's media. A sibling that ends ends the rendition: it is /// never rebound onto whatever serves the path next. struct Media { + lazy: bool, handle: Mutex, track: String, /// The rendition reads another broadcast than the catalog's. @@ -128,7 +129,8 @@ struct Media { struct Handle { binding: Arc, - /// The query holding the track instance alive without subscribing to its media. + /// The query pinning the track instance. Even an info query creates publisher demand, + /// so a lazy rendition holds it only while serving media. track: Option, /// The bound sibling ended, so only its retained rows drain. ended: bool, @@ -147,17 +149,23 @@ impl Handle { } impl Media { - fn bind(source: &moq_mux::Source, reference: &impl hang::catalog::Reference, track: &str) -> moq_mux::Result { + fn bind( + source: &moq_mux::Source, + reference: &impl hang::catalog::Reference, + track: &str, + lazy: bool, + ) -> moq_mux::Result { let rel = reference.broadcast(); let mut handle = Handle { binding: Arc::new(source.bind(rel, reference.instance())?), track: None, ended: false, }; - if let Poll::Ready(Ok(broadcast)) = handle.binding.poll_broadcast(&kio::Waiter::noop()) { + if !lazy && let Poll::Ready(Ok(broadcast)) = handle.binding.poll_broadcast(&kio::Waiter::noop()) { handle.capture(&broadcast, track); } Ok(Self { + lazy, handle: Mutex::new(handle), track: track.to_string(), // `bind` already rejected an escaping reference, so both sides name a path. @@ -180,7 +188,9 @@ impl Media { fn poll_sync(&self, waiter: &kio::Waiter, window: &segments::Producer) -> Arc { let mut handle = self.handle.lock().expect("media lock poisoned"); if let Poll::Ready(Ok(broadcast)) = handle.binding.poll_broadcast(waiter) { - handle.capture(&broadcast, &self.track); + if !self.lazy { + handle.capture(&broadcast, &self.track); + } if self.sibling && !handle.ended && broadcast.poll_closed(waiter).is_ready() { // End and Restart both drain only the pinned instance. Looking its path up again // could serve a replacement under the old track's rows. @@ -393,7 +403,7 @@ pub struct Rendition { /// What every rendition of the broadcast shares: discontinuity counts and the run. fanout: renditions::Fanout, /// Follows the media track's groups into [`live`](Self::live); ends when the rendition drops. - watcher: Mutex>, + watcher: Mutex, /// Set once the catalog drops or replaces the rendition (see [`retire`](Self::retire)). retiring: tokio::sync::watch::Sender, /// Which segments this instance lists. @@ -418,6 +428,72 @@ pub struct Rendition { build: Mutex>, } +#[derive(Default)] +struct Watcher { + media: Option, + idle: Option, + #[cfg(feature = "server")] + requests: usize, + stopped: bool, +} + +/// Keeps lazy media demanded for the whole response, including its first encode. +#[cfg(feature = "server")] +pub(crate) struct Demand(Option>); + +#[cfg(feature = "server")] +impl Drop for Demand { + fn drop(&mut self) { + let Some(rendition) = self.0.take() else { return }; + let mut watcher = rendition.watcher.lock().expect("watcher lock poisoned"); + watcher.requests -= 1; + if watcher.requests != 0 || watcher.stopped { + return; + } + // RFC 8216 6.3.4 permits a reload after one target duration (half when unchanged). + // Three target durations allow that reload and two more intervals of request latency. + // Release also forgets the listed rows, so never before a live window would have evicted + // every row the last response listed: the playlist window plus one target duration, since + // eviction happens only as a later segment is listed. Start at completion, so an encode or + // slow fetch never expires mid-request. + let target = Duration::from_secs(rendition.target_duration()); + let duration = target + .saturating_mul(3) + .max(rendition.fanout.window().saturating_add(target)); + let weak = Arc::downgrade(&rendition); + watcher.idle = Some(rendition.fanout.spawn(async move { + kio::clock::sleep(duration).await; + if let Some(rendition) = weak.upgrade() { + let mut claim = rendition.claim.lock().expect("claim lock poisoned"); + let mut watcher = rendition.watcher.lock().expect("watcher lock poisoned"); + if watcher.requests == 0 { + // A finished window keeps listing what it listed, inherited predecessor rows + // included, and keeps its track pin: reads of its retained groups go through + // it, since a closed sibling refuses any later lookup. Its finished watcher + // stays too, so no request follows the ended track again. + if rendition.live.finished() { + return; + } + rendition.media.handle.lock().expect("media lock poisoned").track.take(); + watcher.media.take(); + // A resumed watcher never rewrites already-addressed segments or lists a + // partial segment it had open when the last player left. + claim.start = claim.next; + claim.pending = false; + if claim.retired { + // A replacement waits for this window to close before reading its own + // track. Stopping the watcher must finish that handover too. + rendition.live.cut(); + } else { + rendition.live.clear(); + rendition.previous.lock().expect("previous lock poisoned").take(); + } + } + } + })); + } +} + impl Rendition { /// Whether `config` describes the same media this rendition is already serving, i.e. whether /// it decodes and muxes identically from the same track. Fields the publisher revises @@ -526,7 +602,7 @@ impl Rendition { ) -> moq_mux::Result { let track = config.track(&name); let container = Arc::new(moq_mux::catalog::hang::Container::try_from(config)?); - let media = Media::bind(source, config, track)?; + let media = Media::bind(source, config, track, fanout.lazy)?; Ok(Self { tag: tag(track, interval, &normalize_video(config)), container, @@ -546,7 +622,7 @@ impl Rendition { clock, live: Arc::new(segments::Producer::new(0)), fanout, - watcher: Mutex::new(None), + watcher: Mutex::default(), retiring: tokio::sync::watch::Sender::new(false), claim: Mutex::default(), previous: Mutex::new(None), @@ -569,7 +645,7 @@ impl Rendition { ) -> moq_mux::Result { let track = config.track(&name); let container = Arc::new(moq_mux::catalog::hang::Container::try_from(config)?); - let media = Media::bind(source, config, track)?; + let media = Media::bind(source, config, track, fanout.lazy)?; Ok(Self { tag: tag(track, interval, &normalize_audio(config)), container, @@ -589,7 +665,7 @@ impl Rendition { clock, live: Arc::new(segments::Producer::new(0)), fanout, - watcher: Mutex::new(None), + watcher: Mutex::default(), retiring: tokio::sync::watch::Sender::new(false), claim: Mutex::default(), previous: Mutex::new(None), @@ -605,7 +681,29 @@ impl Rendition { let watcher = self .fanout .spawn(watch_media(Arc::downgrade(self), self.retiring.subscribe())); - *self.watcher.lock().expect("watcher lock poisoned") = Some(watcher); + self.watcher.lock().expect("watcher lock poisoned").media = Some(watcher); + } + + #[cfg(feature = "server")] + pub(crate) fn is_lazy(&self) -> bool { + self.media.lazy + } + + #[cfg(feature = "server")] + pub(crate) fn demand(self: &Arc) -> Demand { + let mut watcher = self.watcher.lock().expect("watcher lock poisoned"); + if !self.media.lazy { + return Demand(None); + } + watcher.idle.take(); + watcher.requests += 1; + if watcher.media.is_none() && !watcher.stopped { + watcher.media = Some( + self.fanout + .spawn(watch_media(Arc::downgrade(self), self.retiring.subscribe())), + ); + } + Demand(Some(self.clone())) } /// Take over from a retired predecessor (see [`retire`](Self::retire)): keep its target @@ -882,8 +980,10 @@ impl Rendition { /// End the window at the segments listed now, because another publisher run replaced this /// one, and stop following its media: what the replaced run lists later is never listed. pub(crate) fn cut(&self) { - let watcher = self.watcher.lock().expect("watcher lock poisoned").take(); - drop(watcher); + let mut watcher = self.watcher.lock().expect("watcher lock poisoned"); + watcher.stopped = true; + watcher.idle.take(); + watcher.media.take(); self.live.cut(); } @@ -1092,6 +1192,25 @@ impl Rendition { self.fanout.failure().is_none() && self.check().is_err() } + /// Whether a track query pins this rendition's media instance. + #[cfg(all(test, feature = "server"))] + pub(crate) fn pinned(&self) -> bool { + self.media.handle.lock().expect("media lock poisoned").track.is_some() + } + + /// Poll until the broadcast serving this rendition's media settles: `true` once it resolved, + /// `false` if it failed, however. A listing advertises only a resolved rendition, so never + /// one whose authorization is pending or whose media can't be read. + #[cfg_attr( + not(feature = "server"), + allow(dead_code, reason = "only the `server` feature uses it") + )] + pub(crate) fn poll_resolved(&self, waiter: &kio::Waiter) -> Poll { + self.poll_media(waiter) + .poll_broadcast(waiter) + .map(|result| result.is_ok()) + } + /// Poll until listings of this rendition refuse (see [`check`](Self::check)), waking when its /// media is denied or its media track fails. #[cfg_attr( diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index a88d19e0cc1..84a293945cf 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -121,6 +121,7 @@ pub(crate) struct Producer { #[derive(Default)] struct State { renditions: BTreeMap>, + media: Option, /// The catalog's confirmed end, distinct from dropping its watcher. finished: Option>, // Cache catalog quality order without counting it as a holder. @@ -159,6 +160,7 @@ async fn hold(binding: moq_mux::Binding, track: String) { /// channel open (so `renditions()` cursors would never terminate). #[derive(Clone)] pub(crate) struct Fanout { + pub(super) lazy: bool, feed: kio::Producer, /// Starts each rendition's background work on the broadcaster's driver. spawner: Spawner, @@ -169,6 +171,7 @@ impl Producer { Self { state: kio::Producer::new(State::default()), fanout: Fanout { + lazy: false, feed: kio::Producer::new(Feed { targets: Vec::new(), discontinuities: Discontinuities::default(), @@ -184,6 +187,12 @@ impl Producer { } } + pub fn lazy(window: Duration, spawner: Spawner) -> Self { + let mut producer = Self::new(window, spawner); + producer.fanout.lazy = true; + producer + } + /// A producer whose renditions run on a driver spawned on the ambient runtime. #[cfg(test)] pub(super) fn spawned(window: Duration) -> Self { @@ -233,6 +242,22 @@ impl Producer { self.state.read().renditions.get(&(kind, name.to_string())).cloned() } + #[cfg(feature = "server")] + pub fn poll_get(&self, waiter: &kio::Waiter, kind: Kind, name: &str) -> Poll>> { + match self.state.poll_ref(waiter, |state| { + if state.media.is_some() { + Poll::Ready(state.renditions.get(&(kind, name.to_string())).cloned()) + } else { + Poll::Pending + } + }) { + Poll::Ready(Ok(rendition)) => Poll::Ready(rendition), + // Read through the closed state's guard: `get` would lock it again. + Poll::Ready(Err(state)) => Poll::Ready(state.renditions.get(&(kind, name.to_string())).cloned()), + Poll::Pending => Poll::Pending, + } + } + /// Snapshot video by name and audio in catalog quality order (for the master playlist). pub fn snapshot(&self) -> Vec> { let current = self.state.read(); @@ -260,6 +285,19 @@ impl Producer { self.state.read().renditions.is_empty() } + /// Whether any rendition could still be listed: enabled and not failed on its own. + #[cfg_attr( + not(feature = "server"), + allow(dead_code, reason = "only the `server` feature uses it") + )] + pub fn listable(&self) -> bool { + self.state + .read() + .renditions + .values() + .any(|rendition| rendition.is_enabled() && !rendition.failed_alone()) + } + /// Release every rendition the map is holding. /// /// The map lives in shared state, so a surviving [`Consumer`] would otherwise keep every @@ -436,14 +474,25 @@ impl Producer { not(feature = "server"), allow(dead_code, reason = "only the `server` feature uses it") )] - pub(crate) async fn advertised(&self) { + pub(crate) fn poll_advertised(&self, waiter: &kio::Waiter) -> Poll<()> { + self.state + .poll_ref(waiter, |current| { + if current.renditions.values().any(|rendition| { + rendition.is_enabled() && (self.fanout.lazy || rendition.poll_advertised(waiter).is_ready()) + }) { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .map(|_| ()) + } + + #[cfg(feature = "server")] + pub(crate) async fn catalogued(&self) { let _ = kio::wait(|waiter| { self.state.poll_ref(waiter, |current| { - if current - .renditions - .values() - .any(|rendition| rendition.is_enabled() && rendition.poll_advertised(waiter).is_ready()) - { + if current.media.is_some() { Poll::Ready(()) } else { Poll::Pending @@ -530,7 +579,7 @@ impl Producer { if let Some(err) = &feed.failed { rendition.fail(err.clone()); rendition.close(); - } else { + } else if !self.fanout.lazy { rendition.watch(); } feed.targets.push(Arc::downgrade(&rendition)); @@ -618,6 +667,7 @@ impl Producer { let Ok(mut current) = self.state.write() else { return; }; + current.media = Some(catalog.media); // Renditions the catalog dropped or reconfigured. Retire each as it goes so any cursor // over it drains and ends, instead of parking on a window that never finishes -- the @@ -657,7 +707,8 @@ impl Producer { let current = &mut *current; match interval { Some(_) => current.waiting.clear(), - None => self.wait(source, catalog, &mut current.waiting), + None if !self.fanout.lazy => self.wait(source, catalog, &mut current.waiting), + None => current.waiting.clear(), } let mut added = Vec::new(); @@ -807,6 +858,20 @@ fn next_event(current: &BTreeMap>, seen: &BTreeMap, /// The resource requested under the broadcast. pub resource: Resource, } +/// The contribution catalog and stream named inside a broadcast's instance URL. +/// +/// A contribution's resource nests under the served broadcast as +/// `{broadcast}/@{instance}/contribution/{id}/{kind}/{rendition}/@{contribution-instance}/` +/// followed by the [`Resource`] file (`media.m3u8`, `init.{hash}.mp4`, or `seg/{segment}.m4s`). +/// The broadcast's own `@{instance}` comes first, so [`Route::broadcast`] stays the parent's +/// path and a path-based policy check covers its contributions. A denied contribution answers +/// `403`. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct Contribution { + /// The contribution ID in the parent catalog, percent-decoded. + pub id: String, + /// The contribution's publisher instance, percent-decoded and without its `@`. + pub instance: String, +} + /// A resource under a broadcast. #[derive(Clone, Debug, PartialEq, Eq)] #[non_exhaustive] @@ -72,7 +91,7 @@ impl Route { /// segment cannot, so an encoded separator cannot smuggle a different broadcast past a /// policy check on [`broadcast`](Self::broadcast). pub fn parse(path: &str) -> Option { - let parts = path + let mut parts = path .strip_prefix('/')? .split('/') .map(|part| { @@ -86,6 +105,39 @@ impl Route { return None; } + // Put the child instance after the rendition: this suffix cannot also be an + // ordinary rendition URL for a longer broadcast path. The kind would occupy + // that URL's @instance slot, so the two grammars are disjoint. + let tail = match parts.as_slice() { + [.., directory, _] if directory == "seg" => 2, + [.., file] if file == "media.m3u8" || init_hash(file).is_some() => 1, + _ => 0, + }; + let end = parts.len() - tail; + let contribution = if tail != 0 { + match &parts[..end] { + [.., parent, directory, id, kind, _, child] + if directory == "contribution" + && instance_id(parent).is_some() + && Kind::parse(kind).is_some() + && instance_id(child).is_some() => + { + // Every slot has the contribution shape; otherwise the ordinary arms below + // parse the untouched parts as a longer broadcast path. + let contribution = Contribution { + id: id.clone(), + instance: instance_id(child)?, + }; + parts.remove(end - 1); + parts.drain(end - 5..end - 3); + Some(contribution) + } + _ => None, + } + } else { + None + }; + let (broadcast, resource) = match parts.as_slice() { [broadcast @ .., file] if file == "master.m3u8" => (broadcast, Resource::Master), [broadcast @ .., file] if file == "manifest.mpd" => (broadcast, Resource::Manifest), @@ -123,6 +175,7 @@ impl Route { } Some(Self { broadcast: broadcast.join("/"), + contribution, resource, }) } @@ -153,6 +206,7 @@ mod tests { Route::parse("/project/live/@run/video/cam%231%2Fmain%3Falt/media.m3u8"), Some(Route { broadcast: "project/live".to_string(), + contribution: None, resource: Resource::Media { instance: "run".to_string(), kind: Kind::Video, @@ -259,4 +313,49 @@ mod tests { assert!(Route::parse("/project/private%2F/master.m3u8").is_none()); assert!(Route::parse("/project%2Fprivate/master.m3u8").is_none()); } + + #[test] + fn contribution_resources_keep_the_parent_policy_path() { + for suffix in ["media.m3u8", "init.abcd.mp4", "seg/abcd.7.m4s"] { + let route = Route::parse(&format!( + "/room/live/@parent/contribution/a%2Fb/video/same%2Fname/@child/{suffix}" + )) + .unwrap(); + assert_eq!(route.broadcast, "room/live"); + assert_eq!( + route.contribution, + Some(Contribution { + id: "a/b".into(), + instance: "child".into() + }) + ); + let instance = match route.resource { + Resource::Media { instance, .. } + | Resource::Init { instance, .. } + | Resource::Segment { instance, .. } => instance, + _ => panic!("wrong resource"), + }; + assert_eq!(instance, "parent"); + } + let own = Route::parse("/room/@parent/contribution/id/@child/video/main/media.m3u8").unwrap(); + assert_eq!(own.broadcast, "room/@parent/contribution/id"); + assert!(own.contribution.is_none()); + assert!(Route::parse("/room%2Flive/@parent/contribution/id/video/main/@child/media.m3u8").is_none()); + // Contribution-shaped but for the child slot: an ordinary URL on a longer broadcast path. + let own = Route::parse("/room/@parent/contribution/id/video/@run/audio/seg/media.m3u8").unwrap(); + assert_eq!(own.broadcast, "room/@parent/contribution/id/video"); + assert!(own.contribution.is_none()); + match own.resource { + Resource::Media { + instance, + kind, + rendition, + } => { + assert_eq!(instance, "run"); + assert_eq!(kind, Kind::Audio); + assert_eq!(rendition, "seg"); + } + _ => panic!("wrong resource"), + } + } } diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 5bf2177af05..1a426db9cc2 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -304,6 +304,14 @@ impl Producer { self.close(); } + /// Whether no further segment will ever be listed: the media ended, the window failed, + /// or it was closed. + #[cfg(feature = "server")] + pub fn finished(&self) -> bool { + let state = self.state.read(); + state.ended || state.failed.is_some() || state.is_closed() + } + /// Why the window failed, if it did. pub fn failure(&self) -> Option { self.state.read().failed.clone() diff --git a/rs/moq-hls/src/server/contributions_tests.rs b/rs/moq-hls/src/server/contributions_tests.rs new file mode 100644 index 00000000000..5bfdc1ed23e --- /dev/null +++ b/rs/moq-hls/src/server/contributions_tests.rs @@ -0,0 +1,640 @@ +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use hang::catalog::{AudioCodec, AudioConfig, Contribution, Join, VideoCodec, VideoConfig}; +use moq_mux::catalog::hang::Catalog; +use moq_net::{Instance, Timestamp, track}; +use tower::ServiceExt; + +use super::Server; + +struct Published { + publication: moq_net::origin::Publication, + catalog: moq_json::snapshot::Producer, + video: track::Producer, + audio: track::Producer, +} + +fn catalog() -> Catalog { + let mut catalog = Catalog::default(); + catalog.media.join = Join::new(Duration::from_secs(2)); + let mut video = VideoConfig::new(VideoCodec::VP8); + video.coded_width = Some(320); + video.coded_height = Some(240); + video.bitrate = Some(500_000); + catalog.video.renditions.insert("same".into(), video); + let mut audio = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + audio.track = Some("audio".into()); + catalog.audio.renditions.insert("sound".into(), audio); + catalog +} + +fn publish(origin: &moq_net::origin::Producer, path: &str, instance: &Instance, value: &Catalog) -> Published { + let publication = origin.create_broadcast(path).unwrap(); + let track = publication + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut catalog = moq_json::snapshot::Producer::new(track, Default::default()); + catalog.update(moq_net::Timed::from(value).at(Timestamp::ZERO)).unwrap(); + let video = publication + .create_track("same", hang::container::track_info(1)) + .unwrap(); + // Separate tracks allow the test to observe exactly which rendition is requested. + let audio = publication + .create_track("audio", hang::container::track_info(2)) + .unwrap(); + publication + .announce(moq_net::origin::Route::default().with_instance(instance.clone(), 0)) + .unwrap(); + Published { + publication, + catalog, + video, + audio, + } +} + +fn frames(track: &track::Producer, first: u64) { + for sequence in first..first + 3 { + let mut group = track.create_group(sequence.into()).unwrap(); + hang::container::Frame { + timestamp: Timestamp::from_secs(sequence * 2).unwrap(), + payload: bytes::Bytes::from_static(if track.name() == "audio" { + &[0xf8, 0xff, 0xfe] + } else { + &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00] + }), + } + .write_to(&mut group) + .unwrap(); + group.finish().unwrap(); + } +} + +async fn request(server: &Server, uri: &str) -> (StatusCode, String) { + let response = server + .router() + .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); + (status, String::from_utf8_lossy(&body).into_owned()) +} + +async fn master(server: &Server, contains: &str, present: bool) -> String { + let deadline = kio::clock::now() + Duration::from_secs(10); + loop { + let (status, body) = request(server, "/room/live/master.m3u8?jwt=viewer").await; + if status == StatusCode::OK && body.contains(contains) == present { + return body; + } + assert!( + kio::clock::now() < deadline, + "master never reached {contains:?} = {present}: {status} {body}" + ); + kio::clock::sleep(Duration::from_millis(10)).await; + } +} + +fn parent_catalog() -> Catalog { + let mut catalog = catalog(); + catalog.audio.renditions.get_mut("sound").unwrap().track = Some("audio".into()); + catalog.contributions.insert( + "rung/a".into(), + Contribution::new(moq_net::path::Relative::new("./ladder").unwrap()), + ); + catalog +} + +#[tokio::test(start_paused = true)] +async fn a_contribution_does_not_hide_the_parents_first_variants() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + let _contribution = publish(&origin, "room/ladder", &child, &catalog()); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let broadcaster = server.broadcaster("room/live").await.unwrap(); + let at = crate::export::Contribution { + id: "rung/a".into(), + instance: child.to_string(), + }; + assert!( + broadcaster + .contribution(&at, crate::export::Kind::Video, "same") + .await + .unwrap() + .is_some() + ); + let mut pending = Box::pin(request(&server, "/room/live/master.m3u8")); + assert!(futures::poll!(pending.as_mut()).is_pending()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + let (status, body) = pending.await; + assert_eq!(status, StatusCode::OK); + assert!(body.contains(&format!("@{root}/video/same/media.m3u8"))); + assert!(body.contains(&format!("contribution/rung%2Fa/video/same/@{child}/media.m3u8"))); +} + +#[tokio::test(start_paused = true)] +async fn an_idle_predecessor_does_not_strand_a_requested_reconfigure() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + let mut value = catalog(); + let mut contribution = publish(&origin, "room/ladder", &child, &value); + frames(&contribution.video, 0); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, "contribution/", true).await; + let uri = format!("/room/live/@{root}/contribution/rung%2Fa/video/same/@{child}/media.m3u8"); + assert_eq!(request(&server, &uri).await.0, StatusCode::OK); + let next = contribution + .publication + .create_track("next", hang::container::track_info(1)) + .unwrap(); + let video = value.video.renditions.get_mut("same").unwrap(); + video.track = Some("next".into()); + video.coded_width = Some(640); + contribution + .catalog + .update(moq_net::Timed::from(&value).at(Timestamp::ZERO)) + .unwrap(); + master(&server, "RESOLUTION=640x240", true).await; + // The new response lists the predecessor's retained rows while waiting for its + // last pending segment. Its body keeps the replacement requested past that idle window. + let held = server + .router() + .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(held.status(), StatusCode::OK); + tokio::time::advance(Duration::from_secs(21)).await; + let mut demand = next.demand(); + kio::clock::timeout(Duration::from_secs(1), async { + while demand.wanted_changed().await.unwrap().is_none() {} + }) + .await + .expect("the requested replacement must start after the idle predecessor releases"); + assert_eq!(contribution.video.demand().wanted(), None); + frames(&next, 3); + drop(held); +} + +#[tokio::test(start_paused = true)] +async fn a_finished_parent_catalog_keeps_its_contributions() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let mut parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + let _contribution = publish(&origin, "room/ladder", &child, &catalog()); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let listed = format!("contribution/rung%2Fa/video/same/@{child}"); + master(&server, &listed, true).await; + // A clean end freezes the final catalog, entries included. + parent.catalog.finish().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + master(&server, &listed, true).await; +} + +#[tokio::test(start_paused = true)] +async fn a_failed_parent_catalog_stops_its_contributions() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let Published { + publication: _parent, + catalog: parent_catalog_track, + video, + audio, + } = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&video, 0); + frames(&audio, 0); + let contribution = publish(&origin, "room/ladder", &child, &catalog()); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, "contribution/", true).await; + let uri = format!("/room/live/@{root}/contribution/rung%2Fa/video/same/@{child}/media.m3u8"); + parent_catalog_track.abort(moq_net::Error::Cancel).unwrap(); + let deadline = kio::clock::now() + Duration::from_secs(10); + // A live contribution's playlist waits for media this test never writes, so bound each request. + while !matches!( + kio::clock::timeout(Duration::from_secs(1), request(&server, &uri)).await, + Ok((StatusCode::NOT_FOUND, _)) + ) { + assert!( + kio::clock::now() < deadline, + "the contribution outlived its parent catalog" + ); + kio::clock::sleep(Duration::from_millis(10)).await; + } + assert!( + !request(&server, "/room/live/master.m3u8") + .await + .1 + .contains("contribution/") + ); + let mut catalog = contribution.catalog.demand(); + kio::clock::timeout(Duration::from_secs(1), async { + while catalog.wanted_changed().await.unwrap().is_some() {} + }) + .await + .expect("the contribution catalog is released"); +} + +#[tokio::test(start_paused = true)] +async fn idle_release_waits_out_the_playlist_window() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + let contribution = publish(&origin, "room/ladder", &child, &catalog()); + frames(&contribution.video, 0); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, "contribution/", true).await; + let uri = format!("/room/live/@{root}/contribution/rung%2Fa/video/same/@{child}/media.m3u8"); + assert_eq!(request(&server, &uri).await.0, StatusCode::OK); + let settle = || kio::clock::sleep(Duration::from_millis(10)); + settle().await; + // Three target durations (12 s) would forget rows the 16 s window still lists. + tokio::time::advance(Duration::from_secs(13)).await; + settle().await; + assert!(contribution.video.demand().wanted().is_some()); + tokio::time::advance(Duration::from_secs(8)).await; + settle().await; + assert_eq!(contribution.video.demand().wanted(), None); +} + +#[tokio::test(start_paused = true)] +async fn idle_release_keeps_an_ended_playlists_inherited_segments() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + let mut value = catalog(); + let mut contribution = publish(&origin, "room/ladder", &child, &value); + frames(&contribution.video, 0); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, "contribution/", true).await; + let uri = format!("/room/live/@{root}/contribution/rung%2Fa/video/same/@{child}/media.m3u8"); + assert_eq!(request(&server, &uri).await.0, StatusCode::OK); + // Reconfigure onto another track; the predecessor ends with its own track. + let next = contribution + .publication + .create_track("next", hang::container::track_info(1)) + .unwrap(); + let video = value.video.renditions.get_mut("same").unwrap(); + video.track = Some("next".into()); + video.coded_width = Some(640); + contribution + .catalog + .update(moq_net::Timed::from(&value).at(Timestamp::ZERO)) + .unwrap(); + master(&server, "RESOLUTION=640x240", true).await; + contribution.video.finish().unwrap(); + let held = server + .router() + .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(held.status(), StatusCode::OK); + frames(&next, 3); + next.finish().unwrap(); + drop(held); + let deadline = kio::clock::now() + Duration::from_secs(10); + let ended = loop { + let (status, playlist) = request(&server, &uri).await; + if status == StatusCode::OK && playlist.contains("#EXT-X-ENDLIST") { + break playlist; + } + assert!( + kio::clock::now() < deadline, + "the playlist never ended: {status} {playlist}" + ); + kio::clock::sleep(Duration::from_millis(10)).await; + }; + assert!(ended.contains("#EXT-X-MEDIA-SEQUENCE:0"), "{ended}"); + assert!(ended.contains("#EXT-X-DISCONTINUITY"), "{ended}"); + // Past the idle window, the ended playlist still lists its predecessor's segments. + // Let the idle task start its timer, and run once it fires, before a request renews demand. + kio::clock::sleep(Duration::from_millis(10)).await; + tokio::time::advance(Duration::from_secs(21)).await; + kio::clock::sleep(Duration::from_millis(10)).await; + let (status, after) = request(&server, &uri).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(after, ended); + // The finished window keeps the track pin its retained groups are read through. + let broadcaster = server.broadcaster("room/live").await.unwrap(); + let at = crate::export::Contribution { + id: "rung/a".into(), + instance: child.to_string(), + }; + let rendition = broadcaster + .contribution(&at, crate::export::Kind::Video, "same") + .await + .unwrap() + .unwrap(); + assert!(rendition.pinned()); +} + +#[tokio::test(start_paused = true)] +async fn a_contribution_rendition_whose_broadcast_failed_is_not_listed() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + let other = origin.requests("room/other").unwrap(); + let child = Instance::random(); + let mut value = catalog(); + value.audio.renditions.clear(); + value.video.renditions.get_mut("same").unwrap().broadcast = Some(moq_net::path::Relative::new("./other").unwrap()); + let _contribution = publish(&origin, "room/ladder", &child, &value); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, &format!("@{root}/video/same"), true).await; + // Not a denial, but its media still can't be read. + kio::clock::timeout(Duration::from_secs(5), other.requested_broadcast()) + .await + .expect("the binding never asked") + .unwrap() + .reject(moq_net::Error::Unroutable); + kio::clock::sleep(Duration::from_secs(1)).await; + let listed = master(&server, &format!("@{root}/video/same"), true).await; + assert!(!listed.contains("contribution/"), "{listed}"); +} + +#[tokio::test(start_paused = true)] +async fn a_contribution_answers_the_master_when_no_own_rendition_is_listable() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let mut value = parent_catalog(); + value.video.renditions.get_mut("same").unwrap().enabled = false; + value.audio.renditions.get_mut("sound").unwrap().enabled = false; + let _parent = publish(&origin, "room/live", &root, &value); + let _contribution = publish(&origin, "room/ladder", &child, &catalog()); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let broadcaster = server.broadcaster("room/live").await.unwrap(); + let at = crate::export::Contribution { + id: "rung/a".into(), + instance: child.to_string(), + }; + assert!( + broadcaster + .contribution(&at, crate::export::Kind::Video, "same") + .await + .unwrap() + .is_some() + ); + // Disabled own renditions can't be listed, so the master needn't wait out its timeout. + let start = kio::clock::now(); + let (status, body) = request(&server, "/room/live/master.m3u8").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!( + body.contains(&format!("contribution/rung%2Fa/video/same/@{child}")), + "{body}" + ); + assert!(kio::clock::now() - start < Duration::from_secs(1)); +} + +#[tokio::test(start_paused = true)] +async fn a_contribution_rendition_is_listed_once_its_broadcast_resolves() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + // The contribution's only rendition reads a sibling whose route has not answered yet. + let other = origin.requests("room/other").unwrap(); + let child = Instance::random(); + let mut value = catalog(); + value.audio.renditions.clear(); + value.video.renditions.get_mut("same").unwrap().broadcast = Some(moq_net::path::Relative::new("./other").unwrap()); + let _contribution = publish(&origin, "room/ladder", &child, &value); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let listed = master(&server, &format!("@{root}/video/same"), true).await; + assert!( + !listed.contains("contribution/"), + "a pending binding may yet be denied: {listed}" + ); + let request = kio::clock::timeout(Duration::from_secs(5), other.requested_broadcast()) + .await + .expect("the binding never asked") + .unwrap(); + request.serve(moq_net::broadcast::Info::new().produce()); + master(&server, &format!("contribution/rung%2Fa/video/same/@{child}"), true).await; +} + +#[tokio::test(start_paused = true)] +#[tracing_test::traced_test] +async fn listing_is_catalog_only_and_requests_own_the_lazy_media_lifetime() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let child = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + let mut child_catalog = catalog(); + child_catalog.audio.renditions.get_mut("sound").unwrap().track = Some("audio".into()); + let contribution = publish(&origin, "room/ladder", &child, &child_catalog); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let prefix = format!("@{root}/contribution/rung%2Fa"); + let listed = master(&server, &prefix, true).await; + assert!(listed.contains(&format!("@{root}/video/same/media.m3u8?jwt=viewer"))); + assert!(listed.contains(&format!("{prefix}/video/same/@{child}/media.m3u8?jwt=viewer"))); + assert!(listed.contains("NAME=\"rung/a/sound\"")); + assert_eq!(contribution.video.demand().wanted(), None); + assert_eq!(contribution.audio.demand().wanted(), None); + assert!(contribution.catalog.demand().wanted().is_some()); + + let uri = format!("/room/live/{prefix}/video/same/@{child}/media.m3u8"); + let mut pending = Box::pin(request(&server, &uri)); + assert!(futures::poll!(pending.as_mut()).is_pending()); + let mut demand = contribution.video.demand(); + while demand.wanted_changed().await.unwrap().is_none() {} + assert_eq!(contribution.audio.demand().wanted(), None); + // The idle window is 20 s (the 16 s playlist window plus TARGETDURATION 4), and this first + // encode is slower than all of it. + tokio::time::advance(Duration::from_secs(21)).await; + assert!(futures::poll!(pending.as_mut()).is_pending()); + assert!(contribution.video.demand().wanted().is_some()); + frames(&contribution.video, 0); + let (status, playlist) = pending.await; + assert_eq!(status, StatusCode::OK, "{playlist}"); + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(playlist.contains("#EXT-X-TARGETDURATION:4")); + assert!(!playlist.contains("#EXT-X-ENDLIST")); + let held = server + .router() + .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(held.status(), StatusCode::OK); + tokio::time::advance(Duration::from_secs(21)).await; + assert!( + contribution.video.demand().wanted().is_some(), + "a slow body still holds demand" + ); + drop(held); + // The reload window starts at completion, and another request renews it. + tokio::time::advance(Duration::from_secs(19)).await; + assert!(contribution.video.demand().wanted().is_some()); + assert_eq!(request(&server, &uri).await.0, StatusCode::OK); + tokio::time::advance(Duration::from_secs(19)).await; + assert!(contribution.video.demand().wanted().is_some()); + tokio::time::advance(Duration::from_secs(2)).await; + while demand.wanted_changed().await.unwrap().is_some() {} + assert_eq!(contribution.audio.demand().wanted(), None); + // A later request creates fresh demand and indexes later segments on the same grid. + let mut resumed = Box::pin(request(&server, &uri)); + assert!(futures::poll!(resumed.as_mut()).is_pending()); + while demand.wanted_changed().await.unwrap().is_none() {} + frames(&contribution.video, 3); + let (status, playlist) = resumed.await; + assert_eq!(status, StatusCode::OK); + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:3"), "{playlist}"); + assert_eq!(contribution.audio.demand().wanted(), None); + // Neither the DASH manifest nor the recorder cursor expands the contribution. + assert!( + !request(&server, "/room/live/manifest.mpd") + .await + .1 + .contains("contribution") + ); + let broadcaster = server.broadcaster("room/live").await.unwrap(); + let mut recorder = broadcaster.renditions(); + for _ in 0..2 { + assert!(recorder.next().await.unwrap().is_some()); + } + assert!(futures::poll!(Box::pin(recorder.next()).as_mut()).is_pending()); + drop((broadcaster, server)); + while demand.wanted_changed().await.unwrap().is_some() {} +} + +#[tokio::test(start_paused = true)] +#[tracing_test::traced_test] +async fn late_instances_replace_unpinned_entries_and_do_not_replace_pins() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let first = Instance::random(); + let second = Instance::random(); + let mut value = parent_catalog(); + let mut pin = value.contributions["rung/a"].clone(); + pin.instance = Some(first.clone()); + value.contributions.insert("pinned".into(), pin); + let parent = publish(&origin, "room/live", &root, &value); + frames(&parent.video, 0); + frames(&parent.audio, 0); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, "contribution/", false).await; + let old = publish(&origin, "room/ladder", &first, &catalog()); + let old_uri = format!("contribution/rung%2Fa/video/same/@{first}"); + let listed = master(&server, &old_uri, true).await; + assert!(listed.contains(&format!("contribution/pinned/video/same/@{first}"))); + assert_eq!(old.video.demand().wanted(), None); + old.publication.unannounce(); + master(&server, "contribution/", false).await; + old.publication + .announce(moq_net::origin::Route::default().with_instance(first.clone(), 0)) + .unwrap(); + master(&server, &format!("contribution/pinned/video/same/@{first}"), true).await; + let new = publish(&origin, "room/ladder", &second, &catalog()); + let listed = master(&server, &format!("contribution/rung%2Fa/video/same/@{second}"), true).await; + assert!(!listed.contains("contribution/pinned/")); + assert!(!listed.contains(&old_uri)); + assert_eq!(new.video.demand().wanted(), None); + assert_eq!( + request(&server, &format!("/room/live/@{root}/{old_uri}/media.m3u8")) + .await + .0, + StatusCode::NOT_FOUND + ); + new.publication.unannounce(); + master(&server, &format!("contribution/pinned/video/same/@{first}"), true).await; +} + +#[tokio::test(start_paused = true)] +#[tracing_test::traced_test] +async fn contributions_take_the_roots_grid_and_denied_ones_are_not_advertised() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + let child = Instance::random(); + // The contribution declares its own interval, which HLS ignores for the root's. + let mut value = catalog(); + value.media.join = Join::new(Duration::from_secs(1)); + let contribution = publish(&origin, "room/ladder", &child, &value); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + master(&server, "contribution/", true).await; + let patterns = ["room/live".parse::().unwrap()].into_iter().collect(); + let denied = Server::new(origin.consume().scope("", &patterns).unwrap(), Default::default()).unwrap(); + master(&denied, "contribution/", false).await; + for resource in ["media.m3u8", "init.abcd.mp4", "seg/abcd.0.m4s"] { + assert_eq!( + request( + &denied, + &format!("/room/live/@{root}/contribution/rung%2Fa/video/same/@{child}/{resource}") + ) + .await + .0, + StatusCode::FORBIDDEN + ); + } + assert_eq!(contribution.video.demand().wanted(), None); + let uri = format!("/room/live/@{root}/contribution/rung%2Fa/video/same/@{child}/media.m3u8"); + let mut pending = Box::pin(request(&server, &uri)); + assert!(futures::poll!(pending.as_mut()).is_pending()); + let mut demand = contribution.video.demand(); + kio::clock::timeout(Duration::from_secs(10), async { + while demand.wanted_changed().await.unwrap().is_none() {} + }) + .await + .expect("the request never demanded the contribution's media"); + frames(&contribution.video, 0); + let (status, playlist) = kio::clock::timeout(Duration::from_secs(10), pending) + .await + .expect("the contribution never listed a segment"); + assert_eq!(status, StatusCode::OK, "{playlist}"); + // Twice the root's 2 s interval; its own 1 s interval would give 2. + assert!(playlist.contains("#EXT-X-TARGETDURATION:4"), "{playlist}"); +} + +#[tokio::test(start_paused = true)] +async fn a_refused_contribution_is_listed_once_announced() { + let origin = moq_tokio::origin::spawn(); + let root = Instance::random(); + let parent = publish(&origin, "room/live", &root, &parent_catalog()); + frames(&parent.video, 0); + frames(&parent.audio, 0); + // An on-demand route that drops its prepared answer refuses the first resolution. + let claim = origin.requests("room/ladder").unwrap(); + let server = Server::new(origin.consume(), Default::default()).unwrap(); + let refuse = async { + let request = kio::clock::timeout(Duration::from_secs(5), claim.requested_broadcast()) + .await + .expect("never asked") + .unwrap(); + drop(request.prepare().unwrap()); + }; + let own = format!("@{root}/video/same"); + let (listed, ()) = futures::join!(master(&server, &own, true), refuse); + assert!(!listed.contains("contribution/"), "{listed}"); + // The refusing route is not asked again on a timer. + assert!( + kio::clock::timeout(Duration::from_secs(30), claim.requested_broadcast()) + .await + .is_err() + ); + drop(claim); + let child = Instance::random(); + let _contribution = publish(&origin, "room/ladder", &child, &catalog()); + master(&server, &format!("contribution/rung%2Fa/video/same/@{child}"), true).await; +} diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index 88b1bf9ede4..bcd26910f4b 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -61,6 +61,8 @@ //! # } //! ``` +#[cfg(test)] +mod contributions_tests; mod routes; use std::collections::HashMap; @@ -281,7 +283,7 @@ impl Inner { /// broadcaster. fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Broadcaster { let source = moq_mux::Source::new(self.origin.consume(), broadcast); - let (broadcaster, driver) = Broadcaster::start(source, self.config.clone()); + let (broadcaster, driver) = Broadcaster::start(source, self.config.clone(), Some(self.origin.clone())); tokio::spawn(driver.run()); broadcaster } diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index 54524e9450b..e6d9261eb2d 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -1,6 +1,8 @@ //! axum handlers for the HLS endpoints. +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use std::time::Duration; use axum::Router; @@ -49,19 +51,19 @@ impl Server { instance, kind, rendition, - } => media(self, At::new(broadcast, instance, *kind, rendition), query).await, + } => media(self, At::new(route, instance, *kind, rendition), query).await, Resource::Init { instance, kind, rendition, hash, - } => init(self, At::new(broadcast, instance, *kind, rendition), hash).await, + } => init(self, At::new(route, instance, *kind, rendition), hash).await, Resource::Segment { instance, kind, rendition, segment: id, - } => segment(self, At::new(broadcast, instance, *kind, rendition), id).await, + } => segment(self, At::new(route, instance, *kind, rendition), id).await, } } } @@ -103,9 +105,24 @@ async fn manifest(server: &Server, broadcast: &str, query: Option<&str>) -> Resp } async fn media(server: &Server, at: At<'_>, query: Option<&str>) -> Response { - let Some(rendition) = rendition_for(server, at).await else { - return not_found(); + let rendition = match rendition_for(server, at).await { + Ok(Some(rendition)) => rendition, + Ok(None) => return not_found(), + Err(err) => return server_error(err), }; + let demand = rendition.demand(); + if rendition.is_lazy() { + // Unbounded on purpose: this request's demand is what starts an on-demand encode, whose + // first segment can take longer than `READY_TIMEOUT`, and a 404 then would tell the player + // the rendition is missing while it is still starting. The wait ends with the client's + // connection, which drops the demand with it. + let response = match rendition.playlist(query).await { + Ok(Some(playlist)) => m3u8(playlist), + Ok(None) => not_found(), + Err(err) => server_error(err), + }; + return demanded(response, demand); + } match kio::clock::timeout(READY_TIMEOUT, rendition.playlist(query)).await { Ok(Ok(Some(playlist))) => m3u8(playlist), Ok(Ok(None)) | Err(_) => not_found(), @@ -114,32 +131,76 @@ async fn media(server: &Server, at: At<'_>, query: Option<&str>) -> Response { } async fn init(server: &Server, at: At<'_>, hash: &str) -> Response { - let Some(rendition) = rendition_for(server, at).await else { - return not_found(); + let rendition = match rendition_for(server, at).await { + Ok(Some(rendition)) => rendition, + Ok(None) => return not_found(), + Err(err) => return server_error(err), }; - media_result(rendition.init(hash).await, server) + let demand = rendition.demand(); + let response = media_result(rendition.init(hash).await, server); + if rendition.is_lazy() { + demanded(response, demand) + } else { + response + } } async fn segment(server: &Server, at: At<'_>, id: &segments::Id) -> Response { - let Some(rendition) = rendition_for(server, at).await else { - return not_found(); + let rendition = match rendition_for(server, at).await { + Ok(Some(rendition)) => rendition, + Ok(None) => return not_found(), + Err(err) => return server_error(err), }; - media_result(rendition.segment(id).await, server) + let demand = rendition.demand(); + let response = media_result(rendition.segment(id).await, server); + if rendition.is_lazy() { + demanded(response, demand) + } else { + response + } +} + +/// Retain demand through the HTTP body's lifetime, including backpressure from a slow player. +fn demanded(response: Response, demand: crate::export::Demand) -> Response { + let (parts, body) = response.into_parts(); + Response::from_parts(parts, axum::body::Body::new(Demanded { body, _demand: demand })) +} + +struct Demanded { + body: axum::body::Body, + _demand: crate::export::Demand, +} + +impl http_body::Body for Demanded { + type Data = Bytes; + type Error = axum::Error; + fn poll_frame( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, axum::Error>>> { + Pin::new(&mut self.body).poll_frame(cx) + } + fn is_end_stream(&self) -> bool { + self.body.is_end_stream() + } + fn size_hint(&self) -> http_body::SizeHint { + self.body.size_hint() + } } /// A rendition of one publisher instance, as an instance-scoped URL names it. #[derive(Clone, Copy)] struct At<'a> { - broadcast: &'a str, + route: &'a Route, instance: &'a str, kind: Kind, rendition: &'a str, } impl<'a> At<'a> { - fn new(broadcast: &'a str, instance: &'a str, kind: Kind, rendition: &'a str) -> Self { + fn new(route: &'a Route, instance: &'a str, kind: Kind, rendition: &'a str) -> Self { Self { - broadcast, + route, instance, kind, rendition, @@ -148,10 +209,22 @@ impl<'a> At<'a> { } /// Resolve a rendition, waiting for the catalog to populate. -async fn rendition_for(server: &Server, at: At<'_>) -> Option> { - let broadcaster = server.instance(at.broadcast, at.instance).await?; +async fn rendition_for(server: &Server, at: At<'_>) -> crate::Result>> { + let Some(broadcaster) = server.instance(&at.route.broadcast, at.instance).await else { + return Ok(None); + }; + if let Some(contribution) = &at.route.contribution { + // Resolve a cold edge's contribution before looking up its rendition. This + // bounds discovery only; the requested rendition's first encode waits below. + return kio::clock::timeout( + READY_TIMEOUT, + broadcaster.contribution(contribution, at.kind, at.rendition), + ) + .await + .unwrap_or(Ok(None)); + } let _ = kio::clock::timeout(READY_TIMEOUT, broadcaster.ready()).await; - broadcaster.rendition(at.kind, at.rendition) + Ok(broadcaster.rendition(at.kind, at.rendition)) } fn m3u8(body: String) -> Response { diff --git a/rs/moq-mux/src/follow.rs b/rs/moq-mux/src/follow.rs index 2e50b15cd17..8182b083c1b 100644 --- a/rs/moq-mux/src/follow.rs +++ b/rs/moq-mux/src/follow.rs @@ -65,7 +65,7 @@ impl Instances { /// /// Each announcement that brings an instance hands it out, so the same one returns after an /// end. Fails once the origin closes, and with any resolution failure, such as a handler - /// refusing the path. + /// refusing the path; the call after a refusal waits for the path's next announcement. pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { loop { let next = match &mut self.feed { diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 508423d1c49..36b64940475 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Following` waits out a refusal ([#7836](https://github.com/moq-dev/moq/pull/7836)) + +After `Following::available` returns a handler's refusal, the next call no longer asks the refusing route again at once. +It waits for the serving route to change: another instance, the path going and coming back, or the same instance over another route. +A caller that retried in a loop now waits instead of spinning; one that gave up on the first error is unaffected. + ### Authored tracks default to a zero max age ([#7729](https://github.com/moq-dev/moq/pull/7729)) `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)` instead of `None`: a superseded group is stale as soon as its successor arrives, so readers get only the live edge; the newest group never goes stale. diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index 65f0f324d90..e4b1b7058d5 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -218,6 +218,9 @@ pub struct Following { /// A change resolved to the broadcast handed out last while it was still open, so it counts /// as delivered only until that broadcast closes. absorbed: bool, + /// The last resolution was refused: any change to the serving route, even the same + /// instance over another route, is worth asking again. + refused: bool, } impl Following { @@ -232,6 +235,7 @@ impl Following { last: None, moved: None, absorbed: false, + refused: false, } } @@ -239,7 +243,9 @@ impl Following { /// [`Consumer::follow`](crate::origin::Consumer::follow). /// /// Returns [`Error::Closed`] once the origin closes, and any other resolution failure - /// as-is. Dropping the future keeps the wait's progress for the next call. + /// as-is. A later call then waits for the serving route to change, even to another route of + /// the same instance, instead of asking the refusing route again. Dropping the future keeps the wait's progress for the + /// next call. #[cfg(feature = "async")] pub async fn available(&mut self) -> Result { kio::wait(|waiter| self.poll_available(waiter)).await @@ -260,8 +266,9 @@ impl Following { // routes may have gone, and another instance's, beneath the same announcement, // taken the path over. It also ends once the caller lets go of it, so only its // instance no longer serving the path makes whatever serves it now next, unless a - // change it absorbed is owed. + // change it absorbed is owed. After a refusal, only a route change asks again. if !self.unseen + && !self.refused && self.follow.serving().is_some() && let Some(last) = self.last.as_ref().map(broadcast::WeakConsumer::consume) && last.poll_closed(waiter).is_ready() @@ -292,8 +299,11 @@ impl Following { .get_or_insert_with(|| self.consumer.routing(self.path.clone(), self.instance.clone())); let resolved = ready!(kio::Task::poll(resolving, waiter)); self.resolving = None; - let broadcast = resolved?; + // Handed out or refused, this announcement is settled: after a refusal, the next + // call waits for the serving route to change rather than asking it again. self.unseen = false; + self.refused = resolved.is_err(); + let broadcast = resolved?; // Resolving can run ahead of the follower: a takeover resolves at once while the // origin holds its restart, and that restart then names the instance handed out. @@ -341,6 +351,8 @@ impl Following { self.moved = None; self.absorbed = false; } + // After a refusal, another route may answer what the last one refused. + AnnounceEvent::Update(_) | AnnounceEvent::Takeover { .. } if self.refused => self.unseen = true, // The same instance over another route, which its subscriptions ride out, or one // continuing what was handed out. AnnounceEvent::Update(_) | AnnounceEvent::Takeover { .. } => return false, @@ -663,6 +675,70 @@ mod tests { result.expect("the selected refusal must answer"), Err(Error::Unroutable) )); + // The refusing route is not asked again, and the next announcement is handed out. + let ((), again) = futures::join!( + none(&mut announced), + kio::clock::timeout(Duration::from_secs(1), claim.requested_broadcast()) + ); + assert!(again.is_err(), "the refusing route was asked again"); + drop(claim); + let instance = Instance::random(); + let _published = origin.publish("live", route(&instance)).unwrap(); + assert_eq!( + available(&mut announced).await.info().instance.as_ref(), + Some(&instance) + ); + } + + /// A refusal is asked again once the same instance serves the path over another route. + #[kio::test] + async fn announced_retries_a_refusal_on_another_route_of_the_instance() { + let origin = Hop::new(1).unwrap().produce(); + let instance = Instance::random(); + let claim = origin.requests_route("", route(&instance), None).unwrap(); + let mut announced = origin.consume().follow("live", None).unwrap(); + let (result, ()) = futures::join!( + kio::clock::timeout(Duration::from_secs(1), announced.available()), + async { + requested(&claim).await.reject(Error::Unroutable); + } + ); + assert!(matches!( + result.expect("the refusal must answer"), + Err(Error::Unroutable) + )); + let _exact = origin.publish("live", route(&instance)).unwrap(); + assert_eq!( + available(&mut announced).await.info().instance.as_ref(), + Some(&instance) + ); + } + + /// A refusal after the instance handed out closed is not asked again on every call: the + /// closed instance's handover is settled until the serving route changes. + #[kio::test] + async fn announced_waits_after_refusing_a_closed_instances_successor() { + let origin = Hop::new(1).unwrap().produce(); + let claim = origin.requests_route("", route(&Instance::random()), None).unwrap(); + let exact = origin.publish("live", route(&Instance::random())).unwrap(); + let mut announced = origin.consume().follow("live", None).unwrap(); + let _first = available(&mut announced).await; + drop(exact); + let (result, ()) = futures::join!( + kio::clock::timeout(Duration::from_secs(1), announced.available()), + async { + requested(&claim).await.reject(Error::Unroutable); + } + ); + assert!(matches!( + result.expect("the refusal must answer"), + Err(Error::Unroutable) + )); + let ((), again) = futures::join!( + none(&mut announced), + kio::clock::timeout(Duration::from_secs(1), claim.requested_broadcast()) + ); + assert!(again.is_err(), "the refusing route was asked again"); } /// A path no pattern can spell is refused rather than followed through the whole scope, diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9ce45cd7c68..b08a1aaec0a 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -6889,7 +6889,7 @@ impl Consumer { /// /// The first [`available`](Following::poll_available) resolves the instance serving the /// path once one is announced, retrying route churn but returning a selected handler's - /// refusal. Each later call waits for another instance to take over (an + /// refusal; the call after a refusal waits for the path's next announcement change. Each later call waits for another instance to take over (an /// [`AnnounceEvent::Restart`]), or for the path to be served again after an /// [`AnnounceEvent::End`], even by the same instance. Drop what was read from the /// previous broadcast then: its tracks end with an error once its routes go. diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 2eb1bb4c3bd..21abed77acb 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -59,7 +59,7 @@ bytes = { workspace = true } bytesize = "2.4.2" futures = { workspace = true } hex = { workspace = true } -http-body = "1.0.1" +http-body = { workspace = true } # A 1.0 prerelease: the latest stable (0.16) is still on reqwest 0.12 and # reqwest-middleware 0.4, while this line matches reqwest 0.13 below. http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-moka", "reqwest-middleware", "url-standard"], default-features = false } diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 93538f69017..62c06d51f4a 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -142,10 +142,10 @@ case "$1" in # The allocation/window/play-buffer/batch/receive-bursts tables keep their stdout in the run; # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-crypto -p moq-json -p moq-archive -p moq-cli -p moq-mux -p moq-audio \ - -p moq-quic -p moq-sock -p moq-tokio -p moq-transcode --features moq-quic/fuzz \ + -p moq-quic -p moq-sock -p moq-tokio -p moq-transcode -p moq-hls --features moq-quic/fuzz \ --bench protect --bench sign --bench codec --bench table --bench allocations --bench window \ --bench producer --bench play_buffer --bench jitter --bench timeline --bench decode_clock --bench ts_programs --bench ts_video \ - --bench bus --bench batches --bench parameter_sets --bench sent_packets --bench demux --bench receive_bursts --bench ladder --bench fanout + --bench bus --bench batches --bench parameter_sets --bench sent_packets --bench demux --bench receive_bursts --bench ladder --bench fanout --bench contributions criterion audio-capture -p moq-audio --features capture --bench capture # moq-auth's published release is its own dev-dependency, so `-p moq-auth` is ambiguous. # The concurrent cases' latency percentiles stay in the run's stdout.