From 44b22f53d2e76a9d5f9d4cdfb21262b400ca3970 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:37:31 -0700 Subject: [PATCH 1/9] quest: claim stats-epoch Co-Authored-By: Claude Opus 5.5 From c5a43c0891cec8bb3c77e3d76c8eab57e8b839c1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:47:31 -0700 Subject: [PATCH 2/9] feat(stats)!: publish each group announcement under its own epoch Stats broadcasts move to [/]/node//@, minted per group announcement, so a restarted node or a group returning from idle never reuses a broadcast name or the group numbers a relay cached under it. NodePath gains the parsed epoch, and the aggregator treats each epoch as its own node. Co-Authored-By: Claude Opus 5.5 --- demo/relay/localhost.toml | 6 +- demo/web/src/stats.html | 2 +- demo/web/src/stats.ts | 12 +- doc/bin/inspect.md | 11 +- doc/bin/relay/config.md | 6 +- doc/concept/stats.md | 29 ++-- quest/m0/broadcast-epoch/README.md | 14 +- quest/m0/broadcast-epoch/stats-epoch.md | 49 ------- quest/m0/broadcast-epoch/stats-split.md | 8 +- rs/moq-relay/src/cluster.rs | 24 ++-- rs/moq-relay/src/stats.rs | 25 ++-- rs/moq-stats/src/aggregate.rs | 75 +++++----- rs/moq-stats/src/consume.rs | 2 +- rs/moq-stats/src/lib.rs | 73 ++++++---- rs/moq-stats/src/produce.rs | 179 +++++++++++++++--------- 15 files changed, 277 insertions(+), 238 deletions(-) delete mode 100644 quest/m0/broadcast-epoch/stats-epoch.md diff --git a/demo/relay/localhost.toml b/demo/relay/localhost.toml index 85d25bac2cb..65879f25793 100644 --- a/demo/relay/localhost.toml +++ b/demo/relay/localhost.toml @@ -28,10 +28,10 @@ public = "**" # Publish a single per-node stats broadcast on the cluster origin: # -# .stats/node/local/host +# .stats/node/local/host/@ # -# carrying publisher.json, subscriber.json, and sessions.json tracks. See -# doc/bin/relay/config.md for the field semantics. +# under a fresh epoch each run, carrying publisher.json, subscriber.json, and +# sessions.json tracks. See doc/bin/relay/config.md for the field semantics. [stats] enabled = true diff --git a/demo/web/src/stats.html b/demo/web/src/stats.html index 132e6102a96..537d6491a06 100644 --- a/demo/web/src/stats.html +++ b/demo/web/src/stats.html @@ -14,7 +14,7 @@

MoQ Relay Stats

-

Live cluster activity, auto-discovered under .stats/node.

+

Live cluster activity, auto-discovered under .stats/node, one card per node epoch.

` carrying JSON tracks that snapshot current activity. We - * auto-discover all of those nodes (announcements under `.stats/node`), so this - * works for a single relay and for a cluster alike, then aggregate each node and - * let you drill into one. + * `.stats/node//@` carrying JSON tracks that snapshot current + * activity, under a fresh epoch each run. We auto-discover all of those + * broadcasts (announcements under `.stats/node`), so this works for a single + * relay and for a cluster alike, then aggregate each node and let you drill into + * one. Nodes are keyed by `/@`, so a restarted relay is a new + * entry rather than a counter reset. * * Per-node tracks we read: * publisher.json egress (relay -> downstream viewers) @@ -100,7 +102,7 @@ const isSystem = (path: string) => path.startsWith("."); // ---- State ---------------------------------------------------------------- -// Discovered nodes -> their latest stats frames. +// Discovered `/@` keys -> their latest stats frames. const nodeStats = new Signals.Signal>({}); const selectedNode = new Signals.Signal(undefined); diff --git a/doc/bin/inspect.md b/doc/bin/inspect.md index d2e3420e8dd..28cd8d19575 100644 --- a/doc/bin/inspect.md +++ b/doc/bin/inspect.md @@ -69,13 +69,14 @@ Dial the `.stats` path, which the token must cover: ```bash moq --connect http://localhost:4443/.stats announced -moq --connect http://localhost:4443/.stats --broadcast node/local/host fetch publisher.json | jq -curl http://localhost:4443/fetch/.stats/node/local/host/publisher.json | jq +moq --connect http://localhost:4443/.stats --broadcast node/local/host/@ fetch publisher.json | jq +curl http://localhost:4443/fetch/.stats/node/local/host/@/publisher.json | jq ``` -The node name (`local/host` here) is `stats.node`. Each fetch returns the -newest snapshot: a JSON object of cumulative counters keyed by broadcast path. -`publisher.json` is egress, `subscriber.json` is ingress, and `sessions.json` +The node name (`local/host` here) is `stats.node`, and `` is the +UUID that `announced` lists after it, new each time the relay starts. Each +fetch returns the newest snapshot: a JSON object of cumulative counters keyed +by broadcast path. `publisher.json` is egress, `subscriber.json` is ingress, and `sessions.json` counts sessions per auth root. Fetch twice and divide by the interval for a rate. The `.json.z` twins carry compressed patches, which these tools print as raw bytes. [Stats](/concept/stats) describes every field. diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index be3b7bd7026..55750fad678 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -220,7 +220,7 @@ the task running for as long as you hold it. ```toml [stats] enabled = true -prefix = ".stats" # Broadcasts appear under /node/. +prefix = ".stats" # Broadcasts appear under /node//@. interval = 1 # Seconds between snapshots. node = "sjc/1" # Disambiguates relays sharing a cluster. depth = 1 # Also bucket by the first N path segments (per tenant). @@ -231,7 +231,9 @@ Each node publishes `publisher.json`, `subscriber.json`, and `sessions.json` tracks (plus compressed `.json.z` twins) of cumulative counters per broadcast and auth root, split by a **tier** label chosen by the auth server's grant or `--cluster-tier`, which is what makes billing per customer or per region -possible. [Stats](/concept/stats) describes the paths, tracks, and encodings; +possible. Each run, and each group returning after its linger, announces a +fresh `@`, so a restart never reuses a broadcast name. +[Stats](/concept/stats) describes the paths, tracks, and encodings; read them with the [`moq-stats`](https://docs.rs/moq-stats) crate. ## \[iroh] diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 8d23a0b01e9..2544b08300c 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -20,12 +20,17 @@ it. Traffic under the prefix is never counted, so serving stats doesn't generate more stats. ```text -/node/ depth 0: one broadcast per node -//node/ depth N: one broadcast per group per node +/node//@ depth 0: one broadcast per node +//node//@ depth N: one broadcast per group per node ``` - `` tells relays sharing a cluster apart. It may span several segments - (`sjc/1`), and is omitted along with its slash when unset: `/node`. + (`sjc/1`), and is omitted along with its slash when unset: + `/node/@`. +- `@` is a [publisher epoch](/concept/moq-lite#publisher-epochs), minted + each time the broadcast is announced. A restarted node or a group returning + from idle publishes under a new name, so no relay serves it groups cached + under the old one. A reader treats each epoch as its own set of counters. - `` is the first `depth` segments of each broadcast path (for traffic) or auth root (for sessions), so a consumer can scope an announce to one tenant. A path shorter than `depth` groups under all of its segments. @@ -39,10 +44,11 @@ for a linger (five minutes by default) after its last one leaves. A group that returns within the linger keeps its broadcast, so viewer churn doesn't unannounce and re-announce it across the mesh; while it lingers empty, its tracks hold `{}`. Once the linger elapses with the group still empty, the -broadcast is unannounced. Group numbers keep increasing across recreated -tracks and group broadcasts for the producer's life; they may have gaps. A -recreated compressed track starts a new group with a full snapshot, never a -delta whose compression state belonged to its previous writer. +broadcast is unannounced and its counters dropped; a group that returns later +announces a new epoch counted from zero. Within one epoch, group numbers keep +increasing across recreated tracks; they may have gaps. A recreated compressed +track starts a new group with a full snapshot, never a delta whose compression +state belonged to its previous writer. ## Tracks @@ -132,9 +138,12 @@ Every counter is a cumulative, monotonic unsigned integer. A rate is the difference between two frames divided by the time between them, and a live count is started minus ended. A frame never shows ended above started. -A counter going **down** means the relay restarted or the entry was dropped -and re-created. Treat it as the start of a fresh segment rather than a -negative rate. +A new epoch starts every counter from zero, so a reader summing a node over +time adds each epoch's counters rather than diffing across them. Within one +epoch, a counter going **down** means the entry was dropped and re-created. +Treat it as the start of a fresh segment rather than a negative rate. A relay +that predates epochs publishes without the `@` segment, and a counter +going down there also means it restarted. A reader ignores unknown fields, so a newer relay can add counters, and defaults a missing field to zero, so it can read an older relay. diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index b7ccfa5f5c0..9db89ffcc29 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -51,10 +51,10 @@ Decided: bare-name viewer behind one needs a publisher that opts out with the raw prefix route. Document this rather than promise it works. - Publishers on the default publish path, such as moq-boy and moq-room, - inherit the epoch from Origin. moq-stats mints its own through - [Stats epochs](/quest/m0/broadcast-epoch/stats-epoch.md), which also gates the release - (decided 2026-10-04): a restarted stats node under a reused name stalls its - viewers the same way. + inherit the epoch from Origin. moq-stats mints its own per + [group announcement](/doc/concept/stats.md#broadcasts), which also gates the + release (decided 2026-10-04): a restarted stats node under a reused name + stalls its viewers the same way. - Decided in the 2026-10-05 audit: the m1 quests gating this line (stats epochs, the bounded stats aggregate it requires, and retracted demand release) moved under it, and the OBS half of GStreamer and OBS moved to m1 @@ -74,6 +74,11 @@ This README owns: - A `doc/concept` page on broadcast naming: what an epoch is, publish and consume behavior, takeover and fallback, bare-path resolution, and the prefix-route opt-out. +- When the release cut carries stats epochs, drop + [#4810](https://github.com/moq-dev/moq/pull/4810)'s wall-clock group seed + and its `doc/concept/stats.md` sentence from `release`; release-to-main + back-merges keep `main`'s `rs/moq-stats` and `doc/concept/stats.md` until + then. MoQ Pro's VOD `storage.json` moves from the same seed to its own epoch. ## Required @@ -84,5 +89,4 @@ This README owns: - [Bindings](/quest/m0/broadcast-epoch/bindings.md) - moq-ffi and every wrapper expose the epoch and inherit the default - [GStreamer](/quest/m0/broadcast-epoch/gst.md) - moqsink publishes each run under a fresh epoch - [Remove `--hop`](/quest/m0/broadcast-epoch/hop-removal.md) - `moq` takes an optional `--epoch` instead of `--hop`, a plain publisher declares a random Hop ID, and the per-session hop stamp is gone -- [Stats epochs](/quest/m0/broadcast-epoch/stats-epoch.md) - moq-stats publishes each group announcement under its own epoch, so neither a restarted node nor a returning idle group stalls its viewers - [Stats totals and prefix tracks](/quest/m0/broadcast-epoch/stats-split.md) - the same release retires the per-path stats maps for totals and on-demand prefix tracks (decided 2026-10-05) diff --git a/quest/m0/broadcast-epoch/stats-epoch.md b/quest/m0/broadcast-epoch/stats-epoch.md deleted file mode 100644 index 85f21240f1f..00000000000 --- a/quest/m0/broadcast-epoch/stats-epoch.md +++ /dev/null @@ -1,49 +0,0 @@ -# [S] Stats epochs - -## Goal - -Each stats group broadcast publishes under its own epoch, -`[/]/node//@`, minted each time the group is -announced, so neither a restarted node nor a group returning from idle reuses -a broadcast name or the group numbers a relay cached under it. - -## Plan - -- Use the shared `moq_net::Epoch`, not a stats-local UUID (decided - 2026-10-02). -- One epoch per group announcement (decided 2026-10-05, replacing 2026-10-02's - one epoch per `Producer`). A group that goes idle unannounces after its - linger and drops its totals; when it returns it announces under a new epoch - counted from zero. Why: a producer never holds an idle group's state, so its - memory is bounded by active and lingering groups, and readers see every - reset as a new path rather than detecting one. The cost: a reader that - misses every frame across the linger loses that epoch's tail; billing - under-bills by that tail, consistent with the 0-bill baseline. - At depth 0 the single broadcast never unannounces, so its epoch still lasts - the producer's life. -- [#4739](https://github.com/moq-dev/moq/pull/4739)'s original commits are - prior art: path syntax, `parse_node_path`, the - aggregator's restarted-epoch handling, and the tests. The PR landed only the - producer-wide group allocator. -- Decide what an unset node becomes once the epoch follows it (#4739 used - `local`). -- The aggregator treats a new epoch as a new node: its counters add to the - merged total instead of regressing it, and the old epoch folds into the - retired total once `aggregate::Config::grace` elapses. Test it. -- Expose each group broadcast's epoch (a per-group accessor replacing - #4739's `Producer::epoch()`) so a consumer can tell epochs apart. - MoQ Pro's VOD `storage.json` mints its own epoch rather than sharing a - producer-wide one, which no longer exists. -- `release` carries a temporary seed instead - ([#4810](https://github.com/moq-dev/moq/pull/4810)): a producer's first group - number is wall-clock microseconds, so a restarted node numbers above its - previous run. Epochs replace it, so it never reaches `main`: release-to-main - back-merges keep `main`'s `rs/moq-stats` and `doc/concept/stats.md`. Drop the - seed and its `doc/concept/stats.md` sentence from `release` when this ships - there. MoQ Pro's VOD `storage.json` seeds the same way and moves to its - own epoch with it. -- demo/web stats keys include the epoch. Update `doc/concept/stats.md`, - `doc/bin/relay/config.md`, and the relay stats config docs. - -Public API: path shape change for every stats consumer. Wire: stats broadcast -names gain a trailing epoch segment. diff --git a/quest/m0/broadcast-epoch/stats-split.md b/quest/m0/broadcast-epoch/stats-split.md index 51d3addf001..4488d1d3eeb 100644 --- a/quest/m0/broadcast-epoch/stats-split.md +++ b/quest/m0/broadcast-epoch/stats-split.md @@ -19,7 +19,7 @@ Decided 2026-10-05 (planned from moq-dev/moq.pro#2202): reader undercounts. The maps also grow with every live broadcast, and readers like billing want only per-project sums. - **Totals.** Per group broadcast (`[/]/node//@`), - per tier and role, cumulative within the [stats epoch](/quest/m0/broadcast-epoch/stats-epoch.md). + per tier and role, cumulative within the [stats epoch](/doc/concept/stats.md#broadcasts). Every group announcement, and so every restart, starts a new epoch counted from zero; nothing is serialized to disk. `Registry`'s unpruned lifetime totals (`rs/moq-net/src/stats.rs`) are node-wide, per tier and role only, so they @@ -32,7 +32,7 @@ Decided 2026-10-05 (planned from moq-dev/moq.pro#2202): ends. Today a path that left drops out of frames while the group lingers; with totals, a group that returns within the linger continues its totals. A zero linger is valid: a returning group then always takes a new epoch. After the linger it unannounces and drops its - totals; a return announces under a new [epoch](/quest/m0/broadcast-epoch/stats-epoch.md) + totals; a return announces under a new [epoch](/doc/concept/stats.md#broadcasts) counted from zero. Totals are cumulative and a lingering group's frames all repeat its final totals, so a reader that misses every frame across the linger loses that epoch's tail; billing under-bills by that tail, @@ -106,10 +106,6 @@ Broadcasts page reads one prefix per visible row and group header, its `announce reads totals only, and its customer stats feed serves this format summed across nodes. -## Required - -- [Stats epochs](/quest/m0/broadcast-epoch/stats-epoch.md) - totals are cumulative within an epoch - ## Related - [Media stats](/quest/m1/stats/README.md) - publisher and viewer media stats diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 30c5ff0b180..3f3cee41268 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2007,6 +2007,18 @@ mod tests { } } + /// The path, epoch included, that the stats broadcast of node `test` is announced under. + async fn stats_path(consumer: &origin::Consumer) -> moq_net::PathOwned { + let mut announced = consumer.consume().with_hidden(true).announced(); + loop { + let (update, active) = next_update(&mut announced).await.expect("origin closed"); + let (name, epoch) = update.prefix.split_epoch(); + if active && name.as_str() == ".stats/node/test" && epoch.is_some() { + return update.prefix.to_owned(); + } + } + } + /// The next announcement without blocking, skipping the caught-up marker. fn try_next_announced(announced: &mut moq_net::announce::Consumer) -> Option { loop { @@ -2096,12 +2108,10 @@ mod tests { let stats = config.build(cluster.origin.clone()); let cluster = cluster.with_stats(stats); - let path = moq_net::Path::new(".stats").join("node").join("test"); let consumer = cluster.origin.consume(); - tokio::time::timeout(std::time::Duration::from_secs(5), consumer.routed(&path)) + let path = tokio::time::timeout(std::time::Duration::from_secs(5), stats_path(&consumer)) .await - .expect("stats broadcast announced within 5s") - .expect("stats broadcast present"); + .expect("stats broadcast announced within 5s"); let broadcast = consumer .request_broadcast(&path) .await @@ -2665,11 +2675,9 @@ mod tests { .expect("broadcast resolves") .expect("broadcast present"); - let stats_path = moq_net::Path::new(".stats").join("node").join("test"); - tokio::time::timeout(Duration::from_secs(5), consumer.routed(&stats_path)) + tokio::time::timeout(Duration::from_secs(5), stats_path(&consumer)) .await - .expect("stats announced") - .expect("stats present"); + .expect("stats announced"); } /// A reserved (0) or out-of-range (>= 2^62) `cluster.id` is rejected rather diff --git a/rs/moq-relay/src/stats.rs b/rs/moq-relay/src/stats.rs index 12fa7084239..3a504c8131a 100644 --- a/rs/moq-relay/src/stats.rs +++ b/rs/moq-relay/src/stats.rs @@ -14,8 +14,9 @@ use serde::{Deserialize, Serialize}; /// /// Set `enabled = true` to attach a [`moq_stats::Producer`] to every session /// the relay accepts (and every cluster dial). The producer publishes a single -/// `/node/` broadcast (or `/node` when [`Self::node`] is -/// unset) on the cluster origin. Each broadcast carries plain `.json` tracks +/// `/node//@` broadcast (or `/node/@` when +/// [`Self::node`] is unset) on the cluster origin, under a fresh epoch each run. +/// Each broadcast carries plain `.json` tracks /// (a JSON map of broadcast path to a cumulative counter snapshot per frame) /// plus compressed `.json.z` siblings; see `moq_stats` for the wire format and /// per-field semantics. @@ -55,8 +56,8 @@ pub struct Config { /// Node identifier appended to the advertised stats path to disambiguate /// broadcasts when multiple relays share a cluster origin. Without this, - /// peer relays would publish to the same `/node` path and the - /// origin's single-source delivery would drop all but one. + /// peer relays publish under `/node/@` and a reader can't + /// tell which epoch belongs to which relay. /// /// May be multi-segment (e.g. `sjc/1`, `sjc/2`) when a region has multiple /// hosts; the segments nest under a shared region key on the advertised @@ -65,11 +66,12 @@ pub struct Config { pub node: Option, /// Number of leading broadcast-path segments to bucket stats by, one - /// broadcast per bucket at `//node/`. Defaults to 0: a - /// single `/node/` broadcast for the whole node. Set to 1 to - /// publish a per-first-segment broadcast (e.g. per tenant), so a consumer can - /// announce-scope to just that group rather than slurping every node's full - /// stats. See [`moq_stats::produce::Config::depth`]. + /// broadcast per bucket at `//node//@`. Defaults + /// to 0: a single `/node//@` broadcast for the whole + /// node. Set to 1 to publish a per-first-segment broadcast (e.g. per + /// tenant), so a consumer can announce-scope to just that group rather than + /// slurping every node's full stats. See + /// [`moq_stats::produce::Config::depth`]. #[usage( long = "stats-depth", env = "MOQ_STATS_DEPTH", @@ -81,8 +83,9 @@ pub struct Config { /// How long a group's stats broadcast stays announced after the group's /// last session and traffic leave, e.g. "5m" or "30s". Defaults to 5 /// minutes. A group that returns within it keeps its broadcast, so viewer - /// churn doesn't unannounce and re-announce it across the mesh. Only applies - /// at `depth` 1 or more. See [`moq_stats::produce::Config::linger`]. + /// churn doesn't unannounce and re-announce it across the mesh; one that + /// returns after it announces under a new epoch. Only applies at `depth` 1 + /// or more. See [`moq_stats::produce::Config::linger`]. #[usage(skip)] #[serde(with = "crate::duration::serde_option")] pub linger: Option, diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 50001c3ffa2..2bfce5c7a34 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -1,7 +1,7 @@ //! The aggregating half: fold a group's per-node stats broadcasts into one view. //! //! A single-broadcast [`Consumer`](crate::Consumer) reads one -//! `//node/` broadcast. This reader watches an origin's +//! `//node//@` broadcast. This reader watches an origin's //! announce stream for *every* node broadcast in a group and folds their //! cumulative counters into one merged frame per `(tier, role)`, so a downstream //! sees a project's whole live traffic as if it came from a single node. @@ -102,10 +102,11 @@ impl Default for Config { /// boot-lifetime counters intact never looks like new traffic. After the grace, /// a departed node's contribution folds into one retired total and the node is /// forgotten, so memory follows live nodes and the keys ever reported, not -/// every node ever seen. A node -/// returning within the grace with a lower counter (a restarted relay) -/// regresses the merged counter, the same reset contract a single node's own -/// restart follows. Presence is not sticky: a departed node stops counting +/// every node ever seen. Each epoch is its own node: a restarted producer, or a +/// group returning from idle, announces a new epoch whose counters add to the +/// old epoch's kept contribution instead of regressing it. Only a producer that +/// predates epochs and returns within the grace with a lower counter regresses +/// the merged counter. Presence is not sticky: a departed node stops counting /// sessions immediately. pub struct Consumer { origin: origin::Consumer, @@ -299,7 +300,7 @@ struct Merged { name: String, config: moq_json::snapshot::consumer::Config, /// One entry per live or recently departed node broadcast, keyed by - /// absolute announced path. + /// absolute announced path, so each epoch is its own entry. nodes: HashMap>, /// How long a departed node waits before folding into `retired`. grace: Duration, @@ -582,7 +583,7 @@ mod tests { /// A stats producer publishing one node's broadcasts on `origin`, grouped at /// depth 1 (so feeding a broadcast under `/...` announces - /// `.stats//node/`). + /// `.stats//node//@`). fn node_producer(origin: &origin::Producer, node: &str) -> Producer { Producer::new( produce::Config::new() @@ -672,7 +673,8 @@ mod tests { /// A hand-published node broadcast at `.stats//node/` with a /// plain default-tier traffic track, so a test controls the exact frames and /// can fail one node's reader alone (the registry-driven producer only - /// publishes whole broadcasts). Dropping it unannounces the node. + /// publishes whole broadcasts). Dropping it unannounces the node, and a new + /// one at the same path is that node reconnecting with its epoch intact. #[allow(dead_code)] struct NodeBroadcast { source: broadcast::Producer, @@ -774,10 +776,11 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn reannounce_with_higher_counter_stays_monotonic() { - // A node's stats session reconnects with its cumulative counters intact - // and still growing: the total holds through the swap, then advances, - // never dipping. + async fn restarted_producer_adds_a_new_epoch() { + // A restarted node announces a new epoch with fresh counters. The new + // epoch adds to the old one's kept contribution instead of regressing + // the total, and the old epoch folds into the retired total once the + // grace elapses. let origin = produce_origin(); let node_a = node_producer(&origin, "a"); let node_b = node_producer(&origin, "b"); @@ -786,54 +789,48 @@ mod tests { let _fb = feed(&node_b, Tier::default(), "acme", "acme/room", 40).await; drive_tick().await; - let agg = Consumer::new(origin.consume(), Config::new().with_depth(1)); + let agg = Consumer::new(origin.consume(), Config::new().with_depth(1).with_grace(GRACE)); let mut traffic = agg.traffic(&Tier::default(), Role::Publisher); read_until_bytes(&mut traffic, "acme/room", 140).await; - // Node A's broadcast goes away and comes back with a higher counter. drop(fa); drop(node_a); drive_tick().await; let node_a = node_producer(&origin, "a"); - let _fa = feed(&node_a, Tier::default(), "acme", "acme/room", 120).await; + let _fa = feed(&node_a, Tier::default(), "acme", "acme/room", 30).await; drive_tick().await; - // The kept contribution holds the total at 140 until the new frame - // replaces it, landing on 120 + 40. - let frame = read_monotonic_until(&mut traffic, "acme/room", 140, 160).await; - assert_eq!(frame.get("acme/room").expect("entry").bytes, 160); + let frame = read_monotonic_until(&mut traffic, "acme/room", 140, 170).await; + assert_eq!(frame.get("acme/room").expect("entry").bytes, 170, "100 + 30 + 40"); + assert_eq!(traffic.inner.nodes.len(), 3, "the old epoch lingers for its grace"); + + tokio::time::advance(GRACE).await; + settle(&mut traffic).await; + assert_eq!(traffic.inner.nodes.len(), 2, "the old epoch folded"); + assert_eq!(traffic.inner.retired.get("acme/room").map(|t| t.bytes), Some(100)); } #[tokio::test(start_paused = true)] - async fn restarted_node_regresses_the_total() { - // A node that returns with a fresh counter (it restarted) replaces its - // contribution wholesale: the total regresses once, the existing - // fresh-segment contract. + async fn legacy_restart_regresses_the_total() { + // A producer that predates epochs returns at the same path with a fresh + // counter: it replaces its contribution wholesale, and the total + // regresses once, the fresh-segment contract. let origin = produce_origin(); - let node_a = node_producer(&origin, "a"); - let node_b = node_producer(&origin, "b"); - - let fa = feed(&node_a, Tier::default(), "acme", "acme/room", 100).await; - let _fb = feed(&node_b, Tier::default(), "acme", "acme/room", 40).await; - drive_tick().await; + let mut node_a = NodeBroadcast::new(&origin, "acme", "a"); + let mut node_b = NodeBroadcast::new(&origin, "acme", "b"); + node_a.publish("acme/room", 100); + node_b.publish("acme/room", 40); let agg = Consumer::new(origin.consume(), Config::new().with_depth(1)); let mut traffic = agg.traffic(&Tier::default(), Role::Publisher); read_until_bytes(&mut traffic, "acme/room", 140).await; - // Node A restarts: its broadcast returns with a lower counter. - drop(fa); drop(node_a); - drive_tick().await; - - let node_a = node_producer(&origin, "a"); - let _fa = feed(&node_a, Tier::default(), "acme", "acme/room", 30).await; - drive_tick().await; + settle(&mut traffic).await; + let mut node_a = NodeBroadcast::new(&origin, "acme", "a"); + node_a.publish("acme/room", 30); - // The restarted frame replaces A's contribution, so the total drops to - // 30 + 40, a genuine per-node regression downstream treats as a fresh - // segment. An earlier frame may retire A's live gauges first. loop { let frame = traffic.next().await.expect("read").expect("frame"); if frame.get("acme/room").map(|t| t.bytes) == Some(70) { diff --git a/rs/moq-stats/src/consume.rs b/rs/moq-stats/src/consume.rs index 068fa32211a..7dd4bfceade 100644 --- a/rs/moq-stats/src/consume.rs +++ b/rs/moq-stats/src/consume.rs @@ -29,7 +29,7 @@ impl Config { } } -/// Reads one published stats broadcast (a `/node/` announce), +/// Reads one published stats broadcast (a `/node//@` announce), /// yielding typed frames per track. /// /// Subscribe to the traffic and session tracks you care about with diff --git a/rs/moq-stats/src/lib.rs b/rs/moq-stats/src/lib.rs index 99b2411232e..d43bb639be8 100644 --- a/rs/moq-stats/src/lib.rs +++ b/rs/moq-stats/src/lib.rs @@ -14,10 +14,12 @@ //! //! # Wire format //! -//! A [`Producer`] publishes one broadcast per node at `/node/` -//! (default prefix `.stats`), or one per group of leading broadcast-path -//! segments at `//node/`; parse announce paths back with -//! [`parse_node_path`]. Each [`Tier`] carries `publisher.json`, +//! A [`Producer`] publishes one broadcast per node at +//! `/node//@` (default prefix `.stats`), or one per group +//! of leading broadcast-path segments at `//node//@`; +//! parse announce paths back with [`parse_node_path`]. Each announcement mints +//! a fresh [`Epoch`], so a restarted node or a group returning from idle never +//! reuses a broadcast name. Each [`Tier`] carries `publisher.json`, //! `subscriber.json`, and `sessions.json` tracks of cumulative [`Traffic`] and //! [`Presence`] counters, plus `.json.z` siblings encoded with //! [`moq_json::snapshot`]; compute names with [`traffic_track`] / @@ -37,7 +39,7 @@ use std::collections::BTreeMap; /// can depend on this crate alone. pub use moq_net::stats::{Handle, Presence, Registry, Role, Tier, Traffic}; -use moq_net::{AsPath, Path, PathOwned}; +use moq_net::{AsPath, Epoch, Path, PathOwned}; /// One frame off a traffic track: cumulative counters keyed by broadcast path. pub type TrafficFrame = BTreeMap; @@ -70,7 +72,7 @@ pub fn sessions_track(tier: &Tier, compressed: bool) -> String { name } -/// A parsed stats broadcast path: `[/]/node[/]`. +/// A parsed stats broadcast path: `[/]/node[/][/@]`. /// See [`parse_node_path`]. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] @@ -80,10 +82,13 @@ pub struct NodePath { pub group: PathOwned, /// The node suffix, empty when the producer has no node configured. pub node: PathOwned, + /// The announcement this broadcast belongs to; a new epoch is a new set of + /// counters. `None` from a producer that predates epochs. + pub epoch: Option, } /// Parse a stats broadcast announce path published under `prefix` with the -/// given grouping `depth`, splitting it into its group and node parts. +/// given grouping `depth`, splitting it into its group, node, and epoch parts. /// /// Returns `None` when the path is not under `prefix` or has no `node` /// category segment where one is expected (which also filters out sibling @@ -116,9 +121,12 @@ pub fn parse_node_path(prefix: impl AsPath, depth: usize, path: impl AsPath) -> } let node = segments.collect::>().join("/"); + let node = Path::new(&node); + let (node, epoch) = node.split_epoch(); Some(NodePath { group: Path::new(&group.join("/")).to_owned(), - node: Path::new(&node).to_owned(), + node: node.to_owned(), + epoch, }) } @@ -144,40 +152,51 @@ mod tests { #[test] fn parse_node_path_variants() { - let parse = |depth, path| parse_node_path(".stats", depth, path); + let parse = |depth, path: &str| parse_node_path(".stats", depth, path); + let epoch = Epoch::mint(); + let node = |group: &str, node: &str, epoch: Option<&Epoch>| { + Some(NodePath { + group: Path::new(group).to_owned(), + node: Path::new(node).to_owned(), + epoch: epoch.cloned(), + }) + }; // Depth 0: no group segment. assert_eq!( - parse(0, ".stats/node/sjc"), - Some(NodePath { - group: Path::empty().to_owned(), - node: Path::new("sjc").to_owned(), - }) + parse(0, &format!(".stats/node/sjc/@{epoch}")), + node("", "sjc", Some(&epoch)) ); assert_eq!( - parse(0, ".stats/node/sjc/1").unwrap().node, - Path::new("sjc/1").to_owned(), + parse(0, &format!(".stats/node/sjc/1/@{epoch}")), + node("", "sjc/1", Some(&epoch)), "multi-segment node" ); assert_eq!( - parse(0, ".stats/node"), - Some(NodePath { - group: Path::empty().to_owned(), - node: Path::empty().to_owned(), - }), + parse(0, &format!(".stats/node/@{epoch}")), + node("", "", Some(&epoch)), "nodeless path" ); // Depth 1: one group segment, as published per tenant. assert_eq!( - parse(1, ".stats/acme/node/sjc"), - Some(NodePath { - group: Path::new("acme").to_owned(), - node: Path::new("sjc").to_owned(), - }) + parse(1, &format!(".stats/acme/node/sjc/@{epoch}")), + node("acme", "sjc", Some(&epoch)) ); // A shorter broadcast path yields a shorter group; still parses. - assert_eq!(parse(1, ".stats/node/sjc").unwrap().group, Path::empty().to_owned()); + assert_eq!( + parse(1, &format!(".stats/node/sjc/@{epoch}")), + node("", "sjc", Some(&epoch)) + ); + + // A producer that predates epochs: the node is the whole suffix. + assert_eq!(parse(0, ".stats/node/sjc"), node("", "sjc", None)); + assert_eq!(parse(0, ".stats/node"), node("", "", None)); + assert_eq!( + parse(0, ".stats/node/sjc/@local"), + node("", "sjc/@local", None), + "not a UUIDv7" + ); // Not ours: wrong prefix, sibling category, group deeper than depth. assert_eq!(parse(0, "other/node/sjc"), None); diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index d54c8f36a53..32aea6ae828 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -8,7 +8,7 @@ use std::time::Duration; use std::task::Poll; use moq_net::stats::{Presence, Registry, Report, Role, Tier, Traffic}; -use moq_net::{Path, PathOwned, broadcast, kio, origin, track}; +use moq_net::{Epoch, Path, PathOwned, broadcast, kio, origin, track}; use serde::Serialize; use web_async::spawn; use web_async::time::Instant; @@ -30,9 +30,10 @@ pub struct Config { /// When `None`, [`Producer::new`] spawns no task and publishes nothing. pub origin: Option, /// Top-level path stats are published under (default `.stats`). The full - /// advertised path is `/node/` (or `/node` when - /// `node` is unset). Also the registry's exclude prefix, so serving a - /// stats broadcast doesn't generate more stats. + /// advertised path is `/node//@` (or + /// `/node/@` when `node` is unset), with a fresh epoch per + /// announcement. Also the registry's exclude prefix, so serving a stats + /// broadcast doesn't generate more stats. pub prefix: PathOwned, /// Node suffix that disambiguates broadcasts from different relays sharing a /// cluster origin. Set this on every node in multi-relay deployments. May be @@ -44,12 +45,13 @@ pub struct Config { pub interval: Duration, /// How many leading broadcast-path segments to use as a grouping key. /// - /// Default `0` publishes one `/node/` broadcast carrying every - /// path. `1` publishes one broadcast per first segment at - /// `//node/`, and larger values include more leading - /// segments. Group broadcasts are announced while their group has live traffic, - /// plus the [`Self::linger`]; at depth `0`, the single broadcast stays - /// announced for the producer's life. + /// Default `0` publishes one `/node//@` broadcast + /// carrying every path. `1` publishes one broadcast per first segment at + /// `//node//@`, and larger values include more + /// leading segments. Group broadcasts are announced while their group has + /// live traffic, plus the [`Self::linger`], and a group returning after that + /// announces under a new epoch counted from zero. At depth `0`, the single + /// broadcast and its epoch last for the producer's life. pub depth: usize, /// How long a group broadcast stays announced after its group's last entry /// leaves. Default 5 minutes. @@ -205,7 +207,6 @@ impl Producer { depth, linger, interval, - sequence: Arc::new(AtomicU64::new(0)), }; spawn(task.run(Arc::downgrade(&keepalive))); @@ -232,7 +233,6 @@ struct Task { depth: usize, linger: Duration, interval: Duration, - sequence: Arc, } impl Task { @@ -407,10 +407,11 @@ impl Serialize for Frame { } } -/// Writes encoded snapshots with group numbers from the producer-wide -/// allocator. The allocator outlives on-demand tracks and group broadcasts, so -/// a recreated track resumes past any group a subscriber has cached, without -/// retaining a tombstone per requested name. +/// Writes encoded snapshots with group numbers from its group broadcast's +/// allocator. The allocator outlives on-demand tracks, so a recreated track +/// resumes past any group a subscriber has cached, without retaining a +/// tombstone per requested name. A new group broadcast is a new epoch, so its +/// allocator starts over. struct Snapshot { track: track::Producer, encoder: moq_json::snapshot::Encoder>, @@ -793,7 +794,7 @@ impl TierNames { impl GroupPublisher { fn create(task: &Task, group: &Path) -> Option { - let advertised = advertised_path(&task.prefix, group, task.node()); + let advertised = advertised_path(&task.prefix, group, task.node(), &Epoch::mint()); let broadcast = match task.origin.publish(&advertised, origin::Route::default()) { Ok(broadcast) => broadcast, Err(err) => { @@ -803,14 +804,15 @@ impl GroupPublisher { }; tracing::debug!(advertised = %advertised, "stats: publishing broadcast"); - let mut traffic = TrackFamily::new(task.sequence.clone()); - let mut sessions = TrackFamily::new(task.sequence.clone()); + let sequence = Arc::new(AtomicU64::new(0)); + let mut traffic = TrackFamily::new(sequence.clone()); + let mut sessions = TrackFamily::new(sequence.clone()); // The default tier's tracks always exist, even while idle. let tier = Tier::default(); for role in [Role::Publisher, Role::Subscriber] { let name = traffic_track(&tier, role, false); - match TrackPair::create(&broadcast, &name, task.sequence.clone()) { + match TrackPair::create(&broadcast, &name, sequence.clone()) { Ok(pair) => { traffic.tracks.insert(name, pair); } @@ -821,7 +823,7 @@ impl GroupPublisher { } } let name = sessions_track(&tier, false); - match TrackPair::create(&broadcast, &name, task.sequence.clone()) { + match TrackPair::create(&broadcast, &name, sequence) { Ok(pair) => { sessions.tracks.insert(name, pair); } @@ -1083,8 +1085,8 @@ fn group_key(path: &str, depth: usize) -> &str { } } -fn advertised_path(prefix: &Path, group: &Path, node: Option<&str>) -> PathOwned { - // `//node/`. The group segment is empty at depth 0. +fn advertised_path(prefix: &Path, group: &Path, node: Option<&str>, epoch: &Epoch) -> PathOwned { + // `//node//@`. The group segment is empty at depth 0. // The fixed `node` category leaves room for sibling categories (e.g. // `//cluster` for relay-mesh stats) under the same prefix. let mut out = prefix.as_str().to_string(); @@ -1097,6 +1099,8 @@ fn advertised_path(prefix: &Path, group: &Path, node: Option<&str>) -> PathOwned out.push('/'); out.push_str(node); } + out.push_str("/@"); + out.push_str(epoch.as_str()); PathOwned::from(out) } @@ -1273,6 +1277,14 @@ mod tests { (update.prefix.as_str().to_string(), broadcast) } + /// `path` without its trailing epoch, which every stats broadcast carries. + fn name(path: &str) -> String { + let path = moq_net::Path::new(path); + let (name, epoch) = path.split_epoch(); + assert!(epoch.is_some(), "{path} has no epoch"); + name.as_str().to_string() + } + /// Advance past one publish interval so the task drains and writes frames. async fn drive_tick() { tokio::time::advance(Duration::from_millis(1100)).await; @@ -1351,10 +1363,24 @@ mod tests { #[tokio::test(start_paused = true)] async fn new_normalizes_and_drops_empty_node() { let (_producer, origin) = test_producer(Some("/sjc//1/")); - assert_eq!(announced(&origin).await.0, ".stats/node/sjc/1"); + assert_eq!(name(&announced(&origin).await.0), ".stats/node/sjc/1"); let (_producer, origin) = test_producer(Some("///")); - assert_eq!(announced(&origin).await.0, ".stats/node"); + assert_eq!(name(&announced(&origin).await.0), ".stats/node"); + } + + /// A restarted producer announces under a new epoch, so a relay never serves + /// it groups cached under its previous run. + #[tokio::test(start_paused = true)] + async fn restarted_producer_announces_a_new_epoch() { + let (first, origin) = test_producer(Some("sjc")); + let before = announced(&origin).await.0; + drop(first); + + let (_second, origin) = test_producer(Some("sjc")); + let after = announced(&origin).await.0; + assert_eq!(name(&before), name(&after)); + assert_ne!(before, after); } #[tokio::test(start_paused = true)] @@ -1366,14 +1392,14 @@ mod tests { let _f1 = feed(producer.registry(), Tier::default(), "foo/bar", true, 1, 8).await; let _f2 = feed(producer.registry(), Tier::default(), "baz/qux", true, 1, 8).await; - assert_eq!(announced(&origin).await.0, ".stats/node/sjc/1"); + assert_eq!(name(&announced(&origin).await.0), ".stats/node/sjc/1"); } #[tokio::test(start_paused = true)] async fn task_announces_without_node_suffix() { let (producer, origin) = test_producer(None); let _f = feed(producer.registry(), Tier::default(), "foo/bar", true, 1, 8).await; - assert_eq!(announced(&origin).await.0, ".stats/node"); + assert_eq!(name(&announced(&origin).await.0), ".stats/node"); } #[tokio::test(start_paused = true)] @@ -1535,16 +1561,22 @@ mod tests { (producer, origin, events) } - /// The announce events buffered so far, as `(path, active)`. + /// The announce events buffered so far, as `(path, active)` sorted by path. fn take_events(events: &mut announce::Consumer) -> Vec<(String, bool)> { use futures::FutureExt; let mut out = Vec::new(); while let Some(Some((route, active))) = next_update(events).now_or_never() { out.push((route.prefix.as_str().to_string(), active)); } + out.sort(); out } + /// `events` with each path's epoch stripped. + fn names(events: &[(String, bool)]) -> Vec<(String, bool)> { + events.iter().map(|(path, active)| (name(path), *active)).collect() + } + const ACME: &str = ".stats/acme/node/sjc"; const FEED: &str = ".stats/feed/node/sjc"; @@ -1557,11 +1589,21 @@ mod tests { // session rooted at `feed` (group `feed`). let first = feed(registry, Tier::default(), "acme/live", true, 1, 100).await; drive_tick().await; - let mut started = take_events(&mut events); - started.sort(); - assert_eq!(started, [(ACME.to_string(), true), (FEED.to_string(), true)]); - let acme = origin.consume().request_broadcast(ACME).await.expect("resolve"); - let sessions = origin.consume().request_broadcast(FEED).await.expect("resolve"); + let started = take_events(&mut events); + assert_eq!(names(&started), [(ACME.to_string(), true), (FEED.to_string(), true)]); + let [(acme_path, _), (feed_path, _)] = &started[..] else { + unreachable!() + }; + let acme = origin + .consume() + .request_broadcast(acme_path.as_str()) + .await + .expect("resolve"); + let sessions = origin + .consume() + .request_broadcast(feed_path.as_str()) + .await + .expect("resolve"); assert_eq!(read_last_frame(&acme, "publisher.json").await["acme/live"].bytes, 100); // The viewer leaves: both groups empty but stay announced, reading zero. @@ -1586,6 +1628,8 @@ mod tests { "a return within the linger re-announces nothing" ); assert_eq!(read_last_frame(&acme, "publisher.json").await["acme/live"].bytes, 50); + let mut track = subscribe(&acme, "publisher.json").await; + let old_sequence = track.next_group().await.expect("ok").expect("group").sequence; // The return re-armed the linger: it unannounces only once it elapses // again with the group still empty. @@ -1597,18 +1641,29 @@ mod tests { for _ in 0..4 { drive_tick().await; } - let mut ended = take_events(&mut events); - ended.sort(); - assert_eq!(ended, [(ACME.to_string(), false), (FEED.to_string(), false)]); + let ended: Vec<_> = started.iter().map(|(path, _)| (path.clone(), false)).collect(); + assert_eq!(take_events(&mut events), ended); - // A return after the linger is a new broadcast counted from zero. + // A return after the linger is a new epoch counted from zero, with + // group numbers starting over. let _third = feed(registry, Tier::default(), "acme/live", true, 1, 25).await; drive_tick().await; - let mut restarted = take_events(&mut events); - restarted.sort(); - assert_eq!(restarted, [(ACME.to_string(), true), (FEED.to_string(), true)]); - let acme = origin.consume().request_broadcast(ACME).await.expect("resolve"); + let restarted = take_events(&mut events); + assert_eq!(names(&restarted), [(ACME.to_string(), true), (FEED.to_string(), true)]); + assert_ne!(restarted[0].0, *acme_path, "a returning group mints a new epoch"); + assert_ne!(restarted[1].0, *feed_path, "a returning group mints a new epoch"); + let acme = origin + .consume() + .request_broadcast(restarted[0].0.as_str()) + .await + .expect("resolve"); assert_eq!(read_last_frame(&acme, "publisher.json").await["acme/live"].bytes, 25); + let mut track = subscribe(&acme, "publisher.json").await; + let group = track.next_group().await.expect("ok").expect("group"); + assert!( + group.sequence < old_sequence, + "a new epoch numbers its groups from zero" + ); } #[tokio::test(start_paused = true)] @@ -1623,9 +1678,10 @@ mod tests { drive_tick().await; assert_eq!(take_events(&mut events), []); drive_tick().await; - let mut ended = take_events(&mut events); - ended.sort(); - assert_eq!(ended, [(ACME.to_string(), false), (FEED.to_string(), false)]); + assert_eq!( + names(&take_events(&mut events)), + [(ACME.to_string(), false), (FEED.to_string(), false)] + ); } #[tokio::test(start_paused = true)] @@ -1704,27 +1760,19 @@ mod tests { #[test] fn advertised_path_with_and_without_node() { - let prefix = Path::new(".stats"); - let empty = Path::empty(); - assert_eq!( - advertised_path(&prefix, &empty, Some("sjc")).as_str(), - ".stats/node/sjc" - ); - assert_eq!( - advertised_path(&prefix, &empty, Some("sjc/1")).as_str(), - ".stats/node/sjc/1" - ); - assert_eq!(advertised_path(&prefix, &empty, None).as_str(), ".stats/node"); - assert_eq!( - advertised_path(&prefix, &Path::new("acme"), Some("sjc")).as_str(), - ".stats/acme/node/sjc" - ); - - let prefix = Path::new("metrics"); - assert_eq!( - advertised_path(&prefix, &Path::new("demo/room"), Some("lon")).as_str(), - "metrics/demo/room/node/lon" - ); + let epoch = Epoch::mint(); + let path = |prefix: &str, group: &str, node| { + let path = advertised_path(&Path::new(prefix), &Path::new(group), node, &epoch); + path.as_str() + .strip_suffix(&format!("/@{epoch}")) + .expect("epoch") + .to_string() + }; + assert_eq!(path(".stats", "", Some("sjc")), ".stats/node/sjc"); + assert_eq!(path(".stats", "", Some("sjc/1")), ".stats/node/sjc/1"); + assert_eq!(path(".stats", "", None), ".stats/node"); + assert_eq!(path(".stats", "acme", Some("sjc")), ".stats/acme/node/sjc"); + assert_eq!(path("metrics", "demo/room", Some("lon")), "metrics/demo/room/node/lon"); } #[test] @@ -2027,7 +2075,6 @@ mod tests { origin: produce_origin(), prefix: PathOwned::from(".stats"), node: None, - sequence: Arc::new(AtomicU64::new(0)), depth, linger: DEFAULT_LINGER, interval: Duration::from_secs(1), From c20b880517bf1eb3d36ca6f32f4f79047a456582 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 00:52:51 -0700 Subject: [PATCH 3/9] test(stats): restart a producer on a shared origin Covers the old epoch ending and the new one announcing the same name on one origin, notes that a short linger multiplies aggregator state, and updates the m1 stats quests to the epoched path. Co-Authored-By: Claude Opus 5.5 --- quest/m1/stats/README.md | 2 +- quest/m1/stats/health.md | 2 +- rs/moq-stats/src/produce.rs | 41 +++++++++++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/quest/m1/stats/README.md b/quest/m1/stats/README.md index 35c1e31e385..8ab5c24e835 100644 --- a/quest/m1/stats/README.md +++ b/quest/m1/stats/README.md @@ -26,7 +26,7 @@ catalog keyed by rendition, not a stats track per rendition and not a sum per kind. - **Media stats leave moq-stats.** The relay is media-agnostic and keeps - `Traffic`, `Presence`, and `.stats/node/` unchanged. Media stats are + `Traffic`, `Presence`, and `.stats/node//@` unchanged. Media stats are hang tracks, discovered through the catalog, so no `Producer` extension, `Merge` wrapper, or flattened generic is needed. One layout for relay and clients is given up on purpose. diff --git a/quest/m1/stats/health.md b/quest/m1/stats/health.md index fd2e81a8967..106c4e70772 100644 --- a/quest/m1/stats/health.md +++ b/quest/m1/stats/health.md @@ -29,7 +29,7 @@ The model reads the shapes this line settled after track named in the catalog, keyed by rendition alias, and viewers' `.echo` broadcasts. moq.pro's plan predates that and read client stats from a `.stats` broadcast, which #4510 replaced with these tracks (the relay's own -`.stats/node/` is unchanged); do not revive the client one. +`.stats/node//@` is unchanged); do not revive the client one. Guidance, to be settled while building: diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 32aea6ae828..b2f0c514186 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -60,6 +60,10 @@ pub struct Config { /// leaving and another arriving causes no unannounce and re-announce across /// the mesh. While it lingers empty, every track reads `{}`. Zero /// unannounces on the first drain the group is empty. Unused at depth `0`. + /// + /// A return after the linger is a new epoch, which an aggregating reader + /// keeps as its own node for its grace, so a short linger on a flapping + /// group multiplies that reader's state. pub linger: Duration, } @@ -1369,18 +1373,39 @@ mod tests { assert_eq!(name(&announced(&origin).await.0), ".stats/node"); } - /// A restarted producer announces under a new epoch, so a relay never serves - /// it groups cached under its previous run. + /// A producer restarted on the same origin ends its old epoch and announces + /// the same name under a new one, so a relay never serves it groups cached + /// under its previous run. #[tokio::test(start_paused = true)] async fn restarted_producer_announces_a_new_epoch() { - let (first, origin) = test_producer(Some("sjc")); - let before = announced(&origin).await.0; + let origin = produce_origin(); + let mut events = origin.consume().with_hidden(true).announced(); + let producer = || { + Producer::new( + Config::new() + .with_origin(origin.clone()) + .with_node(PathOwned::from("sjc")), + ) + }; + + let first = producer(); + drive_tick().await; + let [(before, true)] = &take_events(&mut events)[..] else { + panic!("expected one announce"); + }; + assert_eq!(name(before), ".stats/node/sjc"); + drop(first); + drive_tick().await; + assert_eq!(take_events(&mut events), [(before.clone(), false)]); - let (_second, origin) = test_producer(Some("sjc")); - let after = announced(&origin).await.0; - assert_eq!(name(&before), name(&after)); - assert_ne!(before, after); + let _second = producer(); + drive_tick().await; + let [(after, true)] = &take_events(&mut events)[..] else { + panic!("expected one announce"); + }; + assert_eq!(name(after), name(before)); + assert_ne!(after, before, "a restart mints a new epoch"); } #[tokio::test(start_paused = true)] From 28bbc7ba11a289aec9bcb3f554e8ea8206e3ae2a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 15:46:46 -0700 Subject: [PATCH 4/9] style(stats): rustfmt the epoch tests Co-Authored-By: Claude Opus 5.5 --- rs/moq-stats/src/produce.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 2068bf09a48..dae050a6375 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1417,7 +1417,11 @@ mod tests { let [(path, true, before)] = &take_epochs(&mut events)[..] else { panic!("expected one announce"); }; - let stats = origin.consume().request_broadcast(path.as_str()).await.expect("resolve"); + let stats = origin + .consume() + .request_broadcast(path.as_str()) + .await + .expect("resolve"); let mut old = subscribe(&stats, "publisher.json").await; while try_next_frame(&mut old).is_some() {} @@ -1437,7 +1441,11 @@ mod tests { let end = old.next_group().now_or_never().expect("the old subscription ended"); assert!(!matches!(end, Ok(Some(_))), "still reading the old epoch"); - let stats = origin.consume().request_broadcast(path.as_str()).await.expect("resolve"); + let stats = origin + .consume() + .request_broadcast(path.as_str()) + .await + .expect("resolve"); assert_eq!(read_last_frame(&stats, "publisher.json").await["foo/bar"].bytes, 30); } @@ -1716,12 +1724,18 @@ mod tests { panic!("expected both groups to announce again, got {restarted:?}"); }; assert_eq!((acme_again.as_str(), feed_again.as_str()), (ACME, FEED)); - assert!(new_acme > acme_epoch && new_feed > feed_epoch, "a returning group mints a new epoch"); + assert!( + new_acme > acme_epoch && new_feed > feed_epoch, + "a returning group mints a new epoch" + ); let acme = origin.consume().request_broadcast(ACME).await.expect("resolve"); assert_eq!(read_last_frame(&acme, "publisher.json").await["acme/live"].bytes, 25); let mut track = subscribe(&acme, "publisher.json").await; let group = track.next_group().await.expect("ok").expect("group"); - assert!(group.sequence < old_sequence, "a new epoch numbers its groups from zero"); + assert!( + group.sequence < old_sequence, + "a new epoch numbers its groups from zero" + ); } #[tokio::test(start_paused = true)] From 9e09044469369e65c810354c6e12b54b89427777 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 00:26:21 -0700 Subject: [PATCH 5/9] fix(stats)!: pin publisher epochs across metadata changes Co-Authored-By: GPT-6 --- doc/concept/moq-lite.md | 2 + doc/lib/js/net.md | 1 + doc/lib/rs/index.md | 2 +- doc/lib/rs/moq-net.md | 1 + doc/setup/upgrade.md | 2 +- js/net/bench/broadcasts.ts | 35 +++ js/net/src/origin.test.ts | 50 ++++ js/net/src/origin.ts | 52 +++- rs/hang/examples/subscribe.rs | 10 +- rs/moq-bench/src/connection.rs | 6 +- rs/moq-boy/src/input.rs | 2 +- rs/moq-c/src/origin.rs | 2 +- rs/moq-cli/tests/import.rs | 2 +- rs/moq-e2ee/tests/transport.rs | 2 +- rs/moq-ffi/src/origin.rs | 2 +- rs/moq-ffi/src/test.rs | 4 +- rs/moq-hls/src/export/mod.rs | 22 +- rs/moq-hls/src/server/routes.rs | 13 +- rs/moq-mux/src/catalog/hang/consumer.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 2 +- rs/moq-mux/src/source.rs | 6 +- rs/moq-net/benches/origin.rs | 14 +- rs/moq-net/benches/viewers.rs | 2 +- rs/moq-net/src/ietf/publisher.rs | 4 +- rs/moq-net/src/lite/publisher.rs | 8 +- rs/moq-net/src/lite/subscriber.rs | 6 +- rs/moq-net/src/model/origin.rs | 231 +++++++++-------- rs/moq-net/tests/announce_to_serve.rs | 2 +- rs/moq-net/tests/broadcast_close.rs | 4 +- rs/moq-net/tests/datagram.rs | 4 +- rs/moq-net/tests/fetch_cancel.rs | 4 +- rs/moq-net/tests/finished_broadcast_mock.rs | 2 +- rs/moq-net/tests/goaway.rs | 2 +- rs/moq-net/tests/head_fetch.rs | 2 +- rs/moq-net/tests/history_groups.rs | 6 +- rs/moq-net/tests/rejoin.rs | 16 +- rs/moq-net/tests/route_change.rs | 4 +- rs/moq-net/tests/session_close.rs | 2 +- rs/moq-net/tests/subscribe_tracks.rs | 2 +- .../tests/subscription_end_integrity.rs | 7 +- rs/moq-net/tests/track_abort_over_session.rs | 4 +- rs/moq-net/tests/track_info_cancel.rs | 2 +- rs/moq-net/tests/track_tail.rs | 2 +- rs/moq-net/tests/unannounce_release.rs | 2 +- rs/moq-net/tests/wide_sequence.rs | 4 +- rs/moq-relay/src/cluster.rs | 6 +- rs/moq-relay/src/internal.rs | 2 +- rs/moq-relay/tests/auth_lifetime.rs | 4 +- rs/moq-relay/tests/cluster_unknown.rs | 2 +- rs/moq-relay/tests/embed.rs | 2 +- rs/moq-relay/tests/goaway_cluster.rs | 14 +- rs/moq-relay/tests/public_rules.rs | 5 +- rs/moq-relay/tests/runtime_uring.rs | 4 +- rs/moq-relay/tests/runtime_workers.rs | 2 +- rs/moq-relay/tests/smoke.rs | 10 +- rs/moq-room/src/room.rs | 2 +- rs/moq-rtmp/src/dial.rs | 2 +- rs/moq-rtmp/src/server.rs | 2 +- rs/moq-srt/src/ts.rs | 6 +- rs/moq-stats/benches/aggregate.rs | 36 ++- rs/moq-stats/src/aggregate.rs | 240 +++++++++++++++++- rs/moq-stats/src/consume.rs | 4 +- rs/moq-stats/src/produce.rs | 16 +- rs/moq-tokio/examples/clock.rs | 2 +- rs/moq-tokio/src/client.rs | 2 +- rs/moq-tokio/src/server.rs | 2 +- rs/moq-tokio/tests/backend.rs | 10 +- rs/moq-tokio/tests/broadcast.rs | 44 ++-- rs/moq-tokio/tests/max_age.rs | 2 +- .../tests/subscription_end_integrity.rs | 5 +- rs/moq-tokio/tests/unannounce.rs | 2 +- rs/moq-transcode/examples/transcode.rs | 2 +- rs/moq-uring/benches/session_lite.rs | 5 +- rs/moq-uring/tests/session.rs | 10 +- rs/moq-uring/tests/web.rs | 5 +- rs/moq-wasm/src/lib.rs | 2 +- 76 files changed, 708 insertions(+), 300 deletions(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 3a9da934da7..4cb3e85056c 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -177,6 +177,8 @@ link (no epoch) and a moq-lite 07 link (with one), the epoch-carrying route supersedes the other each time it appears, so a flapping moq-lite 07 link cuts the viewers resolved through the older one. +Stats aggregation retains each known publisher epoch separately. When epoch metadata disappears or returns, it carries only the immediately outgoing same-path contribution across that transition, retaining the last snapshot until a fresh snapshot replaces it. If recovery names a different known epoch, the outgoing known lineage stays in grace and the incoming epoch resumes its own contribution. Older unversioned routes cannot distinguish a reconnect from a restart, so that fresh snapshot may regress counters; other known epochs retained during grace are never chosen as substitutes. + ### Hidden broadcasts A path segment starting with `.` hides a route from discovery, the way a diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index a16b9de00c2..03a615b97a5 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -49,6 +49,7 @@ for (;;) { ``` - **Origins** hold the broadcasts, not the connection: closing a session unannounces them but leaves them created for the next one. `origin.request(path)` resolves an announced local broadcast with no round trip, so a page that watches what it publishes reads its own copy, unless a cheaper route announces the same path. A refusal from the most specific route ends the request: `request.closed` settles with the handler's error and no broader route is asked. Create, populate, then `announce()` for an exact path; use `dynamic(prefix, route)` when the set of paths is not known: an exact-path subscribe before the tracks exist is refused, and nobody, local or remote, can see or reach a broadcast until it announces. +- **Epoch pinning**: `consumer.request(path, { epoch })` pins an announced publisher instance. The request stays unresolved and reports `unroutable` when the winning route names a different or missing epoch, including while an asynchronous handler answers. Omit `epoch` for unpinned lookup. - **Connections** race WebTransport against WebSocket. `new Connection({ url })` pools one connection per relay URL and reconnects with backoff, which the elements use. Supplying WebTransport/WebSocket options, discovery, delay, `goaway`, or a caller-owned origin selects a private loop; explicit `share: true` refuses those options. On a relay's GOAWAY the connection dials the replacement at once while the old session keeps serving its groups until it closes or the `goaway.handover` cap (default 10s, lowered to the relay's deadline) passes; the handle and its origin stay the same. An empty GOAWAY redials the same URL through a fresh DNS resolve. A redirect is followed onto the same host by default (`goaway.redirect: "follow"` lets the relay pick the host, `"ignore"` never moves), and moves the pooled entry to the new URL; a malformed or refused one, including a host change under a pinned certificate, ends the connection with `Error.RefusedRedirect` instead of reconnecting. `closed` settles when the handle is released (`null` on a clean close); the failure that stopped retrying the current URL is `error`, and a new URL recovers the same handle. A connection owns one send-rate sampler and one `Bandwidth.Allocator`; publishers reserve against it so their encoder targets sum to the estimate instead of each matching it. - **Bandwidth** (`Bandwidth.Allocator`) divides the connection's send-rate estimate by track priority, max-min fair within a tier. An idle track claims nothing. The receive side is untouched. - **Discovery** by any pattern scope (`origin.announced(scope)`, such as `room/*/chat`; default everything). A route event's `kind` says whether the route started, updated, or ended; its `prefix` is the covered prefix relative to the origin, and `captures` reports what the scope's wildcards matched when the prefix pins them. The consumer is an async iterable. `origin.broadcasts(scope)` is a live `Getter>` of the same covered prefixes for UIs that need the current set. A borrowed `Connection.origin` also exposes `dynamic(prefix, route)` for serving paths on demand. diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index eb232992793..8401bb330b1 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -56,7 +56,7 @@ let mut announced = consumer.announced(); while let Some(event) = announced.next().await { // Skip updates and retractions. let moq_net::announce::Event::Start(update) = event else { continue }; - let broadcast = consumer.request_broadcast(&update.prefix).await?; + let broadcast = consumer.request_broadcast(&update.prefix, update.route.epoch).await?; let catalog = broadcast .track(hang::Catalog::DEFAULT_NAME)? .subscribe(hang::Catalog::default_subscription()) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index ba2a81162b2..73b9165a815 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -17,6 +17,7 @@ above ([hang](/lib/rs/hang)); relays and CDNs implement only this. - **Origins** scope what a session can see, and merge duplicate subscriptions so a broadcast is pulled upstream once no matter how many local readers. - **Broadcasts** are created unannounced and invisible to everyone, then announced as an exact route, or served below a prefix with `dynamic`. A consumer of the same origin sees exactly what a peer sees. Discovery accepts pattern unions; events carry the advertised prefix and captures for a complete match. +- **Resolution** uses `consumer.request_broadcast(path, epoch)` with an owned `Epoch` to pin a publisher instance, or `None` for unpinned lookup. A pinned request refuses a different or missing epoch at request time and asynchronous resolution. - **Epochs** identify publisher instances with a canonical UUIDv7 carried on each route (`Route::epoch`): the newest wins a path, and only routes with the same epoch resume a subscription; see [publisher epochs](/concept/moq-lite#publisher-epochs). - **Patterns** (`Pattern`, `Patterns`) are re-exported from [`moq-pattern`](https://docs.rs/moq-pattern). Literal `Path` stays a coordinate. - **Tracks** carry groups with a priority, an optional publisher retention window (`Info::max_age`), and a timescale. Subscribers set their own priority and max age and can change them live. diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 38402d9ef96..84203094fd9 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -239,7 +239,7 @@ Other changes to a deployment: `origin::Config`. - **Announcements are prefix routes** (#3225, #3770). `announce::Update` is `{ prefix, route, kind, captures }`: skip `!update.kind.is_active()` and - resolve the broadcast with `consumer.request_broadcast(&update.prefix)`. + resolve the broadcast with `consumer.request_broadcast(&update.prefix, update.route.epoch)`. Serve a subtree on demand with `origin.dynamic(prefix, route)`. - **Tracks.** `with_latency_max` / `latency_max` is `with_max_age` / `max_age`. `write_datagram(Datagram)` is `insert_datagram(sequence, timestamp, payload)` diff --git a/js/net/bench/broadcasts.ts b/js/net/bench/broadcasts.ts index 80a37a3488d..b912e5dfe0b 100644 --- a/js/net/bench/broadcasts.ts +++ b/js/net/bench/broadcasts.ts @@ -1,6 +1,7 @@ /** Sweep observers against routes for one route update, and against tracks for one demand edge. */ import { Producer as BroadcastProducer } from "../src/broadcast.ts"; +import * as Epoch from "../src/epoch.ts"; import { Producer } from "../src/origin.ts"; import * as Path from "../src/path.ts"; @@ -85,3 +86,37 @@ for (const trackCount of [8, 32, 128]) { broadcast.close(); } } + +// Resolve one touched path while both the table and that path's readers grow. +console.log("request,routes,readers,update_and_read_us"); +const epoch = Epoch.parse("01900000-0000-7000-8000-000000000001"); +for (const pinned of [false, true]) { + for (const routeCount of routeCounts) { + for (const readerCount of observerCounts) { + const origin = new Producer(); + const reader = origin.consume(); + const options = pinned ? { epoch } : {}; + const publishers = Array.from({ length: routeCount }, (_, index) => { + const source = origin.createBroadcast(Path.from(`room/${index}`)); + source.announce({ epoch }); + return source; + }); + const background = publishers.map((_, index) => reader.request(Path.from(`room/${index}`), options)); + const requests = Array.from({ length: readerCount }, () => reader.request(Path.from("room/0"), options)); + const start = performance.now(); + for (let index = 0; index < updates; index++) { + publishers[0].announce({ epoch, cost: BigInt(index + 1) }); + await Promise.resolve(); + for (const request of requests) { + if (!request.active.peek()) throw new Error("same-epoch request lost its front"); + checksum++; + } + } + console.log( + `${pinned ? "pinned" : "unpinned"},${routeCount},${readerCount},${(((performance.now() - start) * 1000) / updates).toFixed(1)}`, + ); + for (const request of [...requests, ...background]) request.close(); + origin.close(); + } + } +} diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index f1254022dd9..d7acecd348a 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -1901,3 +1901,53 @@ test("a peer naming another epoch is refused rather than served a different inst local.close(); origin.close(); }); + +test("request pins its epoch while an unpinned sibling follows the newest publisher", () => { + const origin = new Producer(); + const path = Path.from("room"); + const first = origin.createBroadcast(path); + first.announce({ epoch: EPOCH }); + const reader = origin.consume(); + const pinned = reader.request(path, { epoch: EPOCH }); + const unpinned = reader.request(path); + expect(pinned.active.peek()).toBeDefined(); + const newerEpoch = Epoch.parse("01900000-0000-7000-8000-000000000002"); + const second = origin.createBroadcast(path); + second.announce({ epoch: newerEpoch }); + expect(pinned.active.peek()).toBeUndefined(); + expect(pinned.unroutable.peek()).toBe(true); + expect(unpinned.active.peek()).toBeDefined(); + const stale = reader.request(path, { epoch: EPOCH }); + expect(stale.active.peek()).toBeUndefined(); + expect(stale.unroutable.peek()).toBe(true); + stale.close(); + pinned.close(); + unpinned.close(); + origin.close(); +}); + +test("request never accepts an old handler after its epoch changes asynchronously", async () => { + const origin = new Producer(); + const prefix = Path.from("room"); + const path = Path.from("room/cam"); + const dynamic = origin.dynamic(prefix, { epoch: EPOCH }); + const pending = origin.consume().request(path, { epoch: EPOCH }); + const requests = dynamic.requested(); + const old = (await requests.next()).value; + const epoch = Epoch.parse("01900000-0000-7000-8000-000000000002"); + dynamic.update({ ...dynamic.route, epoch }); + const current = origin.consume().request(path, { epoch }); + old?.accept(new BroadcastProducer().consume()); + expect(pending.active.peek()).toBeUndefined(); + expect(pending.unroutable.peek()).toBe(true); + expect(current.active.peek()).toBeUndefined(); + const next = (await requests.next()).value; + next?.accept(new BroadcastProducer().consume()); + expect(current.active.peek()).toBeDefined(); + expect(pending.active.peek()).toBeUndefined(); + await requests.return?.(); + pending.close(); + current.close(); + dynamic.close(); + origin.close(); +}); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 2f63a1557c0..7f3b6472adb 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -179,7 +179,13 @@ export interface RequestSlot { announced: number; answer?: broadcast.Consumer; readonly handles: Set>; - readonly route: Signal; + readonly route: Signal; +} + +/** One path resolution and the epoch of the route that can currently serve it. */ +interface Resolution { + readonly front?: broadcast.Consumer; + readonly epoch?: Epoch.Valid; } /** @@ -349,6 +355,11 @@ class ServeState { if (this.closed.peek() !== undefined) return; const err = abort ?? unroutable(); this.closed.set(err); + this.reset(err); + } + + /** Release answers and invalidate pending handlers when the publisher instance changes. */ + reset(err: Error = unroutable()): void { const queued = [...this.pending.values()]; this.pending.clear(); this.queue.mutate((queue) => { @@ -357,11 +368,12 @@ class ServeState { for (const request of queued) { finishRequest(request, err); } - for (const [path, front] of this.served) { + const served = [...this.served]; + this.served.clear(); + for (const [path, front] of served) { front.close(); this.onChange(path); } - this.served.clear(); this.rejected.clear(); this.demanding.clear(); this.settled.update((n) => n + 1); @@ -468,7 +480,7 @@ class OriginState { // Broadcasts materialized from a served route, keyed by exact path. Shared by every // request for the path so repeats reuse one accept; dropped (and closed) when the // providing route goes away or the last request releases it. - materialized = new Map(); + materialized = new Map(); // Paths consumers asked for without waiting for an announcement; attached sessions // answer them with blind subscriptions. Never announced: an answered request is assumed @@ -612,13 +624,13 @@ class OriginState { * before the old one breaks: the current front keeps serving until the new route * answers (then swaps) or refuses (then the request ends). A retracted route swaps at once. */ - route(path: Path.Valid, slot: Pick): broadcast.Consumer | undefined { + route(path: Path.Valid, slot: Pick): Resolution | undefined { const entry = this.bestEntry(path); const local = this.local.peek()?.get(path); if (local && this.localWins(path, entry)) { // Nothing reads a remote front the local broadcast replaced, so close its session subscription. this.releaseMaterialized(path); - return local; + return { front: local, epoch: this.advertisedLocal.peek()?.get(path)?.epoch }; } let cached = this.materialized.get(path); @@ -626,21 +638,23 @@ class OriginState { this.materialized.delete(path); cached = undefined; } - if (cached && cached.entry === entry) return cached.front; + const epoch = entry?.route.peek().epoch; + if (cached && cached.entry === entry && cached.epoch === epoch) return cached; if (!entry?.server) { this.releaseMaterialized(path); - return slot.answer; + return slot.answer ? { front: slot.answer } : undefined; } const served = entry.server.served.get(path); if (served && served.closed.peek() === undefined) { cached?.front.close(); - this.materialized.set(path, { entry, front: served }); - return served; + const resolution = { entry, front: served, epoch }; + this.materialized.set(path, resolution); + return resolution; } entry.server.enqueue(path); - return cached?.front; + return { front: cached?.epoch === epoch ? cached?.front : undefined, epoch }; } } @@ -683,6 +697,8 @@ export interface Table { export interface RequestOptions { /** Wait for a routed announcement when discovery is supported; otherwise subscribe blindly. */ announced?: boolean; + /** Refuse a different publisher instance, including an asynchronous answer or a retry. */ + epoch?: Epoch.Valid; } /** @@ -1255,6 +1271,7 @@ export class Consumer { * feeds from more than one connection. */ request(path: Path.Valid, options: RequestOptions = {}): Requesting { + const epoch = options.epoch; const relative = path; path = this.#scope.path(path); const requests = this.#state.requests.peek(); @@ -1308,8 +1325,9 @@ export class Consumer { let released = false; let source: broadcast.Consumer | undefined; let handle: broadcast.Consumer | undefined; - const own = (front: broadcast.Consumer | undefined): broadcast.Consumer | undefined => { + const own = (resolution: Resolution | undefined): broadcast.Consumer | undefined => { if (released) return undefined; + const front = epoch === undefined || resolution?.epoch === epoch ? resolution?.front : undefined; if (front !== source) { const previous = handle; source = front; @@ -1334,7 +1352,10 @@ export class Consumer { // answer. const unroutable = new Derived( [route, this.#state.answerers, closed], - (front, answerers, ended) => ended !== undefined || (!front && answerers === 0), + (resolution, answerers, ended) => + ended !== undefined || + (epoch !== undefined && resolution?.epoch !== epoch) || + (!resolution?.front && answerers === 0), ); return makeRequesting(relative, active, unroutable, closed, () => { @@ -1607,7 +1628,10 @@ export class Dynamic { */ update(route: Route | { epoch?: Route["epoch"]; hops?: Route["hops"]; cost?: Route["cost"] | bigint }): void { if (this.#closed) throw new Error("dynamic is closed"); - this.#entry.route.set(Route.normalize(route)); + const next = Route.normalize(route); + const previous = this.#entry.route.peek().epoch; + this.#entry.route.set(next); + if (previous !== next.epoch) this.#entry.server?.reset(); this.#state.rebuildOriginated(); this.#state.refreshPrefix(Path.join(this.#entry.scope.root, this.prefix)); this.#state.routes.mutate(() => {}); diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index b213db674db..8ae58595ee6 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -43,14 +43,16 @@ async fn run_session(origin: moq_net::origin::Producer) -> anyhow::Result<()> { async fn run_subscribe(consumer: moq_net::origin::Consumer) -> anyhow::Result<()> { // Wait for a route to be announced, then resolve the broadcast at its path. // The convention is that a publisher announces each broadcast's exact path. - let path = match consumer.announced().next().await.context("origin closed")? { - moq_net::announce::Event::Start(announce) => announce.prefix, + let announce = match consumer.announced().next().await.context("origin closed")? { + moq_net::announce::Event::Start(announce) => announce, // Nothing else can come before the first announcement. event => anyhow::bail!("unexpected {event:?}"), }; - tracing::info!(%path, "broadcast announced"); - let broadcast = consumer.request_broadcast(&path).await?; + tracing::info!(path = %announce.prefix, "broadcast announced"); + let broadcast = consumer + .request_broadcast(&announce.prefix, announce.route.epoch) + .await?; // Read the catalog to discover available tracks. let catalog_track = broadcast diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index c90bf21bea1..fe9f9e1ba9b 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -296,7 +296,7 @@ async fn subscribe_named(consume: moq_net::origin::Consumer, path: String, stats .routed(path.as_str()) .await .ok_or_else(|| anyhow::anyhow!("target broadcast was never announced: {path}"))?; - let broadcast = consume.request_broadcast(path.as_str()).await?; + let broadcast = consume.request_broadcast(path.as_str(), None).await?; drain(broadcast, &stats).await } @@ -352,7 +352,7 @@ async fn subscribe( let mut selected = pool.len() as u64; for path in pool { - let Ok(broadcast) = consume.request_broadcast(path.as_str()).await else { + let Ok(broadcast) = consume.request_broadcast(path.as_str(), None).await else { continue; }; spawn_drain(&mut tasks, path, broadcast, stats.clone()); @@ -370,7 +370,7 @@ async fn subscribe( if own.contains(&path) || !seen.insert(path.clone()) { continue; } - let Ok(broadcast) = consume.request_broadcast(path.as_str()).await else { + let Ok(broadcast) = consume.request_broadcast(path.as_str(), None).await else { continue; }; selected += 1; diff --git a/rs/moq-boy/src/input.rs b/rs/moq-boy/src/input.rs index 61202b11273..82bede431ac 100644 --- a/rs/moq-boy/src/input.rs +++ b/rs/moq-boy/src/input.rs @@ -75,7 +75,7 @@ pub async fn handle_viewers( let viewer_id = update.prefix.to_string(); if active { - let Ok(broadcast) = viewer_origin.request_broadcast(&update.prefix).await else { + let Ok(broadcast) = viewer_origin.request_broadcast(&update.prefix, None).await else { continue; }; tracing::info!(%viewer_id, "viewer connected"); diff --git a/rs/moq-c/src/origin.rs b/rs/moq-c/src/origin.rs index ca3f4b3183f..e10a6f6715e 100644 --- a/rs/moq-c/src/origin.rs +++ b/rs/moq-c/src/origin.rs @@ -295,7 +295,7 @@ impl Origin { mut close: oneshot::Receiver<()>, ) -> Result<(), Error> { // Resolves to an error when no announced route can serve the path. - let pending = consumer.request_broadcast(path.as_str()); + let pending = consumer.request_broadcast(path.as_str(), None); // `biased` so a pending close always wins over a ready broadcast. let broadcast = tokio::select! { diff --git a/rs/moq-cli/tests/import.rs b/rs/moq-cli/tests/import.rs index bab5846621a..6cf401b75d9 100644 --- a/rs/moq-cli/tests/import.rs +++ b/rs/moq-cli/tests/import.rs @@ -66,7 +66,7 @@ async fn import_delivers_the_catalog_finish_at_eof() { .expect("origin closed"), moq_tokio::moq_net::announce::Event::Start(_) ) {} - let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("demo")) + let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("demo", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); diff --git a/rs/moq-e2ee/tests/transport.rs b/rs/moq-e2ee/tests/transport.rs index f2a6b58fb9a..5bdb725bf90 100644 --- a/rs/moq-e2ee/tests/transport.rs +++ b/rs/moq-e2ee/tests/transport.rs @@ -63,7 +63,7 @@ async fn connect_protected(version: Version, track: &str) -> Fixture { } let epoch = route.epoch.unwrap_or_else(|| generation.epoch().clone()); let generation = cred.generation(epoch); - let remote = consumer.request_broadcast(&path).await.unwrap(); + let remote = consumer.request_broadcast(&path, None).await.unwrap(); let subscriber = remote .track(name.as_str()) .unwrap() diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index 4ddbd3f2027..42ebc04b9fc 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -407,7 +407,7 @@ impl MoqOriginConsumer { /// Calling this straight after connecting therefore races the session's announcements /// and can report a live broadcast as unroutable. Await `announced_broadcast` first. pub async fn request_broadcast(&self, path: String) -> Result, MoqError> { - let broadcast = self.inner.request_broadcast(path.as_str()).await?; + let broadcast = self.inner.request_broadcast(path.as_str(), None).await?; Ok(Arc::new(MoqBroadcastConsumer::routed(broadcast, self.inner.clone()))) } } diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 6f6b1a751f9..887478052bd 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -2622,7 +2622,7 @@ async fn cancel_retracts_the_route_synchronously() { let dynamic = serve(&origin, ""); let inner = origin.inner().consume(); - let queued = inner.request_broadcast("x").into_inner(); + let queued = inner.request_broadcast("x", None).into_inner(); assert!( queued.poll_ok(&kio::Waiter::noop()).is_pending(), "the route must serve while the handler lives" @@ -2630,7 +2630,7 @@ async fn cancel_retracts_the_route_synchronously() { drop(queued); dynamic.cancel(); - let verdict = inner.request_broadcast("y").into_inner(); + let verdict = inner.request_broadcast("y", None).into_inner(); match verdict.poll_ok(&kio::Waiter::noop()) { std::task::Poll::Ready(Err(moq_net::Error::Unroutable)) => {} std::task::Poll::Ready(Err(err)) => panic!("unexpected error: {err}"), diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index aac8e24e0cf..38cc92248e8 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -2416,10 +2416,13 @@ mod tests { accept_sibling(&new_server, &new_media).await; // The rendition skips timeline rows until this bind resolves. Joining the // same front waits for attach, so writes below land after that. - tokio::time::timeout(Duration::from_secs(5), origin.consume().request_broadcast("media")) - .await - .expect("the rebound sibling resolves") - .expect("the replacement is routable"); + tokio::time::timeout( + Duration::from_secs(5), + origin.consume().request_broadcast("media", None), + ) + .await + .expect("the rebound sibling resolves") + .expect("the replacement is routable"); let recorder = catalog.enroll_test("video0").unwrap(); // Model source records lost during the outage: every rendition must carry the // same new timeline sequence, including a recorder that starts after this rebind. @@ -2623,10 +2626,13 @@ mod tests { let mut new_media = moq_net::broadcast::Info::new().produce(); let _ = rendition.snapshot(); accept_sibling(&new_server, &new_media).await; - tokio::time::timeout(Duration::from_secs(5), origin.consume().request_broadcast("media")) - .await - .expect("the rebound sibling resolves") - .expect("the replacement is routable"); + tokio::time::timeout( + Duration::from_secs(5), + origin.consume().request_broadcast("media", None), + ) + .await + .expect("the rebound sibling resolves") + .expect("the replacement is routable"); let recorder = catalog.enroll_test("video0").unwrap(); let _new_track = write_routed_media(&mut new_media, NEW, recorder, 6_000_000); let deadline = tokio::time::Instant::now() + Duration::from_secs(5); diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index f2dd64ce2b0..a6f290ae9ae 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -753,7 +753,7 @@ mod tests { let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; - let remote = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("live")) + let remote = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("live", None)) .await .expect("remote resolve timed out") .expect("remote broadcast"); @@ -798,10 +798,13 @@ mod tests { let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); wait_listed(&app, "/room/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; - let remote_media = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("room/source")) - .await - .expect("sibling resolve timed out") - .expect("sibling broadcast"); + let remote_media = tokio::time::timeout( + TIMEOUT, + pair.sub_origin.consume().request_broadcast("room/source", None), + ) + .await + .expect("sibling resolve timed out") + .expect("sibling broadcast"); drop(media); drop(media_broadcast); tokio::time::timeout(TIMEOUT, remote_media.closed()) diff --git a/rs/moq-mux/src/catalog/hang/consumer.rs b/rs/moq-mux/src/catalog/hang/consumer.rs index 236954fe2ba..9a8ed649fbc 100644 --- a/rs/moq-mux/src/catalog/hang/consumer.rs +++ b/rs/moq-mux/src/catalog/hang/consumer.rs @@ -204,7 +204,7 @@ mod test { let origin = crate::source::produce_origin(); let dynamic = origin.dynamic("", Default::default()).unwrap(); - let requesting = origin.consume().request_broadcast("a/pub"); + let requesting = origin.consume().request_broadcast("a/pub", None); let served = broadcast.consume(); // The handler hands its route back: dropping it would retract the route and end // the broadcast it just served. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 580a1446c2a..63c49224e77 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -1658,7 +1658,7 @@ mod tests { .with_cost(5), ) .unwrap(); - let pending = origin.consume().request_broadcast("live"); + let pending = origin.consume().request_broadcast("live", None); let first = moq_net::broadcast::Info::new().produce(); let info = hang::container::track_info(hang::catalog::PRIORITY.video); let first_track = first.create_track("video", info.clone()).unwrap(); diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 9c884ac552d..85d45d4f4b5 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -70,7 +70,7 @@ impl Source { /// Begin resolving the catalog broadcast (the one at this source's path). pub(crate) fn request_catalog(&self) -> kio::Pending { - self.origin.request_broadcast(&self.path) + self.origin.request_broadcast(&self.path, None) } /// Resolve a rendition's optional broadcast reference to an origin path. @@ -110,7 +110,7 @@ impl Source { &self, rel: Option<&moq_net::path::Relative<'_>>, ) -> crate::Result> { - Ok(self.origin.request_broadcast(&self.target(rel)?)) + Ok(self.origin.request_broadcast(&self.target(rel)?, None)) } /// The skipping counterpart to [`Self::request`], returning `None` when `rel` walks above @@ -123,7 +123,7 @@ impl Source { &self, rel: Option<&moq_net::path::Relative<'_>>, ) -> Option> { - Some(self.origin.request_broadcast(&self.resolve_reference(rel)?)) + Some(self.origin.request_broadcast(&self.resolve_reference(rel)?, None)) } /// Remove renditions whose broadcast reference escapes above the origin root. diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 3572263806e..65cab029cce 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -308,7 +308,7 @@ fn bench_announce_fronts(c: &mut Criterion) { // Served but never answered: every request under it parks a front. let _served = producer.dynamic("room", origin::Route::default()).unwrap(); let _requests: Vec<_> = (0..fronts) - .map(|i| consumer.request_broadcast(format!("room/{i}"))) + .map(|i| consumer.request_broadcast(format!("room/{i}"), None)) .collect(); let waiter = kio::Waiter::noop(); // Run each front once so it parks on its upstream request. @@ -402,7 +402,7 @@ fn bench_request(c: &mut Criterion) { let waiter = kio::Waiter::noop(); group.bench_function(BenchmarkId::new("local", publishers), |b| { b.iter(|| { - let pending = fleet.consumer.request_broadcast("room/0"); + let pending = fleet.consumer.request_broadcast("room/0", None); // The front's driver resolves the first request; later ones join it. fleet.driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); pending @@ -415,7 +415,7 @@ fn bench_request(c: &mut Criterion) { b.iter(|| { let result = fleet .consumer - .request_broadcast("room/missing") + .request_broadcast("room/missing", None) .now_or_never() .expect("fails synchronously"); assert!(matches!(result, Err(moq_net::Error::Unroutable))); @@ -453,7 +453,7 @@ fn pool(members: usize, paths: usize) -> Pool { let waiter = kio::Waiter::noop(); let requests: Vec<_> = (0..paths) - .map(|i| consumer.request_broadcast(format!("pool/job-{i}"))) + .map(|i| consumer.request_broadcast(format!("pool/job-{i}"), None)) .collect(); driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); let mut producers = Vec::with_capacity(paths); @@ -546,7 +546,7 @@ fn bench_handoff(c: &mut Criterion) { first.write_frame(Timestamp::ZERO, b"one".as_ref()).unwrap(); first.finish().unwrap(); - let resolved = consumer.request_broadcast("room/live").await.unwrap(); + let resolved = consumer.request_broadcast("room/live", None).await.unwrap(); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); subscription.recv_group().await.unwrap().expect("first group"); @@ -601,7 +601,7 @@ fn bench_relay(c: &mut Criterion) { .map(|i| broadcast.create_track(format!("{i}"), None).unwrap()) .collect(); let mut subscriptions = runtime.block_on(async { - let resolved = producer.consume().request_broadcast("room/live").await.unwrap(); + let resolved = producer.consume().request_broadcast("room/live", None).await.unwrap(); let mut subscriptions = Vec::new(); for i in 0..tracks { let track = resolved.track(&format!("{i}")).unwrap(); @@ -658,7 +658,7 @@ fn bench_parked(c: &mut Criterion) { .collect(); let _writers: Vec<_> = sources.iter().map(|source| source.append_group().unwrap()).collect(); let (subscriptions, mut groups) = runtime.block_on(async { - let resolved = producer.consume().request_broadcast("room/live").await.unwrap(); + let resolved = producer.consume().request_broadcast("room/live", None).await.unwrap(); let mut subscriptions = Vec::new(); let mut groups = Vec::new(); for i in 0..tracks { diff --git a/rs/moq-net/benches/viewers.rs b/rs/moq-net/benches/viewers.rs index 831ac4b5235..0b25499707f 100644 --- a/rs/moq-net/benches/viewers.rs +++ b/rs/moq-net/benches/viewers.rs @@ -107,7 +107,7 @@ impl Room { fn join(&mut self, n: usize) -> moq_net::track::Subscriber { let session = moq_net::fuzz::excluding(self.producer.consume(), Hop::random()); let path = format!("room/{}", n % self.broadcasts); - let resolved = self.drive(session.request_broadcast(path)).unwrap(); + let resolved = self.drive(session.request_broadcast(path, None)).unwrap(); let mut subscription = self.drive(resolved.track("video").unwrap().subscribe(None)).unwrap(); self.drive(subscription.recv_group()) .unwrap() diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 54bf615531a..d4f460f5a91 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -617,7 +617,7 @@ where let broadcast = match self .serving_origin() .await - .request_broadcast(&msg.track_namespace) + .request_broadcast(&msg.track_namespace, None) .await { Ok(broadcast) => broadcast, @@ -1350,7 +1350,7 @@ where // The peer must have seen the announcement to name this namespace, so this // resolves like a SUBSCRIBE does. - let broadcast = match self.serving_origin().await.request_broadcast(&namespace).await { + let broadcast = match self.serving_origin().await.request_broadcast(&namespace, None).await { Ok(broadcast) => broadcast, Err(err) => return self.reject_fetch(stream, msg.request_id, &err, &err.to_string()).await, }; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 68005b99aca..3f7db91fa15 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3640,11 +3640,15 @@ mod tests { let serving = kio::wait(|waiter| publisher.shared.poll_serving_origin(waiter)).await; use futures::FutureExt; assert!( - serving.request_broadcast("echoed/x").now_or_never().unwrap().is_err(), + serving + .request_broadcast("echoed/x", None) + .now_or_never() + .unwrap() + .is_err(), "served the peer its own route" ); assert!( - serving.request_broadcast("local/x").now_or_never().is_none(), + serving.request_broadcast("local/x", None).now_or_never().is_none(), "withheld an independent route" ); } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 97241c143be..113ffcf02b2 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2611,7 +2611,7 @@ mod tests { assert!(announced.ready.is_empty()); let consumer = origin.consume(); - let request = moq_net_sim::spawn(async move { consumer.request_broadcast("room/7").await }); + let request = moq_net_sim::spawn(async move { consumer.request_broadcast("room/7", None).await }); let ready = kio::wait(|waiter| announced.ready.poll_pop(waiter)).await.unwrap(); assert_eq!(ready.as_str(), "room/7"); @@ -2802,7 +2802,7 @@ mod tests { // The peer's own subscription, excluding the hop the server minted for // it, is served from the local front before anything is announced back. let peer = origin.consume().excluding(assigned); - let resolved = peer.request_broadcast("room/host").await.expect("resolves"); + let resolved = peer.request_broadcast("room/host", None).await.expect("resolves"); let mut sub = resolved .track("video") .unwrap() @@ -2847,7 +2847,7 @@ mod tests { // The local front is still the one at the path, and still serving: the // peer's next request joins it rather than minting another. let still = peer - .request_broadcast("room/host") + .request_broadcast("room/host", None) .await .expect("the local front keeps serving"); assert!( diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index dc720f2a6e4..892f85f336b 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4268,7 +4268,7 @@ impl Consumer { let seen = watch.seen(); (watch, seen) }; - match self.request_broadcast(&path).await { + match self.request_broadcast(&path, None).await { Ok(broadcast) => return Ok(broadcast), Err(Error::Unroutable) => { kio::wait(|waiter| watch.poll_changed(waiter, seen)).await; @@ -4298,7 +4298,7 @@ impl Consumer { }) } - /// Resolve a broadcast by exact path. + /// Resolve a broadcast by exact path, optionally pinning its publisher epoch. /// /// Returns a [`kio::Pending`] future, mirroring /// [`track::Consumer::fetch_group`](track::Consumer::fetch_group). Every @@ -4317,14 +4317,23 @@ impl Consumer { /// newer epoch winning the path ends even those with [`Error::Unroutable`], so /// readers request the new broadcast instead of stalling on the old one. /// + /// Pass an owned [`crate::Epoch`] to refuse a different publisher instance, both + /// when requesting and when an asynchronous answer resolves. `None` leaves lookup + /// unpinned. A pinned request never resumes on a route whose epoch disappeared. + /// /// The returned future fails with [`Error::Unroutable`] at once when no /// announced route covers the path, including a broadcast created on this /// origin but not announced. /// A route claims capability, not inventory: resolving a covered path /// succeeds optimistically, and a path that names nothing surfaces as /// [`Error::NotFound`] on its tracks instead. - pub fn request_broadcast(&self, path: impl AsPath) -> kio::Pending { - self.request(path.as_path(), None) + pub fn request_broadcast( + &self, + path: impl AsPath, + epoch: impl Into>, + ) -> kio::Pending { + let epoch = epoch.into(); + self.request(path.as_path(), epoch.as_ref()) } /// [`Self::request_broadcast`], refused with [`Error::Unroutable`] unless the @@ -4915,7 +4924,7 @@ mod tests { broadcast.announce(Route::default()).unwrap(); consumer.announced().assert_next_wait(); - let resolved = consumer.request_broadcast(".stats/node").await.expect("resolves"); + let resolved = consumer.request_broadcast(".stats/node", None).await.expect("resolves"); assert_eq!(resolved.info().path.as_str(), ".stats/node"); } @@ -4938,8 +4947,8 @@ mod tests { let server = producer.dynamic(".svc", Route::default()).unwrap(); let project = mounted(&producer, "p1", &["**"]); - let through = project.request_broadcast(".svc/foo"); - let direct = producer.consume().request_broadcast(".svc/p1/foo"); + let through = project.request_broadcast(".svc/foo", None); + let direct = producer.consume().request_broadcast(".svc/p1/foo", None); let request = queued(&server).await; assert_eq!(request.path().as_str(), ".svc/p1/foo"); @@ -4992,7 +5001,7 @@ mod tests { announced.assert_next_wait(); // The shadowed path never resolves: the mount answers for it. - let err = project.request_broadcast(".svc/forged").await.err().unwrap(); + let err = project.request_broadcast(".svc/forged", None).await.err().unwrap(); assert!(matches!(err, Error::NotFound | Error::Unroutable), "{err:?}"); } @@ -5005,9 +5014,9 @@ mod tests { let _bar = producer.publish(".svc/p1/bar", Route::default()).unwrap(); let granted = mounted(&producer, "p1", &["foo", ".svc/foo"]); - granted.request_broadcast(".svc/foo").await.expect("granted"); + granted.request_broadcast(".svc/foo", None).await.expect("granted"); let refused = granted - .request_broadcast(".svc/bar") + .request_broadcast(".svc/bar", None) .now_or_never() .expect("refused at once"); assert!(matches!(refused, Err(Error::Unauthorized))); @@ -5017,7 +5026,7 @@ mod tests { let narrower = mounted(&producer, "p1", &["foo"]); let refused = narrower - .request_broadcast(".svc/foo") + .request_broadcast(".svc/foo", None) .now_or_never() .expect("refused at once"); assert!(matches!(refused, Err(Error::Unauthorized))); @@ -5026,7 +5035,7 @@ mod tests { // Another project's mount reaches its own target, never this one's. let other = mounted(&producer, "p2", &["**"]); other.clone().with_hidden(true).announced().assert_next_wait(); - let err = other.request_broadcast(".svc/foo").await.err().unwrap(); + let err = other.request_broadcast(".svc/foo", None).await.err().unwrap(); assert!(matches!(err, Error::Unroutable)); } @@ -5072,7 +5081,7 @@ mod tests { announced.assert_next_wait(); // Beneath the mount the target has no room: refused, never handed to a route. - let err = project.request_broadcast(".svc/x").await.err().unwrap(); + let err = project.request_broadcast(".svc/x", None).await.err().unwrap(); assert!(matches!(err, Error::BoundsExceeded(_)), "{err:?}"); let inside = project.scope(".svc/x", &Patterns::from(Pattern::all())).unwrap(); inside.announced().assert_next_wait(); @@ -5178,7 +5187,7 @@ mod tests { let mut announced = project.announced(); announced.assert_next_active(".svc/foo"); - project.request_broadcast(".svc/foo").await.expect("resolves"); + project.request_broadcast(".svc/foo", None).await.expect("resolves"); let mut report = stats::Report::default(); registry.report(&mut report); @@ -5243,7 +5252,7 @@ mod tests { peer.assert_next_ended("room/alice"); broadcast.unannounce(); announced.assert_next_wait(); - let err = consumer.request_broadcast("room/alice").await.err().unwrap(); + let err = consumer.request_broadcast("room/alice", None).await.err().unwrap(); assert!(matches!(err, Error::Unroutable)); // Back on the air, then the end of the broadcast retracts for good. @@ -5594,7 +5603,7 @@ mod tests { .dynamic("room", Route::default().with_hops(hops(&[10])).with_cost(5)) .unwrap(); - let _pending = consumer.request_broadcast("room/alice"); + let _pending = consumer.request_broadcast("room/alice", None); let request = queued(&identified).await; assert_eq!(request.path().as_str(), "room/alice"); assert!( @@ -5757,7 +5766,7 @@ mod tests { assert_eq!(route.hops, hops(&[10])); // The request goes upstream rather than to the local broadcast. - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; let upstream = broadcast::Info::new().produce(); request.accept(&upstream); @@ -5775,7 +5784,7 @@ mod tests { .publish("room/alice", Route::default().with_epoch(epoch()).with_cost(5)) .unwrap(); let first = consumer - .request_broadcast("room/alice") + .request_broadcast("room/alice", None) .await .expect("resolves locally"); @@ -5789,7 +5798,7 @@ mod tests { let upstream = broadcast::Info::new().produce(); request.accept(&upstream); - let second = consumer.request_broadcast("room/alice").await.expect("resolves"); + let second = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert!(first.is_clone(&second), "the newcomer joins the front"); } @@ -5805,7 +5814,7 @@ mod tests { let server = producer.dynamic("room/alice", Route::default()).unwrap(); // Were the claim to win, the request would park on its handler forever. - let resolved = moq_net_sim::timeout(Duration::from_secs(1), consumer.request_broadcast("room/alice")) + let resolved = moq_net_sim::timeout(Duration::from_secs(1), consumer.request_broadcast("room/alice", None)) .await .expect("the newer hopless route won the tie") .expect("resolves"); @@ -5867,7 +5876,7 @@ mod tests { .unwrap(); assert!(announced.assert_next_active("room/alice").hops.is_empty()); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert_eq!(resolved.info().path.as_str(), "room/alice"); for _ in 0..20 { moq_net_sim::yield_now().await; @@ -5883,17 +5892,20 @@ mod tests { let consumer = producer.consume(); let broadcast = producer.publish("room/alice", Route::default()).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); broadcast.unannounce(); - let err = consumer.request_broadcast("room/alice").await.err().unwrap(); + let err = consumer.request_broadcast("room/alice", None).await.err().unwrap(); assert!(matches!(err, Error::Unroutable), "joined a retracted front: {err}"); settle(|| resolved.is_closed()).await; assert!(!broadcast.consume().is_closed(), "the broadcast itself lives on"); // Announcing again serves a fresh front. broadcast.announce(Route::default()).unwrap(); - let again = consumer.request_broadcast("room/alice").await.expect("resolves again"); + let again = consumer + .request_broadcast("room/alice", None) + .await + .expect("resolves again"); assert!(!again.is_clone(&resolved)); } @@ -5907,7 +5919,7 @@ mod tests { let broadcast = producer.publish("room/alice", Route::default()).unwrap(); let mut dynamic = broadcast.dynamic(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let track = resolved.track("video").unwrap(); let subscribing = moq_net_sim::spawn(async move { track.subscribe(None).await }); let request = moq_net_sim::timeout(Duration::from_secs(1), dynamic.requested_track()) @@ -5938,7 +5950,7 @@ mod tests { let server = producer .dynamic("room/alice", Route::default().with_hops(hops(&[10]))) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let upstream = broadcast::Info::new().produce(); let dynamic = upstream.dynamic(); queued(&server).await.accept(&upstream); @@ -6036,7 +6048,7 @@ mod tests { let mut front = None; for viewer in 0..1000 { let session = producer.consume().excluding(Hop::random()); - let pending = session.request_broadcast("room/alice"); + let pending = session.request_broadcast("room/alice", None); if viewer == 0 { queued(&server).await.accept(&upstream); } @@ -6069,17 +6081,20 @@ mod tests { let fronts = || producer.shared.lock().fronts.len(); let peer = producer.consume().excluding(origin(7)); - let _local = producer.consume().request_broadcast("room/alice"); - let _peer = peer.request_broadcast("room/alice"); + let _local = producer.consume().request_broadcast("room/alice", None); + let _peer = peer.request_broadcast("room/alice", None); assert_eq!(fronts(), 1, "a hop no chain names excludes nothing"); // The same publisher, now also reached through the peer. let _echo = producer .dynamic("room", Route::default().with_hops(hops(&[10, 7]))) .unwrap(); - let _filtered = peer.request_broadcast("room/alice"); + let _filtered = peer.request_broadcast("room/alice", None); assert_eq!(fronts(), 2, "the peer still shares the plain front"); - let _viewer = producer.consume().excluding(origin(8)).request_broadcast("room/alice"); + let _viewer = producer + .consume() + .excluding(origin(8)) + .request_broadcast("room/alice", None); assert_eq!(fronts(), 2, "a viewer left the plain front"); } @@ -6096,7 +6111,7 @@ mod tests { .unwrap(); view.assert_next_active("room"); - let pending = peer.request_broadcast("room/alice"); + let pending = peer.request_broadcast("room/alice", None); let source = broadcast::Info::new().produce(); queued(&incumbent).await.accept(&source); let resolved = pending.await.expect("resolves through the incumbent"); @@ -6135,7 +6150,7 @@ mod tests { Route::default().with_epoch(epoch()).with_hops(hops(&[10])).with_cost(5), ) .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); + let pending = producer.consume().request_broadcast("room/alice", None); let upstream = broadcast::Info::new().produce(); let mut dynamic = upstream.dynamic(); queued(&first_server).await.accept(&upstream); @@ -6294,7 +6309,7 @@ mod tests { let broadcast = producer.publish("room/alice", Route::default()).unwrap(); let mut dynamic = broadcast.dynamic(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let track = resolved.track("video").unwrap(); let subscribing = moq_net_sim::spawn(async move { track.subscribe(None).await }); let request = moq_net_sim::timeout(Duration::from_secs(1), dynamic.requested_track()) @@ -6349,7 +6364,7 @@ mod tests { let consumer = producer.consume(); let broadcast = producer.publish("room/alice", Route::default()).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); broadcast.unannounce(); broadcast.announce(Route::default()).unwrap(); @@ -6357,7 +6372,7 @@ mod tests { moq_net_sim::yield_now().await; } assert!(!resolved.is_closed(), "the front ended across a reannouncement"); - let again = consumer.request_broadcast("room/alice").await.expect("resolves"); + let again = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert!(again.is_clone(&resolved)); } @@ -6369,7 +6384,7 @@ mod tests { // Created but not announced: nobody can reach it, locally included. let broadcast = producer.create_broadcast("room/alice").unwrap(); let err = consumer - .request_broadcast("room/alice") + .request_broadcast("room/alice", None) .now_or_never() .expect("unroutable is synchronous") .err() @@ -6377,13 +6392,13 @@ mod tests { assert!(matches!(err, Error::Unroutable)); broadcast.announce(Route::default()).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert_eq!(resolved.info().path.as_str(), "room/alice"); drop(broadcast); // Nothing covers an unknown path and no handler exists. let err = consumer - .request_broadcast("room/bob") + .request_broadcast("room/bob", None) .now_or_never() .expect("unroutable is synchronous") .err() @@ -6468,13 +6483,13 @@ mod tests { let refused = producer .consume() - .request_broadcast("room/video") + .request_broadcast("room/video", None) .now_or_never() .expect("an out-of-scope request must be refused synchronously"); assert!(matches!(refused, Err(Error::Unroutable))); assert!(dynamic.requested_broadcast().now_or_never().is_none()); - let _pending = producer.consume().request_broadcast("room/chat"); + let _pending = producer.consume().request_broadcast("room/chat", None); let request = queued(&dynamic).await; assert_eq!(request.path().as_str(), "room/chat"); } @@ -6580,7 +6595,7 @@ mod tests { let dynamic = producer.dynamic(&path, Route::default()).expect("max depth is allowed"); announced.assert_next_active(&path); - let _pending = producer.consume().request_broadcast(&path); + let _pending = producer.consume().request_broadcast(&path, None); let request = queued(&dynamic).await; assert_eq!(request.path().as_str(), path); } @@ -6659,7 +6674,7 @@ mod tests { let server = producer.dynamic("room", Route::default()).unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; assert_eq!(request.path().as_str(), "room/alice"); @@ -6671,7 +6686,7 @@ mod tests { assert_eq!(resolved.info().path.as_str(), "room/alice"); // A repeat request shares the served broadcast instead of re-asking. - let again = consumer.request_broadcast("room/alice").await.expect("resolves"); + let again = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert!(again.is_clone(&resolved)); } @@ -6681,8 +6696,8 @@ mod tests { let consumer = producer.consume(); let server = producer.dynamic("room", Route::default()).unwrap(); - let first = consumer.request_broadcast("room/alice"); - let second = consumer.request_broadcast("room/alice"); + let first = consumer.request_broadcast("room/alice", None); + let second = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; // Only one request reaches the server. @@ -6702,7 +6717,7 @@ mod tests { let consumer = producer.consume(); let server = producer.dynamic("room", Route::default()).unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); drop(server); let err = pending.await.err().unwrap(); @@ -6710,7 +6725,7 @@ mod tests { // With the route gone, later requests are unroutable immediately. let err = consumer - .request_broadcast("room/alice") + .request_broadcast("room/alice", None) .now_or_never() .expect("unroutable") .err() @@ -6760,7 +6775,7 @@ mod tests { // The requester's own bytes must not be served back to it. let excluded = producer.consume().excluding(origin(7)); let err = excluded - .request_broadcast("room/alice") + .request_broadcast("room/alice", None) .now_or_never() .expect("unroutable") .err() @@ -6769,7 +6784,7 @@ mod tests { // A clean requester resolves through the route (the request queues). let clean = producer.consume().excluding(origin(8)); - let pending = clean.request_broadcast("room/alice"); + let pending = clean.request_broadcast("room/alice", None); assert!(pending.now_or_never().is_none()); } @@ -7010,7 +7025,7 @@ mod tests { let err = producer .consume() .local() - .request_broadcast("remote/alice") + .request_broadcast("remote/alice", None) .now_or_never() .expect("unroutable") .err() @@ -7019,7 +7034,7 @@ mod tests { assert!( producer .consume() - .request_broadcast("remote/alice") + .request_broadcast("remote/alice", None) .now_or_never() .is_none() ); @@ -7074,7 +7089,7 @@ mod tests { // toward an upstream, while the full view queues it on the core's route. let err = core .consume() - .request_broadcast("core/alice") + .request_broadcast("core/alice", None) .now_or_never() .expect("unroutable") .err() @@ -7083,7 +7098,7 @@ mod tests { assert!( producer .consume() - .request_broadcast("core/alice") + .request_broadcast("core/alice", None) .now_or_never() .is_none() ); @@ -7098,7 +7113,7 @@ mod tests { let consumer = producer.consume(); let server = producer.dynamic("room", Route::default()).unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; request.reject(Error::Unroutable); let err = moq_net_sim::timeout(Duration::from_secs(5), pending) @@ -7109,7 +7124,7 @@ mod tests { assert!(matches!(err, Error::Unroutable)); // The route still stands and serves the next path. - let pending = consumer.request_broadcast("room/bob"); + let pending = consumer.request_broadcast("room/bob", None); let request = queued(&server).await; assert_eq!(request.path().as_str(), "room/bob"); let served = broadcast::Info::new().produce(); @@ -7127,7 +7142,7 @@ mod tests { let winner = producer.dynamic("room", Route::default()).unwrap(); let sibling = producer.dynamic("room", Route::default().with_cost(2)).unwrap(); - let mut pending = Box::pin(consumer.request_broadcast("room/alice")); + let mut pending = Box::pin(consumer.request_broadcast("room/alice", None)); assert!((&mut pending).now_or_never().is_none()); queued(&winner).await.reject(Error::NotFound); let mut result = None; @@ -7152,7 +7167,7 @@ mod tests { let consumer = producer.consume(); let stale = producer.dynamic("room", Route::default()).unwrap(); - let mut pending = Box::pin(consumer.request_broadcast("room/alice")); + let mut pending = Box::pin(consumer.request_broadcast("room/alice", None)); assert!((&mut pending).now_or_never().is_none()); let request = queued(&stale).await; @@ -7178,7 +7193,7 @@ mod tests { let consumer = producer.consume(); let stale = producer.dynamic("room", Route::default()).unwrap(); - let mut pending = Box::pin(consumer.request_broadcast("room/alice")); + let mut pending = Box::pin(consumer.request_broadcast("room/alice", None)); assert!((&mut pending).now_or_never().is_none()); let request = queued(&stale).await; @@ -7288,7 +7303,7 @@ mod tests { std::mem::forget(track); } - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); queued(&server).await.accept(&source); let resolved = pending.await.expect("resolves"); @@ -7321,7 +7336,7 @@ mod tests { let _narrow = producer.announce(".dash", Route::default()).unwrap(); let err = consumer - .request_broadcast(".dash/pid") + .request_broadcast(".dash/pid", None) .now_or_never() .expect("unroutable") .err() @@ -7329,7 +7344,7 @@ mod tests { assert!(matches!(err, Error::Unroutable)); // Everything else still routes to the broad server. - let _pending = consumer.request_broadcast("room/alice"); + let _pending = consumer.request_broadcast("room/alice", None); let request = queued(&broad_server).await; assert_eq!(request.path().as_str(), "room/alice"); } @@ -7343,7 +7358,7 @@ mod tests { // The root claim is advertised like any other prefix. announced.assert_next_active(""); - let pending = consumer.request_broadcast("anything/at/all"); + let pending = consumer.request_broadcast("anything/at/all", None); let request = queued(&dynamic).await; assert_eq!(request.path().as_str(), "anything/at/all"); @@ -7356,7 +7371,7 @@ mod tests { drop(dynamic); announced.assert_next_ended(""); let err = consumer - .request_broadcast("something/else") + .request_broadcast("something/else", None) .now_or_never() .expect("unroutable") .err() @@ -7379,7 +7394,7 @@ mod tests { // also pins that the check is segment-aware rather than textual. for path in ["tenant-b/live", "tenant-a-other/live"] { let refused = scoped - .request_broadcast(path) + .request_broadcast(path, None) .now_or_never() .expect("an out-of-scope request must be refused synchronously, not queued"); assert!(matches!(refused, Err(Error::Unauthorized))); @@ -7454,7 +7469,7 @@ mod tests { announced.assert_next_active("room"); let _server = producer.dynamic("served", Route::default()).unwrap(); - let pending = consumer.request_broadcast("served/path"); + let pending = consumer.request_broadcast("served/path", None); drop(driver); @@ -7467,7 +7482,7 @@ mod tests { assert!(matches!(producer.announce("x", Route::default()), Err(Error::Closed))); assert!(matches!(producer.create_broadcast("x"), Err(Error::Closed))); let err = consumer - .request_broadcast("y") + .request_broadcast("y", None) .now_or_never() .expect("closed") .err() @@ -7501,7 +7516,7 @@ mod tests { .dynamic("room", Route::default().with_epoch(epoch()).with_hops(hops(first))) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; let source = broadcast::Info::new().produce(); let track = source.create_track("video", None).unwrap(); @@ -7594,7 +7609,7 @@ mod tests { let first = producer .dynamic("room", Route::default().with_hops(hops(&[10])).with_cost(5)) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let source = broadcast::Info::new().produce(); let track = source.create_track("video", None).unwrap(); queued(&first).await.accept(&source); @@ -7635,7 +7650,7 @@ mod tests { ) .unwrap(); let old_track = old.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); deliver(&old_track, &mut subscription, b"old").await; @@ -7648,7 +7663,7 @@ mod tests { let new_track = new.create_track("video", None).unwrap(); assert!(matches!(next_group(&mut subscription).await, Err(Error::Unroutable))); - let replaced = consumer.request_broadcast("room/alice").await.expect("resolves"); + let replaced = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert!(!replaced.is_clone(&resolved)); let mut subscription = replaced.track("video").unwrap().subscribe(None).await.unwrap(); deliver(&new_track, &mut subscription, b"new").await; @@ -7665,15 +7680,15 @@ mod tests { .publish("room/alice", Route::default().with_epoch(crate::Epoch::mint())) .unwrap(); let _old_track = old.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let epoch = crate::Epoch::mint(); let new = producer .publish("room/alice", Route::default().with_epoch(epoch.clone())) .unwrap(); let new_track = new.create_track("video", None).unwrap(); - let named = consumer.request(Path::new("room/alice"), Some(&epoch)); - let bare = consumer.request_broadcast("room/alice"); + let named = consumer.request_broadcast("room/alice", epoch.clone()); + let bare = consumer.request_broadcast("room/alice", None); let named = named.await.expect("the new epoch resolves"); let bare = bare.await.expect("the new epoch resolves"); @@ -7696,7 +7711,7 @@ mod tests { let old = producer .dynamic("room", Route::default().with_epoch(crate::Epoch::mint())) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&old).await; // The answer and the newer epoch land in the same tick. @@ -7753,10 +7768,10 @@ mod tests { .unwrap(); let other: crate::Epoch = "01900000-0000-7000-8000-000000000002".parse().unwrap(); - let refused = consumer.request(Path::new("room/alice"), Some(&other)).await; + let refused = consumer.request_broadcast("room/alice", other).await; assert!(matches!(refused, Err(Error::Unroutable))); consumer - .request(Path::new("room/alice"), Some(&epoch())) + .request_broadcast("room/alice", epoch()) .await .expect("the named epoch resolves"); } @@ -7856,7 +7871,12 @@ mod tests { let frame = group.read_frame().await.expect("read frame").expect("frame"); assert_eq!(&frame.payload[..], b"after", "first hop {first:?}"); - let resolved = rig.producer.consume().request_broadcast("room/alice").await.unwrap(); + let resolved = rig + .producer + .consume() + .request_broadcast("room/alice", None) + .await + .unwrap(); assert!(resolved.is_clone(&rig.resolved), "first hop {first:?} left the front"); assert!( standby.poll_requested_broadcast(&kio::Waiter::noop()).is_pending(), @@ -7884,7 +7904,7 @@ mod tests { .dynamic("room", Route::default().with_epoch(epoch()).with_hops(hops(&[10, 7]))) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&incumbent).await; let source = broadcast::Info::new().produce(); let track = source.create_track("video", None).unwrap(); @@ -7949,7 +7969,7 @@ mod tests { // Session C: a third anonymous session, excluding the hop the server // minted for it, the same split-horizon a live session applies. let consumer = producer.consume().excluding(origin(30)); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server_a).await; let source_a = broadcast::Info::new().produce(); let track_a = source_a.create_track("video", None).unwrap(); @@ -7991,7 +8011,7 @@ mod tests { settle(|| consumer.get_broadcast("room/alice").is_none()).await; settle(|| { matches!( - consumer.request_broadcast("room/alice").now_or_never(), + consumer.request_broadcast("room/alice", None).now_or_never(), Some(Err(Error::Unroutable)) ) }) @@ -8001,7 +8021,7 @@ mod tests { let server_b = producer .dynamic("room", Route::default().with_hops(hops(&[20]))) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server_b).await; let source_b = broadcast::Info::new().produce(); let track_b = source_b.create_track("video", None).unwrap(); @@ -8094,11 +8114,11 @@ mod tests { let consumer = producer.consume(); let first = producer.publish("room/alice", Route::default()).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); // A second source at the same path joins the same front. let second = producer.publish("room/alice", Route::default()).unwrap(); - let again = consumer.request_broadcast("room/alice").await.expect("resolves"); + let again = consumer.request_broadcast("room/alice", None).await.expect("resolves"); assert!(again.is_clone(&resolved)); // Losing one source keeps the front alive; losing both closes it. @@ -8127,7 +8147,7 @@ mod tests { let broadcast = producer.publish("room/alice", Route::default()).unwrap(); let track = broadcast.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved .track("video") .unwrap() @@ -8169,7 +8189,7 @@ mod tests { .dynamic("room", Route::default().with_hops(hops(&[10]))) .unwrap(); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; let source = broadcast::Info::new().produce(); let track = source.create_track("video", None).unwrap(); @@ -8215,7 +8235,7 @@ mod tests { let server = producer .dynamic("room", Route::default().with_hops(hops(&[10]))) .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); + let pending = producer.consume().request_broadcast("room/alice", None); let source = broadcast::Info::new().produce(); queued(&server).await.accept(&source); let resolved = pending.await.unwrap(); @@ -8243,7 +8263,7 @@ mod tests { group.write_frame(crate::Timestamp::ZERO, b"cached".as_ref()).unwrap(); group.finish().unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved .track("video") .unwrap() @@ -8300,7 +8320,7 @@ mod tests { let server = producer .dynamic("live", Route::default().with_hops(hops(&[10]))) .unwrap(); - let pending = producer.consume().request_broadcast("live"); + let pending = producer.consume().request_broadcast("live", None); let source = broadcast::Info::new().produce(); let track = source.create_track("video", None).unwrap(); queued(&server).await.accept(&source); @@ -8339,7 +8359,7 @@ mod tests { let producer = origin(1).produce(); let broadcast = producer.publish("live", Route::default()).unwrap(); let track = broadcast.create_track("video", None).unwrap(); - let resolved = producer.consume().request_broadcast("live").await.unwrap(); + let resolved = producer.consume().request_broadcast("live", None).await.unwrap(); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); let mut aborted = track.append_group().unwrap(); @@ -8378,17 +8398,20 @@ mod tests { group.finish().unwrap(); // The leaf's front view, which the next front serves from. - let leaf_front = leaf_consumer.request_broadcast("room/alice").await.expect("resolves"); + let leaf_front = leaf_consumer + .request_broadcast("room/alice", None) + .await + .expect("resolves"); let mid = origin(2).produce(); let mid_server = mid.dynamic("room", Route::default().with_hops(hops(&[10]))).unwrap(); - let mid_pending = mid.consume().request_broadcast("room/alice"); + let mid_pending = mid.consume().request_broadcast("room/alice", None); queued(&mid_server).await.accept(&leaf_front); let mid_resolved = mid_pending.await.expect("mid resolves"); let edge = origin(3).produce(); let edge_server = edge.dynamic("room", Route::default().with_hops(hops(&[20]))).unwrap(); - let edge_pending = edge.consume().request_broadcast("room/alice"); + let edge_pending = edge.consume().request_broadcast("room/alice", None); queued(&edge_server).await.accept(&mid_resolved); let edge_resolved = edge_pending.await.expect("edge resolves"); @@ -8466,7 +8489,7 @@ mod tests { .publish("room/alice", Route::default().with_epoch(epoch())) .unwrap(); let track = first.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved .track("video") .unwrap() @@ -8516,7 +8539,7 @@ mod tests { let server = producer .dynamic("room", Route::default().with_epoch(epoch()).with_hops(hops(&[10]))) .unwrap(); - let pending = producer.consume().request_broadcast("room/alice"); + let pending = producer.consume().request_broadcast("room/alice", None); let upstream = broadcast::Info::new().produce(); let old = upstream.create_track("video", None).unwrap(); queued(&server).await.accept(&upstream); @@ -8559,7 +8582,7 @@ mod tests { #[moq_net_sim::test] async fn a_returning_reader_is_not_stranded_on_a_dropped_copy() { let (producer, _server, upstream, old, mut subscription) = remote_front().await; - let resolved = producer.consume().request_broadcast("room/alice").await.unwrap(); + let resolved = producer.consume().request_broadcast("room/alice", None).await.unwrap(); // No handler and no track: refuses "video" with NotFound. let _second = producer @@ -8620,7 +8643,7 @@ mod tests { .publish("room/alice", Route::default().with_epoch(epoch())) .unwrap(); let track = first.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = track.append_group().unwrap(); group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); @@ -8656,7 +8679,7 @@ mod tests { .publish("room/alice", Route::default().with_epoch(epoch())) .unwrap(); let track = first.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = track.append_group().unwrap(); group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); @@ -8711,7 +8734,7 @@ mod tests { .publish("room/alice", Route::default().with_epoch(epoch())) .unwrap(); let track = first.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = track.append_group().unwrap(); group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); @@ -8844,7 +8867,7 @@ mod tests { let server = producer.dynamic("room", Route::default()).unwrap(); drop(producer); - let pending = consumer.request_broadcast("room/alice"); + let pending = consumer.request_broadcast("room/alice", None); let request = queued(&server).await; let source = broadcast::Info::new().produce(); request.accept(&source); @@ -8869,7 +8892,7 @@ mod tests { let consumer = producer.consume(); let broadcast = producer.publish("room/alice", Route::default()).unwrap(); let track = broadcast.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let sub = resolved.track("video").unwrap().subscribe(None).await.unwrap(); broadcast.unannounce(); @@ -8893,7 +8916,7 @@ mod tests { let consumer = producer.consume(); let broadcast = producer.publish("room/alice", Route::default()).unwrap(); let mut track = broadcast.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let sub = resolved.track("video").unwrap().subscribe(None).await.unwrap(); let pending = resolved.track("video").unwrap().subscribe(None); @@ -8920,7 +8943,7 @@ mod tests { let consumer = producer.consume(); let broadcast = producer.publish("room/alice", Route::default()).unwrap(); let track = broadcast.create_track("video", None).unwrap(); - let resolved = consumer.request_broadcast("room/alice").await.expect("resolves"); + let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); let sub = resolved.track("video").unwrap().subscribe(None).await.unwrap(); moq_net_sim::timeout(Duration::from_secs(1), track.demand().used()) .await @@ -8990,7 +9013,7 @@ mod tests { // dropped on the spot, taking its `Watch` with it: the request must not // still be holding the table lock the watch unregisters under. drop(producer); - let _pending = consumer.request_broadcast("room/a"); + let _pending = consumer.request_broadcast("room/a", None); } #[test] diff --git a/rs/moq-net/tests/announce_to_serve.rs b/rs/moq-net/tests/announce_to_serve.rs index 395aa5a0097..02a2ad23c42 100644 --- a/rs/moq-net/tests/announce_to_serve.rs +++ b/rs/moq-net/tests/announce_to_serve.rs @@ -53,7 +53,7 @@ fn drain(announced: &mut moq_net::announce::Consumer) -> Vec { /// How a request for the path answers. async fn request(consumer: &origin::Consumer) -> Result { - consumer.request_broadcast(PATH).await.map_err(|err| match err { + consumer.request_broadcast(PATH, None).await.map_err(|err| match err { Error::Unroutable => "unroutable".to_string(), err => err.to_string(), }) diff --git a/rs/moq-net/tests/broadcast_close.rs b/rs/moq-net/tests/broadcast_close.rs index a0440bb155a..bf0700b0701 100644 --- a/rs/moq-net/tests/broadcast_close.rs +++ b/rs/moq-net/tests/broadcast_close.rs @@ -29,7 +29,7 @@ async fn close_then_lookup(publisher: moq_net::origin::Producer, reader: moq_net .await .expect("announce timeout") .expect("routed"); - let handle = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + let handle = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); @@ -43,7 +43,7 @@ async fn close_then_lookup(publisher: moq_net::origin::Producer, reader: moq_net assert!(matches!(handle.track("video"), Err(Error::Unroutable))); assert!(matches!(handle.track("audio"), Err(Error::Unroutable))); assert!(matches!( - consumer.request_broadcast("bcast").await, + consumer.request_broadcast("bcast", None).await, Err(Error::Unroutable) )); assert!(matches!(broadcast.announce(Default::default()), Err(Error::Closed))); diff --git a/rs/moq-net/tests/datagram.rs b/rs/moq-net/tests/datagram.rs index 9386e4727d3..360d0c22162 100644 --- a/rs/moq-net/tests/datagram.rs +++ b/rs/moq-net/tests/datagram.rs @@ -48,7 +48,7 @@ async fn connect_datagram_track() -> Fixture { let consumer = consumer_origin.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let subscriber = remote.track("datagrams").unwrap().subscribe(None).await.unwrap(); Fixture { @@ -140,7 +140,7 @@ async fn ietf_delivers_datagrams_on(version: &str) { let consumer = consumer_origin.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let mut subscriber = remote.track("datagrams").unwrap().subscribe(None).await.unwrap(); // A group first, so the subscription's alias is bound before any datagram lands. diff --git a/rs/moq-net/tests/fetch_cancel.rs b/rs/moq-net/tests/fetch_cancel.rs index 3d137daa15b..94801e0a3da 100644 --- a/rs/moq-net/tests/fetch_cancel.rs +++ b/rs/moq-net/tests/fetch_cancel.rs @@ -47,7 +47,7 @@ async fn abandoned_fetch_reaches_the_publisher(version: &str, relays: u64, stage let consumer = nodes[relays as usize] .consume() - .request_broadcast("room") + .request_broadcast("room", None) .await .unwrap(); let track = consumer.track("video").unwrap(); @@ -171,7 +171,7 @@ async fn a_refetch_within_the_linger_stays_on_the_relay(version: &str) { broadcast.announce(Default::default()).unwrap(); moq_net_sim::sleep(Duration::from_secs(1)).await; - let consumer = relay.consume().request_broadcast("room").await.unwrap(); + let consumer = relay.consume().request_broadcast("room", None).await.unwrap(); let mut waiting = Box::pin(consumer.track("video").unwrap().fetch_group(0, None)); let request = match futures::future::select(std::pin::pin!(dynamic.requested_track()), &mut waiting).await { futures::future::Either::Left((request, _)) => request.expect("the track request reaches the publisher"), diff --git a/rs/moq-net/tests/finished_broadcast_mock.rs b/rs/moq-net/tests/finished_broadcast_mock.rs index 31b6de03271..bdf7267968b 100644 --- a/rs/moq-net/tests/finished_broadcast_mock.rs +++ b/rs/moq-net/tests/finished_broadcast_mock.rs @@ -47,7 +47,7 @@ async fn round(finish_broadcast: bool) -> (Vec>, Option) .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); diff --git a/rs/moq-net/tests/goaway.rs b/rs/moq-net/tests/goaway.rs index 4dbe7922f26..ebefa7d43c9 100644 --- a/rs/moq-net/tests/goaway.rs +++ b/rs/moq-net/tests/goaway.rs @@ -343,7 +343,7 @@ async fn goaway_gates_new_subscribes_moq_lite_04() { // Subscribe BEFORE the GOAWAY and receive a first group. let sub = sub_origin.consume(); sub.routed("test").await.expect("route announced"); - let bc = sub.request_broadcast("test").await.expect("broadcast resolves"); + let bc = sub.request_broadcast("test", None).await.expect("broadcast resolves"); let mut existing = bc.track("video").unwrap().subscribe(None).await.expect("subscribe"); let mut group = track.append_group().expect("append group"); diff --git a/rs/moq-net/tests/head_fetch.rs b/rs/moq-net/tests/head_fetch.rs index 2b43733c534..2323814fa4b 100644 --- a/rs/moq-net/tests/head_fetch.rs +++ b/rs/moq-net/tests/head_fetch.rs @@ -38,7 +38,7 @@ async fn resolve(origin: &moq_net::origin::Producer) -> moq_net::broadcast::Cons .await .expect("announce timeout") .expect("routed"); - moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves") diff --git a/rs/moq-net/tests/history_groups.rs b/rs/moq-net/tests/history_groups.rs index 62db30709fc..5583eec53bb 100644 --- a/rs/moq-net/tests/history_groups.rs +++ b/rs/moq-net/tests/history_groups.rs @@ -71,7 +71,7 @@ async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usiz .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); @@ -217,7 +217,7 @@ async fn a_late_subscriber_receives_the_relays_cached_history() { let first_pair = connect_mock(options).await; let consumer = first.consume(); consumer.routed("bcast").await.unwrap(); - let remote = consumer.request_broadcast("bcast").await.unwrap(); + let remote = consumer.request_broadcast("bcast", None).await.unwrap(); let subscription = Subscription::default() .with_max_age(FOREVER) .with_start(Position::group(0)); @@ -257,7 +257,7 @@ async fn a_late_subscriber_receives_the_relays_cached_history() { let late_pair = connect_mock(options).await; let consumer = late.consume(); consumer.routed("bcast").await.unwrap(); - let remote = consumer.request_broadcast("bcast").await.unwrap(); + let remote = consumer.request_broadcast("bcast", None).await.unwrap(); let mut late_sub = remote.track("history").unwrap().subscribe(subscription).await.unwrap(); moq_net_sim::timeout(TIMEOUT, read_history(&mut late_sub)) .await diff --git a/rs/moq-net/tests/rejoin.rs b/rs/moq-net/tests/rejoin.rs index f8621f9cd67..f725f85a8d1 100644 --- a/rs/moq-net/tests/rejoin.rs +++ b/rs/moq-net/tests/rejoin.rs @@ -49,7 +49,7 @@ async fn rejoin_recovers_the_group_reset_on_leave() { let consumer = relay.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let ts = |ms| Timestamp::from_millis(ms).unwrap(); let prefs = || track::Subscription::default().with_max_age(Duration::from_secs(10)); @@ -142,7 +142,7 @@ async fn rejoin_skips_a_cache_kept_by_another_handle() { let consumer = relay.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let ts = |ms| Timestamp::from_millis(ms).unwrap(); // Held for fetches only: it keeps the relay's copy without subscribing. @@ -214,7 +214,7 @@ async fn rejoin_mid_group_keeps_the_head_for_later_readers() { let consumer = client.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload, b"a0".as_ref()); @@ -226,7 +226,7 @@ async fn rejoin_mid_group_keeps_the_head_for_later_readers() { assert_eq!(rejoined.read_frame().await.unwrap().unwrap().payload, b"a0".as_ref()); track.demand().used().await.unwrap(); - let later = relay.consume().request_broadcast("bench").await.unwrap(); + let later = relay.consume().request_broadcast("bench", None).await.unwrap(); let mut sub = later.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(group.sequence, 0, "{version}"); @@ -286,7 +286,7 @@ async fn rejoin_goes_live_without_the_join_head() { let consumer = client.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload, b"a0".as_ref()); @@ -313,7 +313,7 @@ async fn rejoin_goes_live_without_the_join_head() { .unwrap(); next.finish().unwrap(); - let local = relay.consume().request_broadcast("bench").await.unwrap(); + let local = relay.consume().request_broadcast("bench", None).await.unwrap(); let mut sub = local.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(group.sequence, 1, "{version}"); @@ -350,7 +350,7 @@ async fn rejoin_during_the_cancel_skips_the_cache() { let consumer = relay.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let ts = |ms| Timestamp::from_millis(ms).unwrap(); let mut group = track.append_group().unwrap(); @@ -423,7 +423,7 @@ async fn leaving_after_the_cache_window_keeps_the_latest_group() { let pair = connect_mock(options).await; let consumer = client.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); (remote, pair) } }; diff --git a/rs/moq-net/tests/route_change.rs b/rs/moq-net/tests/route_change.rs index f52ecdf8500..e42c0ce8392 100644 --- a/rs/moq-net/tests/route_change.rs +++ b/rs/moq-net/tests/route_change.rs @@ -101,7 +101,7 @@ impl Topology { let consumer = subscriber.consume(); consumer.routed("live").await.unwrap(); - let remote = consumer.request_broadcast("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); let prefs = track::Subscription::default().with_max_age(Duration::from_secs(60)); let sub = remote.track("video").unwrap().subscribe(prefs).await.unwrap(); @@ -428,7 +428,7 @@ async fn lagging_route_dies(version: Version) -> mpsc::UnboundedReceiver<(u64, m async fn subscribe(origin: &origin::Producer) -> track::Subscriber { let consumer = origin.consume(); consumer.routed("live").await.unwrap(); - let remote = consumer.request_broadcast("live").await.unwrap(); + let remote = consumer.request_broadcast("live", None).await.unwrap(); let preferences = track::Subscription::default().with_max_age(Duration::from_secs(60)); remote.track("video").unwrap().subscribe(preferences).await.unwrap() } diff --git a/rs/moq-net/tests/session_close.rs b/rs/moq-net/tests/session_close.rs index 5795737038e..31d70bc6a8f 100644 --- a/rs/moq-net/tests/session_close.rs +++ b/rs/moq-net/tests/session_close.rs @@ -46,7 +46,7 @@ async fn setup(version: &str) -> Setup { .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); diff --git a/rs/moq-net/tests/subscribe_tracks.rs b/rs/moq-net/tests/subscribe_tracks.rs index 3104018edd3..af34c42403e 100644 --- a/rs/moq-net/tests/subscribe_tracks.rs +++ b/rs/moq-net/tests/subscribe_tracks.rs @@ -57,7 +57,7 @@ async fn setup(version: Version) -> Setup { .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("room")) + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("room", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); diff --git a/rs/moq-net/tests/subscription_end_integrity.rs b/rs/moq-net/tests/subscription_end_integrity.rs index 34d4f78636b..401c75a6d9c 100644 --- a/rs/moq-net/tests/subscription_end_integrity.rs +++ b/rs/moq-net/tests/subscription_end_integrity.rs @@ -64,7 +64,7 @@ async fn round(drop_session: bool) -> (Vec>, Option) { .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); @@ -211,7 +211,10 @@ async fn killed(version: &str, local: bool) -> (Option, Option Pair { .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast")) + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("bcast", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); diff --git a/rs/moq-net/tests/unannounce_release.rs b/rs/moq-net/tests/unannounce_release.rs index 30f786d9132..a3e0f15269b 100644 --- a/rs/moq-net/tests/unannounce_release.rs +++ b/rs/moq-net/tests/unannounce_release.rs @@ -27,7 +27,7 @@ async fn release(version: Version) { let consumer = relay.consume(); consumer.routed("a/b").await.expect("routed"); - let remote = consumer.request_broadcast("a/b").await.expect("resolves"); + let remote = consumer.request_broadcast("a/b", None).await.expect("resolves"); let sub = remote.track("t").unwrap().subscribe(None).await.expect("subscribes"); track.demand().used().await.unwrap(); diff --git a/rs/moq-net/tests/wide_sequence.rs b/rs/moq-net/tests/wide_sequence.rs index 6fa8898a360..536b8c39a74 100644 --- a/rs/moq-net/tests/wide_sequence.rs +++ b/rs/moq-net/tests/wide_sequence.rs @@ -43,13 +43,13 @@ async fn wide_sequence_fails_only_its_subscription() { // lite-07 carries the full 64 bits, so the relay holds the group. relay.consume().routed("bench").await.unwrap(); - let cached = relay.consume().request_broadcast("bench").await.unwrap(); + let cached = relay.consume().request_broadcast("bench", None).await.unwrap(); let mut sub = cached.track("wide").unwrap().subscribe(None).await.unwrap(); assert_eq!(sub.recv_group().await.unwrap().unwrap().sequence, WIDE); let consumer = client.consume(); consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench", None).await.unwrap(); let result = moq_net_sim::timeout(Duration::from_secs(5), async { let mut sub = remote.track("wide").unwrap().subscribe(None).await?; diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index e14196b6b04..6161b25b883 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1980,7 +1980,7 @@ mod tests { .expect("publish at the fleet path"); let subscriber = cluster.subscriber(&token).expect("subscribe grant").consume(); let broadcast = subscriber - .request_broadcast(".svc/foo") + .request_broadcast(".svc/foo", None) .await .expect("resolves through the mount"); assert_eq!(broadcast.info().path.as_str(), ".svc/foo"); @@ -2046,7 +2046,7 @@ mod tests { .expect("stats broadcast present"); assert!(route.epoch.is_some(), "stats announce under an epoch"); let broadcast = consumer - .request_broadcast(&path) + .request_broadcast(&path, None) .await .expect("stats broadcast resolves"); @@ -2608,7 +2608,7 @@ mod tests { assert!(pool.used() > 0, "writes charge the constructed cache pool"); let consumer = cluster.origin.consume(); - tokio::time::timeout(Duration::from_secs(2), consumer.request_broadcast("cam")) + tokio::time::timeout(Duration::from_secs(2), consumer.request_broadcast("cam", None)) .await .expect("broadcast resolves") .expect("broadcast present"); diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index da6f201d32d..a76038a2a4d 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1018,7 +1018,7 @@ mod tests { let (update, active) = next_update(&mut announced).await.unwrap(); assert!(active); let bc = egress - .request_broadcast(moq_net::Path::new(update.prefix.as_str())) + .request_broadcast(moq_net::Path::new(update.prefix.as_str()), None) .await .unwrap(); let mut egress_sub = bc.track("video").unwrap().subscribe(None).await.unwrap(); diff --git a/rs/moq-relay/tests/auth_lifetime.rs b/rs/moq-relay/tests/auth_lifetime.rs index 9163df0bf9b..c8585768c00 100644 --- a/rs/moq-relay/tests/auth_lifetime.rs +++ b/rs/moq-relay/tests/auth_lifetime.rs @@ -252,7 +252,7 @@ async fn connect_and_round_trip(url: &url::Url) -> (moq_tokio::Connection, moq_t assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); let bc = sub_consumer - .request_broadcast("test") + .request_broadcast("test", None) .await .expect("announced broadcast resolves"); @@ -484,7 +484,7 @@ async fn a_moved_tier_retags_the_live_session() { assert_eq!(update.prefix.as_str(), "test"); assert!(announced, "expected announce, got retraction"); let bc = sub_consumer - .request_broadcast("test") + .request_broadcast("test", None) .await .expect("announced broadcast resolves"); let mut track_sub = bc.track("video").unwrap().subscribe(None).await.expect("subscribe"); diff --git a/rs/moq-relay/tests/cluster_unknown.rs b/rs/moq-relay/tests/cluster_unknown.rs index 96f33fc0ad2..81c3ba0728a 100644 --- a/rs/moq-relay/tests/cluster_unknown.rs +++ b/rs/moq-relay/tests/cluster_unknown.rs @@ -130,7 +130,7 @@ async fn read_first_frame(port: u16) -> Result, String> { .map_err(|_| "routed timed out".to_string())? .ok_or_else(|| "origin closed before the broadcast was announced".to_string())?; let broadcast = consumer - .request_broadcast(PATH) + .request_broadcast(PATH, None) .await .map_err(|err| format!("broadcast unroutable: {err}"))?; diff --git a/rs/moq-relay/tests/embed.rs b/rs/moq-relay/tests/embed.rs index eb4026f3d23..e9c0cf0f210 100644 --- a/rs/moq-relay/tests/embed.rs +++ b/rs/moq-relay/tests/embed.rs @@ -283,7 +283,7 @@ async fn embed_and_stop(mut config: Config) { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let announced = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let announced = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("announced broadcast resolves"); diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index f4f497dde9c..d95ec981a1c 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -158,7 +158,10 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { let session_a = within("A accepts", accepted_a.recv()).await.expect("A accepts"); let consumer = cluster.origin.consume(); within("routed via A", consumer.routed("cam")).await.expect("routed"); - let bc = consumer.request_broadcast("cam").await.expect("broadcast resolves"); + let bc = consumer + .request_broadcast("cam", None) + .await + .expect("broadcast resolves"); let mut sub = within("subscribe", bc.track("video").expect("track").subscribe(None)) .await .expect("subscribe"); @@ -304,7 +307,10 @@ async fn cluster_migrates_on_upstream_goaway_inner() { .routed("cam") .await .expect("broadcast announced through sibling A"); - let bc = consumer.request_broadcast("cam").await.expect("broadcast resolves"); + let bc = consumer + .request_broadcast("cam", None) + .await + .expect("broadcast resolves"); let mut sub = bc .track("video") .expect("track handle") @@ -542,7 +548,7 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { let bc = within("broadcast resolves on the subscriber origin", async { let consumer = sub_origin.consume(); consumer.routed("diamond").await?; - consumer.request_broadcast("diamond").await.ok() + consumer.request_broadcast("diamond", None).await.ok() }) .await .expect("broadcast announced"); @@ -771,7 +777,7 @@ async fn cluster_reconnects_on_empty_uri_goaway_inner() { let bc = within("broadcast announced", async { let consumer = cluster.origin.consume(); consumer.routed("cam").await?; - consumer.request_broadcast("cam").await.ok() + consumer.request_broadcast("cam", None).await.ok() }) .await .expect("broadcast announced"); diff --git a/rs/moq-relay/tests/public_rules.rs b/rs/moq-relay/tests/public_rules.rs index 82ef2ea33b0..91dd8afe974 100644 --- a/rs/moq-relay/tests/public_rules.rs +++ b/rs/moq-relay/tests/public_rules.rs @@ -109,7 +109,10 @@ async fn first_broadcast(url: url::Url) -> String { .expect("announcement timeout"); let name = update.prefix.to_string(); - let broadcast = consumer.request_broadcast(&name).await.expect("broadcast resolves"); + let broadcast = consumer + .request_broadcast(&name, None) + .await + .expect("broadcast resolves"); let mut track = broadcast .track("video") .unwrap() diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index 2b013656cdf..51a492afa31 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -150,7 +150,7 @@ async fn uring_workers_serve_webtransport_and_raw_quic() { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .unwrap_or_else(|_| panic!("subscriber {index} request timeout")) .expect("announced broadcast resolves"); @@ -438,7 +438,7 @@ async fn an_mtls_client_authenticates_without_a_token() { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let announced = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let announced = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("announced broadcast resolves"); diff --git a/rs/moq-relay/tests/runtime_workers.rs b/rs/moq-relay/tests/runtime_workers.rs index 5cf7b159973..80bf4b02af9 100644 --- a/rs/moq-relay/tests/runtime_workers.rs +++ b/rs/moq-relay/tests/runtime_workers.rs @@ -110,7 +110,7 @@ async fn workers_serve_quic_and_share_one_origin() { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .unwrap_or_else(|_| panic!("subscriber {index} request timeout")) .expect("announced broadcast resolves"); diff --git a/rs/moq-relay/tests/smoke.rs b/rs/moq-relay/tests/smoke.rs index b42540064c1..85f8316ae34 100644 --- a/rs/moq-relay/tests/smoke.rs +++ b/rs/moq-relay/tests/smoke.rs @@ -179,7 +179,7 @@ async fn relay_websocket_round_trip_uses_newest_version() { // Auth root for `/smoke` is "smoke"; the broadcast "test" announces underneath. assert_eq!(path.as_str(), "test"); let bc = sub_consumer - .request_broadcast(&path) + .request_broadcast(&path, None) .await .expect("announced broadcast resolves"); @@ -296,7 +296,7 @@ async fn hidden_broadcasts_need_a_lite07_opt_in() { continue; } let bc = consumer - .request_broadcast(".x/y") + .request_broadcast(".x/y", None) .await .expect("hidden broadcast resolves"); let mut sub = bc.track("video").unwrap().subscribe(None).await.expect("subscribe"); @@ -472,7 +472,7 @@ async fn relay_websocket_root_path_upgrades() { assert!(active, "expected announce, got retraction"); assert_eq!(path.as_str(), "test"); let bc = sub_consumer - .request_broadcast(&path) + .request_broadcast(&path, None) .await .expect("announced broadcast resolves"); @@ -691,7 +691,7 @@ async fn internal_tcp_round_trip() { assert!(active, "expected announce, got retraction"); assert_eq!(path.as_str(), "test"); let bc = sub_consumer - .request_broadcast(&path) + .request_broadcast(&path, None) .await .expect("announced broadcast resolves"); @@ -790,7 +790,7 @@ async fn internal_unix_round_trip() { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("announced broadcast resolves"); diff --git a/rs/moq-room/src/room.rs b/rs/moq-room/src/room.rs index a0986ff1d94..5b0d7915272 100644 --- a/rs/moq-room/src/room.rs +++ b/rs/moq-room/src/room.rs @@ -99,7 +99,7 @@ impl Room { })); } - let mut request = self.origin.request_broadcast(&path).into_inner(); + let mut request = self.origin.request_broadcast(&path, None).into_inner(); match kio::Task::poll(&mut request, waiter) { Poll::Ready(Ok(broadcast)) => { return Poll::Ready(Some(Event { diff --git a/rs/moq-rtmp/src/dial.rs b/rs/moq-rtmp/src/dial.rs index 1a0572b5adf..70cf1bbd31c 100644 --- a/rs/moq-rtmp/src/dial.rs +++ b/rs/moq-rtmp/src/dial.rs @@ -598,7 +598,7 @@ mod tests { .await .expect("client republish timed out") .expect("broadcast announced in client origin"); - let broadcast = announced.request_broadcast("pulled/cam0").await.unwrap(); + let broadcast = announced.request_broadcast("pulled/cam0", None).await.unwrap(); // It should carry a hang catalog track (proof the FLV demux produced real // media on the far side): subscribe to it and read one catalog frame. diff --git a/rs/moq-rtmp/src/server.rs b/rs/moq-rtmp/src/server.rs index 12b003db56c..6b261808ac8 100644 --- a/rs/moq-rtmp/src/server.rs +++ b/rs/moq-rtmp/src/server.rs @@ -1740,7 +1740,7 @@ mod tests { let consumer = origin.consume(); consumer.routed("live/cam0").await.unwrap(); - let broadcast = consumer.request_broadcast("live/cam0").await.unwrap(); + let broadcast = consumer.request_broadcast("live/cam0", None).await.unwrap(); let info = broadcast.track("0.flv-v").unwrap().query().await.unwrap(); assert_eq!(info.max_age, Some(Duration::from_secs(3))); } diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index 8935c3501cd..2b488af9814 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -421,7 +421,7 @@ mod tests { .await .expect("announce timed out") .expect("the broadcast is announced"); - let broadcast = consumer.request_broadcast(path).await.unwrap(); + let broadcast = consumer.request_broadcast(path, None).await.unwrap(); let mut catalog = moq_mux::catalog::Consumer::::new(&broadcast, CatalogFormat::Hang) .await .unwrap(); @@ -493,7 +493,7 @@ mod tests { let consumer = origin.consume(); consumer.routed("live/cam0").await.unwrap(); - let broadcast = consumer.request_broadcast("live/cam0").await.unwrap(); + let broadcast = consumer.request_broadcast("live/cam0", None).await.unwrap(); let info = broadcast.track("0.avc3").unwrap().query().await.unwrap(); assert_eq!(info.max_age, Some(Duration::from_secs(3))); } @@ -543,7 +543,7 @@ mod tests { .await .expect("announce timed out") .expect("the ingest broadcast is announced"); - let broadcast = consumer.request_broadcast("ingest").await.unwrap(); + let broadcast = consumer.request_broadcast("ingest", None).await.unwrap(); publisher.feed(bytes::Bytes::from_static(BBB5S)).unwrap(); diff --git a/rs/moq-stats/benches/aggregate.rs b/rs/moq-stats/benches/aggregate.rs index e6ab87594a7..0102206aa6a 100644 --- a/rs/moq-stats/benches/aggregate.rs +++ b/rs/moq-stats/benches/aggregate.rs @@ -1,5 +1,6 @@ //! Cost of one merged traffic frame from [`moq_stats::aggregate`], swept over live nodes and //! nodes that have come and gone, with and without the grace that folds departed nodes away. +//! Compare known metadata with one touched publisher losing and recovering its epoch. //! //! Each node reports the same broadcast paths, like a restarted node under a new name. A bounded //! aggregate's cost follows the live nodes; an unbounded one (`grace` never elapsing) also pays for @@ -9,7 +10,7 @@ use std::time::{Duration, Instant}; -use moq_net::origin; +use moq_net::{Epoch, origin}; use moq_stats::{Role, Tier, Traffic, TrafficFrame, aggregate, traffic_track}; /// Broadcast paths each node reports. @@ -23,7 +24,9 @@ const GRACE: Duration = Duration::from_secs(1); /// A hand-published node stats broadcast with a default-tier publisher traffic track. struct Node { - _source: moq_net::broadcast::Producer, + source: moq_net::broadcast::Producer, + epoch: Epoch, + known: bool, traffic: moq_json::snapshot::Producer, frame: TrafficFrame, } @@ -33,7 +36,10 @@ impl Node { let source = origin .create_broadcast(format!(".stats/acme/node/{name}").as_str()) .unwrap(); - source.announce(origin::Route::default()).unwrap(); + let epoch = Epoch::mint(); + source + .announce(origin::Route::default().with_epoch(epoch.clone())) + .unwrap(); let track = source .create_track(traffic_track(&Tier::default(), Role::Publisher, false), None) .unwrap(); @@ -42,7 +48,9 @@ impl Node { .map(|key| (format!("acme/room-{key:02}"), Traffic::default())) .collect(); Self { - _source: source, + source, + epoch, + known: true, traffic: moq_json::snapshot::Producer::new(track, config), frame, } @@ -68,7 +76,7 @@ async fn read_until(traffic: &mut aggregate::TrafficConsumer, want: u64) { } /// Mean wall time per merged frame with `live` nodes publishing after `departed` nodes left. -async fn measure(live: usize, departed: usize, grace: Duration) -> Duration { +async fn measure(live: usize, departed: usize, grace: Duration, mixed: bool) -> Duration { let (origin, driver) = origin::Producer::new(origin::Config::default()); let driver = tokio::spawn(moq_net::time::run(driver)); let agg = aggregate::Consumer::new( @@ -102,6 +110,14 @@ async fn measure(live: usize, departed: usize, grace: Duration) -> Duration { for frame in 0..=FRAMES { let node = &mut nodes[frame as usize % live]; let start = Instant::now(); + if mixed { + node.known = !node.known; + let mut route = origin::Route::default(); + if node.known { + route.epoch = Some(node.epoch.clone()); + } + node.source.announce(route).unwrap(); + } node.publish(1); total += 1; read_until(&mut traffic, total).await; @@ -127,13 +143,15 @@ fn main() { .build() .unwrap(); - println!("live departed grace us/frame"); + println!("live departed grace metadata us/frame"); for live in [1, 16, 128] { for departed in [0, 256, 4096] { for (label, grace) in [("1s", GRACE), ("never", Duration::MAX)] { - let elapsed = runtime.block_on(measure(live, departed, grace)); - let micros = elapsed.as_secs_f64() * 1e6; - println!("{live:>4} {departed:>8} {label:>5} {micros:>11.1}"); + for (metadata, mixed) in [("known", false), ("mixed", true)] { + let elapsed = runtime.block_on(measure(live, departed, grace, mixed)); + let micros = elapsed.as_secs_f64() * 1e6; + println!("{live:>4} {departed:>8} {label:>5} {metadata:>8} {micros:>11.1}"); + } } } } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 1b76fab5782..fc9661f997f 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -106,6 +106,14 @@ impl Default for Config { /// every node ever seen. Each route epoch is its own node: a restarted /// producer, or a group returning from idle, announces a new epoch whose /// counters add to the old epoch's kept contribution instead of regressing it. +/// Losing or recovering epoch metadata carries only the immediately outgoing +/// same-path contribution across the transition, avoiding duplicate counters for +/// one publisher reached through mixed-version links. The latest unversioned +/// snapshot replaces it; that snapshot cannot identify a reconnect or restart. +/// Metadata loss carries only the immediately outgoing same-path contribution. +/// Its known lineage remains in grace if recovery identifies a different epoch. +/// A fresh unversioned snapshot replaces the outgoing snapshot, but cannot +/// distinguish a reconnect from a restart. /// Only a route without an epoch (an older producer, or a session older than /// moq-lite 07) returning within the grace with a lower counter regresses the /// merged counter, the same reset contract a single node's own restart @@ -250,6 +258,10 @@ struct Node { /// The announced path, relative to the announce cursor: what a re-resolve /// after the reader ends requests again. path: PathOwned, + /// The publisher instance every initial and retry resolution must serve. + epoch: Option, + /// The last known publisher identity, retained while route metadata vanishes. + identity: Option, last: Option>, /// When the path unannounced, starting its grace; cleared by a reannounce. /// Only an unannounced node folds, so a still-announced node whose reader @@ -305,6 +317,8 @@ struct Merged { /// One entry per live or recently departed node broadcast, keyed by /// absolute announced path and route epoch, so each epoch is its own entry. nodes: HashMap<(PathOwned, Option), Node>, + /// The immediately outgoing identity, including while it remains in grace. + last_announced: HashMap>, /// How long a departed node waits before folding into `retired`. grace: Duration, /// The summed last frames of every node whose grace elapsed. @@ -328,6 +342,7 @@ impl Merged { json }, nodes: HashMap::new(), + last_announced: HashMap::new(), grace: config.grace, retired: BTreeMap::new(), } @@ -365,9 +380,17 @@ impl Merged { let origin = &self.origin; let grace = self.grace; let retired = &mut self.retired; - self.nodes.retain(|_, node| { + let last_announced = &mut self.last_announced; + self.nodes.retain(|(path, epoch), node| { changed |= advance(node, origin, config, name, waiter); - !node.fold_expired(retired, grace, now) + if node.fold_expired(retired, grace, now) { + if last_announced.get(path) == Some(epoch) { + last_announced.remove(path); + } + false + } else { + true + } }); // Release a churn peak's buckets: walking a map costs its capacity, so a // map that once held thousands of nodes would slow every frame. @@ -395,14 +418,34 @@ impl Merged { if parse_node_path(&self.prefix, self.depth, &absolute).is_none() { return false; } - // A newer epoch replaces the old one with an end then a start, so an - // epoch never changes under a live entry. That end stops the old entry's - // reader before a frame of the new epoch can reach it: the origin queues - // the announce change before it ends the old subscription, and announces - // drain ahead of the readers. + // Publisher replacement can race reader polling on another worker, so every + // reader pins the identity it contributes to, independent of announce order. let key = (absolute, update.route.epoch); if active { + if let Some(previous) = self.last_announced.insert(key.0.clone(), key.1.clone()) { + // An unversioned route cannot identify a restart. Carry only the + // immediately outgoing contribution across loss or recovery of metadata; + // distinct known epochs remain separate, even while several are in grace. + if previous != key.1 + && (previous.is_none() || key.1.is_none()) + && let Some(mut node) = self.nodes.remove(&(key.0.clone(), previous)) + { + node.depart(); + if key.1.is_some() && node.identity.is_some() && node.identity != key.1 { + // Recovery identifies a distinct known publisher. Keep the outgoing + // lineage in grace and re-arm the incoming identity's own entry. + node.epoch = node.identity.clone(); + self.nodes.insert((key.0.clone(), node.identity.clone()), node); + } else { + node.epoch = key.1.clone(); + if key.1.is_some() { + node.identity = key.1.clone(); + } + self.nodes.insert(key.clone(), node); + } + } + } // A route update on a node already tracked (a reprice, or a takeover // with different metadata) keeps the live reader: existing // subscriptions survive a takeover, and the reader re-resolves through @@ -418,13 +461,16 @@ impl Merged { node.fold_expired(&mut self.retired, self.grace, Instant::now()); node.departed = None; if matches!(node.reader, Reader::Ended) { - node.reader = resolve(&self.origin, &node.path); + node.reader = resolve(&self.origin, &node.path, node.epoch.clone()); } false } Entry::Vacant(entry) => { + let epoch = entry.key().1.clone(); entry.insert(Node { - reader: resolve(&self.origin, &path), + reader: resolve(&self.origin, &path, epoch.clone()), + identity: epoch.clone(), + epoch, path, last: None, departed: None, @@ -446,6 +492,7 @@ impl Merged { } else { // A gauge drops its contribution, and its entry, so a departed node // stops counting and its path is not retained. + self.last_announced.remove(&key.0); self.nodes.remove(&key).is_some_and(|old| old.last.is_some()) } } @@ -494,7 +541,7 @@ fn advance( // means nothing serves the path (the retraction that empties the // table also unannounces this node). Poll::Ready(Err(moq_net::Error::Unroutable)) if *queued => { - node.reader = resolve(origin, &node.path); + node.reader = resolve(origin, &node.path, node.epoch.clone()); } Poll::Ready(Err(err)) => { tracing::debug!(?err, name, "stats: node broadcast unresolvable"); @@ -539,7 +586,7 @@ fn advance( return changed; } rearmed = true; - node.reader = resolve(origin, &node.path); + node.reader = resolve(origin, &node.path, node.epoch.clone()); } Poll::Pending => return changed, }, @@ -549,8 +596,8 @@ fn advance( } /// Start resolving `path` (relative to the announce cursor) into a broadcast. -fn resolve(origin: &origin::Consumer, path: &PathOwned) -> Reader { - let pending = origin.request_broadcast(path); +fn resolve(origin: &origin::Consumer, path: &PathOwned, epoch: Option) -> Reader { + let pending = origin.request_broadcast(path, epoch); let queued = pending.is_queued(); Reader::Resolving { pending, queued } } @@ -624,7 +671,7 @@ mod tests { let (_, active) = next_update(&mut announced).await.expect("announce"); assert!(active); - let consumer = egress.request_broadcast(path).await.expect("resolve"); + let consumer = egress.request_broadcast(path, None).await.expect("resolve"); let mut sub = consumer.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = track.append_group().unwrap(); @@ -850,6 +897,171 @@ mod tests { assert_eq!(traffic.inner.nodes.len(), 3, "one entry per epoch"); } + #[test] + fn outgoing_epoch_metadata_migrates_without_choosing_a_grace_entry() { + let origin = produce_origin(); + let path = PathOwned::from(".stats/acme/node/a"); + let a: Epoch = "01900000-0000-7000-8000-000000000001".parse().unwrap(); + let b: Epoch = "01900000-0000-7000-8000-000000000002".parse().unwrap(); + let c: Epoch = "01900000-0000-7000-8000-000000000003".parse().unwrap(); + let d: Epoch = "01900000-0000-7000-8000-000000000004".parse().unwrap(); + let mut merged = Merged::::new(origin.consume(), &Config::new().with_depth(1), "traffic".into()); + let update = |epoch: Option| moq_net::announce::Announce { + prefix: path.clone(), + captures: None, + route: { + let mut route = origin::Route::default(); + route.epoch = epoch; + route + }, + }; + for (epoch, bytes) in [(a.clone(), 100), (b.clone(), 30), (c.clone(), 7)] { + merged.apply_announce(update(Some(epoch.clone())), true); + merged.nodes.get_mut(&(path.clone(), Some(epoch.clone()))).unwrap().last = + Some(BTreeMap::from([("room".into(), { + let mut traffic = Traffic::default(); + traffic.bytes = bytes; + traffic + })])); + merged.apply_announce(update(Some(epoch)), false); + } + assert_eq!(merged.merged()["room"].bytes, 137); + merged.apply_announce(update(None), true); + assert_eq!( + merged.nodes.len(), + 3, + "only the immediately outgoing C becomes unversioned" + ); + assert!(merged.nodes.contains_key(&(path.clone(), Some(a.clone())))); + assert!(merged.nodes.contains_key(&(path.clone(), Some(b.clone())))); + assert!(!merged.nodes.contains_key(&(path.clone(), Some(c.clone())))); + assert_eq!( + merged.merged()["room"].bytes, + 137, + "metadata loss preserves the last snapshot" + ); + + // A fresh unversioned snapshot replaces only the outgoing contribution. + merged.nodes.get_mut(&(path.clone(), None)).unwrap().last = Some(BTreeMap::from([("room".into(), { + let mut traffic = Traffic::default(); + traffic.bytes = 8; + traffic + })])); + merged.apply_announce(update(None), false); + merged.apply_announce(update(Some(a.clone())), true); + assert_eq!( + merged.nodes.len(), + 3, + "recovering A retains both outgoing C and grace B" + ); + assert!(merged.nodes.contains_key(&(path.clone(), Some(c)))); + assert_eq!( + merged.nodes[&(path.clone(), Some(a.clone()))].last.as_ref().unwrap()["room"].bytes, + 100 + ); + assert_eq!( + merged.merged()["room"].bytes, + 138, + "retained A is not overwritten by outgoing C" + ); + + merged.apply_announce(update(Some(a.clone())), false); + merged.apply_announce(update(None), true); + merged.apply_announce(update(None), false); + merged.apply_announce(update(Some(d.clone())), true); + assert_eq!( + merged.nodes.len(), + 4, + "a distinct recovered epoch gets its own contribution" + ); + assert!(merged.nodes.contains_key(&(path.clone(), Some(a)))); + assert!(merged.nodes.contains_key(&(path.clone(), Some(d)))); + assert!(!merged.nodes.contains_key(&(path, None))); + assert_eq!( + merged.merged()["room"].bytes, + 138, + "known grace entries remain counted once each" + ); + } + + /// A replacement on another worker can land after announces drain and before the + /// old reader retries. Its cached frame must never become the old epoch's total. + #[test] + fn replacement_between_announce_drain_and_reader_retry() { + use std::sync::Barrier; + + let now = std::time::Instant::now(); + let (origin, mut driver) = origin::Producer::new(origin::Config::default()); + let epoch_a: Epoch = "01900000-0000-7000-8000-000000000001".parse().unwrap(); + let epoch_b: Epoch = "01900000-0000-7000-8000-000000000002".parse().unwrap(); + let mut old = NodeBroadcast::new(&origin, "acme", "a"); + old.source + .announce(origin::Route::default().with_epoch(epoch_a.clone())) + .unwrap(); + old.publish("acme/room", 100); + let name = traffic_track(&Tier::default(), Role::Publisher, false); + let mut merged = Merged::::new(origin.consume(), &Config::new().with_depth(1), name.clone()); + let initial = futures::executor::block_on(kio::wait(|waiter| { + driver.poll(now, waiter).unwrap(); + merged.poll_next(waiter) + })) + .unwrap() + .unwrap(); + assert_eq!(initial["acme/room"].bytes, 100); + assert!(merged.announce.poll_next(&Waiter::noop()).is_pending()); + + let ready = Barrier::new(2); + let release = Barrier::new(2); + std::thread::scope(|scope| { + let worker = scope.spawn(|| { + let mut new = NodeBroadcast::new(&origin, "acme", "a"); + new.source + .announce(origin::Route::default().with_epoch(epoch_b.clone())) + .unwrap(); + new.publish("acme/room", 30); + let pending = origin.consume().request_broadcast(".stats/acme/node/a", None); + let front = futures::executor::block_on(kio::wait(|waiter| { + driver.poll(now, waiter).unwrap(); + pending.poll_ok(waiter) + })) + .unwrap(); + let pending = front.track(&name).unwrap().subscribe(None); + let subscriber = futures::executor::block_on(kio::wait(|waiter| { + driver.poll(now, waiter).unwrap(); + pending.poll_ok(waiter) + })) + .unwrap(); + let mut warm = moq_json::snapshot::Consumer::::new(subscriber, Default::default()); + let frame = futures::executor::block_on(kio::wait(|waiter| { + driver.poll(now, waiter).unwrap(); + warm.poll_next(waiter) + })) + .unwrap() + .unwrap(); + assert_eq!(frame["acme/room"].bytes, 30); + ready.wait(); + release.wait(); + (new, front, warm, driver) + }); + ready.wait(); + let key = (PathOwned::from(".stats/acme/node/a"), Some(epoch_a)); + let node = merged.nodes.get_mut(&key).unwrap(); + advance(node, &merged.origin, &merged.config, &name, &Waiter::noop()); + let old_bytes = node.last.as_ref().unwrap()["acme/room"].bytes; + // Release before asserting, so a failed regression also joins its worker. + release.wait(); + let (_new, _front, _warm, mut driver) = worker.join().unwrap(); + assert_eq!(old_bytes, 100, "the replacement frame overwrote the old epoch"); + let combined = futures::executor::block_on(kio::wait(|waiter| { + driver.poll(now, waiter).unwrap(); + merged.poll_next(waiter) + })) + .unwrap() + .unwrap(); + assert_eq!(combined["acme/room"].bytes, 130); + }); + } + #[tokio::test(start_paused = true)] async fn unversioned_restart_regresses_the_total() { // A node whose route carries no epoch (an older producer, or a session diff --git a/rs/moq-stats/src/consume.rs b/rs/moq-stats/src/consume.rs index baf0473b53f..3d01b86c872 100644 --- a/rs/moq-stats/src/consume.rs +++ b/rs/moq-stats/src/consume.rs @@ -177,7 +177,7 @@ mod tests { let (_, active) = next_update(&mut announced).await.expect("announce"); assert!(active); - let consumer = egress.request_broadcast(path).await.expect("resolve"); + let consumer = egress.request_broadcast(path, None).await.expect("resolve"); let sub = consumer.track("video").unwrap().subscribe(None).await.unwrap(); Feed { @@ -196,7 +196,7 @@ mod tests { assert!(active); origin .consume() - .request_broadcast(moq_net::Path::new(update.prefix.as_str())) + .request_broadcast(moq_net::Path::new(update.prefix.as_str()), None) .await .expect("resolve") } diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 9005ba725e2..72ec05fe2b5 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1230,7 +1230,7 @@ mod tests { let (_, active) = next_update(&mut announced).await.expect("announce"); assert!(active); - let consumer = egress.request_broadcast(path).await.expect("resolve"); + let consumer = egress.request_broadcast(path, None).await.expect("resolve"); let sub = if subscribe { let mut sub = consumer @@ -1273,7 +1273,7 @@ mod tests { assert!(active); let broadcast = origin .consume() - .request_broadcast(moq_net::Path::new(update.prefix.as_str())) + .request_broadcast(moq_net::Path::new(update.prefix.as_str()), None) .await .expect("resolve"); (update.prefix.as_str().to_string(), broadcast) @@ -1416,7 +1416,7 @@ mod tests { }; let stats = origin .consume() - .request_broadcast(path.as_str()) + .request_broadcast(path.as_str(), None) .await .expect("resolve"); let mut old = subscribe(&stats, "publisher.json").await; @@ -1440,7 +1440,7 @@ mod tests { let stats = origin .consume() - .request_broadcast(path.as_str()) + .request_broadcast(path.as_str(), None) .await .expect("resolve"); assert_eq!(read_last_frame(&stats, "publisher.json").await["foo/bar"].bytes, 30); @@ -1569,7 +1569,7 @@ mod tests { .with_stats(registry.tier(Tier::default()).session("viewer")); async fn view(egress: &origin::Consumer, video: &mut track::Producer, size: usize) { - let broadcast = egress.request_broadcast("foo/bar").await.expect("resolve"); + let broadcast = egress.request_broadcast("foo/bar", None).await.expect("resolve"); let mut sub = broadcast .track("video") .expect("track") @@ -1662,8 +1662,8 @@ mod tests { panic!("expected both groups to announce, got {started:?}"); }; assert_eq!((acme_path.as_str(), feed_path.as_str()), (ACME, FEED)); - let acme = origin.consume().request_broadcast(ACME).await.expect("resolve"); - let sessions = origin.consume().request_broadcast(FEED).await.expect("resolve"); + let acme = origin.consume().request_broadcast(ACME, None).await.expect("resolve"); + let sessions = origin.consume().request_broadcast(FEED, None).await.expect("resolve"); assert_eq!(read_last_frame(&acme, "publisher.json").await["acme/live"].bytes, 100); // The viewer leaves: both groups empty but stay announced, reading zero. @@ -1725,7 +1725,7 @@ mod tests { new_acme > acme_epoch && new_feed > feed_epoch, "a returning group mints a new epoch" ); - let acme = origin.consume().request_broadcast(ACME).await.expect("resolve"); + let acme = origin.consume().request_broadcast(ACME, None).await.expect("resolve"); assert_eq!(read_last_frame(&acme, "publisher.json").await["acme/live"].bytes, 25); let mut track = subscribe(&acme, "publisher.json").await; let group = track.next_group().await.expect("ok").expect("group"); diff --git a/rs/moq-tokio/examples/clock.rs b/rs/moq-tokio/examples/clock.rs index f5e73100ee8..d86f3782730 100644 --- a/rs/moq-tokio/examples/clock.rs +++ b/rs/moq-tokio/examples/clock.rs @@ -112,7 +112,7 @@ async fn main() -> anyhow::Result<()> { Some(event) = announced.next() => match event { announce::Event::Start(update) | announce::Event::Update(update) => { tracing::info!(broadcast = %update.prefix, "broadcast is online, subscribing to track"); - let broadcast = consumer.request_broadcast(&update.prefix).await?; + let broadcast = consumer.request_broadcast(&update.prefix, None).await?; let track = broadcast .track(&track)?.subscribe(None).await?; clock = Some(Subscriber::new(track)); diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index db4a86024f1..ca4be5265ad 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -1703,7 +1703,7 @@ mod tests { }; assert_eq!(update.prefix.as_str(), "test"); let broadcast = sub_consumer - .request_broadcast("test") + .request_broadcast("test", None) .await .expect("broadcast resolves"); let mut track = broadcast diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index e2dca2455cf..ae955a6c24d 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -2258,7 +2258,7 @@ mod tests { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active); - let broadcast = consumer.request_broadcast("test").await.expect("resolve"); + let broadcast = consumer.request_broadcast("test", None).await.expect("resolve"); let mut track = broadcast .track("video") diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index b6aa80178d5..12479d771e2 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -182,7 +182,7 @@ async fn connect_test(config: ConnectTest<'_>) { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -703,7 +703,7 @@ async fn iroh_connect_test(version: Option<&str>) { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -870,7 +870,7 @@ async fn noq_client_close_drains_finished_track() { .await .expect("announce timed out") .expect("origin closed"); - let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -989,7 +989,7 @@ async fn noq_client_close_drains_migrated_predecessor() { .await .expect("announce timed out") .expect("origin closed"); - let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -1124,7 +1124,7 @@ async fn noq_client_close_keeps_predecessor_handover() { .await .expect("announce timed out") .expect("origin closed"); - let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 9141fca8b18..c7e2e24c845 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -90,7 +90,7 @@ async fn broadcast_test(scheme: &str, client_version: Option<&str>, server_versi assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -203,7 +203,7 @@ async fn lite05_timestamp_roundtrip(scheme: &str) { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -326,7 +326,7 @@ async fn lite05_fetch_roundtrip(scheme: &str) { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -436,7 +436,7 @@ async fn transport_fetch_roundtrip(version: &str, served: bool) { .await .expect("announce timed out") .expect("origin closed"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -593,7 +593,7 @@ async fn lite05_fetch_during_subscribe(scheme: &str) { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "test"); assert!(active, "expected announce, got retraction"); - let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let bc = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timed out") .expect("announced broadcast resolves"); @@ -705,7 +705,7 @@ async fn broadcast_moq_lite_05_default_timescale() { assert!(active, "expected announce"); let bc = tokio::time::timeout( TIMEOUT, - sub_consumer.request_broadcast(moq_net::Path::new(update.prefix.as_str())), + sub_consumer.request_broadcast(moq_net::Path::new(update.prefix.as_str()), None), ) .await .expect("request timed out") @@ -803,7 +803,7 @@ async fn broadcast_moq_transport_20_current_group_join() { assert!(announced_active, "expected an announce"); let remote = tokio::time::timeout( TIMEOUT, - sub_consumer.request_broadcast(moq_net::Path::new(announced.prefix.as_str())), + sub_consumer.request_broadcast(moq_net::Path::new(announced.prefix.as_str()), None), ) .await .expect("request timed out") @@ -1121,7 +1121,7 @@ async fn broadcast_route_migration() { let subscription = moq_net::track::Subscription::default() .with_start(moq_net::track::Position::group(1)) .with_max_age(Duration::from_secs(10)); - let broadcast = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("routed broadcast resolves"); @@ -1243,7 +1243,7 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str) { .expect("connect failed"); assert!(next_announce(&mut announcements).await.1); - let remote = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let remote = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("broadcast resolves"); @@ -1363,7 +1363,7 @@ async fn rejoin_replays_a_current_warm_cache(version: &str, open: bool) { .expect("connect failed"); assert!(next_announce(&mut announcements).await.1); - let remote = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let remote = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("broadcast resolves"); @@ -1497,7 +1497,7 @@ async fn route_reannounce_test(version: Option<&str>) { assert!(active, "expected announce"); let initial = update.route; - let broadcast = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + let broadcast = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test", None)) .await .expect("request timeout") .expect("routed broadcast resolves"); @@ -1808,7 +1808,7 @@ async fn max_age_test(version: &str, published: Option) -> Option) -> anyhow::R .ok_or_else(|| anyhow::anyhow!("announcements closed"))?, moq_net::announce::Event::Start(_) ) {} - let front = consumer.request_broadcast("age").await?; + let front = consumer.request_broadcast("age", None).await?; for (i, age) in AGES.into_iter().enumerate() { let track = front.track(&i.to_string())?.subscribe(None).await?; anyhow::ensure!( diff --git a/rs/moq-tokio/tests/subscription_end_integrity.rs b/rs/moq-tokio/tests/subscription_end_integrity.rs index 805be83e339..61978f7a73e 100644 --- a/rs/moq-tokio/tests/subscription_end_integrity.rs +++ b/rs/moq-tokio/tests/subscription_end_integrity.rs @@ -80,7 +80,10 @@ async fn round(drop_session: bool) -> (usize, Option) { .expect("client connect"); consumer.routed("test").await.expect("routed"); - let remote = consumer.request_broadcast("test").await.expect("broadcast resolves"); + let remote = consumer + .request_broadcast("test", None) + .await + .expect("broadcast resolves"); let mut sub = remote.track("video").unwrap().subscribe(None).await.expect("subscribe"); let mut got = 0; diff --git a/rs/moq-tokio/tests/unannounce.rs b/rs/moq-tokio/tests/unannounce.rs index 9e27bae341d..f06bfe88ef3 100644 --- a/rs/moq-tokio/tests/unannounce.rs +++ b/rs/moq-tokio/tests/unannounce.rs @@ -12,7 +12,7 @@ async fn release(unannounce: bool, settle: bool) { let origin = moq_tokio::origin::spawn(); let broadcast = origin.publish("a/b", moq_net::origin::Route::default()).unwrap(); let track = broadcast.create_track("t", None).unwrap(); - let consumer = origin.consume().request_broadcast("a/b").await.unwrap(); + let consumer = origin.consume().request_broadcast("a/b", None).await.unwrap(); let sub = consumer.track("t").unwrap().subscribe(None).await.unwrap(); track.demand().used().await.unwrap(); diff --git a/rs/moq-transcode/examples/transcode.rs b/rs/moq-transcode/examples/transcode.rs index d5431d42e11..7614fd98b09 100644 --- a/rs/moq-transcode/examples/transcode.rs +++ b/rs/moq-transcode/examples/transcode.rs @@ -70,7 +70,7 @@ async fn main() -> anyhow::Result<()> { // Resolve it for real; the session subscribes upstream on demand. let source = consumer - .request_broadcast(&source_path) + .request_broadcast(&source_path, None) .await .context("source broadcast unavailable")?; diff --git a/rs/moq-uring/benches/session_lite.rs b/rs/moq-uring/benches/session_lite.rs index 50efb9797ca..9d7c248e3d5 100644 --- a/rs/moq-uring/benches/session_lite.rs +++ b/rs/moq-uring/benches/session_lite.rs @@ -153,7 +153,10 @@ mod linux { let bc = { let consumer = sub_origin.consume(); consumer.routed("bench").await.expect("broadcast announced"); - consumer.request_broadcast("bench").await.expect("broadcast resolves") + consumer + .request_broadcast("bench", None) + .await + .expect("broadcast resolves") }; let sub = bc .track("data") diff --git a/rs/moq-uring/tests/session.rs b/rs/moq-uring/tests/session.rs index 50fc783670a..2f77771d992 100644 --- a/rs/moq-uring/tests/session.rs +++ b/rs/moq-uring/tests/session.rs @@ -116,7 +116,10 @@ fn lite_session_over_the_worker() { let bc = { let consumer = sub.consume(); consumer.routed("test").await.expect("broadcast announced"); - consumer.request_broadcast("test").await.expect("broadcast resolves") + consumer + .request_broadcast("test", None) + .await + .expect("broadcast resolves") }; let mut track = bc .track("data") @@ -238,7 +241,10 @@ fn two_lite_sessions_share_the_server_socket() { let bc = { let consumer = sub.consume(); consumer.routed("test").await.expect("broadcast announced"); - consumer.request_broadcast("test").await.expect("broadcast resolves") + consumer + .request_broadcast("test", None) + .await + .expect("broadcast resolves") }; let mut track = bc .track("data") diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index c0a6e96bb28..c9d7c087096 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -247,7 +247,10 @@ fn lite_session_over_webtransport() { let bc = { let consumer = sub_origin.consume(); consumer.routed("test").await.expect("broadcast announced"); - consumer.request_broadcast("test").await.expect("broadcast resolves") + consumer + .request_broadcast("test", None) + .await + .expect("broadcast resolves") }; let mut track = bc .track("data") diff --git a/rs/moq-wasm/src/lib.rs b/rs/moq-wasm/src/lib.rs index c4cabe2fdfa..cf6e0c77d4f 100644 --- a/rs/moq-wasm/src/lib.rs +++ b/rs/moq-wasm/src/lib.rs @@ -159,7 +159,7 @@ impl Session { future_to_promise(async move { let request = pin!(async { consumer.routed(path.as_str()).await?; - consumer.request_broadcast(path.as_str()).await.ok() + consumer.request_broadcast(path.as_str(), None).await.ok() }); // The origin outlives the session, so its wait alone never ends on a close. let closed = pin!(session.closed()); From 2dd94f0a396eae0fbe09c64f357095727c4c88cb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 00:45:14 -0700 Subject: [PATCH 6/9] test(stats): expose unversioned retry crossing a recovered epoch This regression intentionally fails until the maintainer chooses how actual resolved epoch metadata or conservative fallback should prevent the remaining cross-worker duplicate contribution. Co-Authored-By: GPT-6 --- rs/moq-stats/src/aggregate.rs | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index fc9661f997f..df710185554 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -984,10 +984,20 @@ mod tests { ); } - /// A replacement on another worker can land after announces drain and before the - /// old reader retries. Its cached frame must never become the old epoch's total. + /// A known replacement must not overwrite the outgoing known epoch's counters. #[test] fn replacement_between_announce_drain_and_reader_retry() { + replacement_while_reader_retries(false); + } + + /// Missing metadata must not let the outgoing lineage consume a recovered known epoch. + #[test] + fn unknown_replacement_between_announce_drain_and_reader_retry() { + replacement_while_reader_retries(true); + } + + /// Arrange replacement on another worker after announcements drain and before retrying. + fn replacement_while_reader_retries(drop_epoch: bool) { use std::sync::Barrier; let now = std::time::Instant::now(); @@ -1008,6 +1018,20 @@ mod tests { .unwrap() .unwrap(); assert_eq!(initial["acme/room"].bytes, 100); + if drop_epoch { + old.source.announce(origin::Route::default()).unwrap(); + let unversioned = futures::executor::block_on(kio::wait(|waiter| { + driver.poll(now, waiter).unwrap(); + merged.poll_next(waiter) + })) + .unwrap() + .unwrap(); + assert_eq!(unversioned["acme/room"].bytes, 100); + assert!(matches!( + merged.nodes[&(PathOwned::from(".stats/acme/node/a"), None)].reader, + Reader::Active(_) + )); + } assert!(merged.announce.poll_next(&Waiter::noop()).is_pending()); let ready = Barrier::new(2); @@ -1044,14 +1068,13 @@ mod tests { (new, front, warm, driver) }); ready.wait(); - let key = (PathOwned::from(".stats/acme/node/a"), Some(epoch_a)); + let key = (PathOwned::from(".stats/acme/node/a"), (!drop_epoch).then_some(epoch_a)); let node = merged.nodes.get_mut(&key).unwrap(); advance(node, &merged.origin, &merged.config, &name, &Waiter::noop()); let old_bytes = node.last.as_ref().unwrap()["acme/room"].bytes; // Release before asserting, so a failed regression also joins its worker. release.wait(); let (_new, _front, _warm, mut driver) = worker.join().unwrap(); - assert_eq!(old_bytes, 100, "the replacement frame overwrote the old epoch"); let combined = futures::executor::block_on(kio::wait(|waiter| { driver.poll(now, waiter).unwrap(); merged.poll_next(waiter) @@ -1059,6 +1082,7 @@ mod tests { .unwrap() .unwrap(); assert_eq!(combined["acme/room"].bytes, 130); + assert_eq!(old_bytes, 100, "the replacement frame overwrote the old epoch"); }); } From 5eb8c19f6c2c88ce5e32d1faa734251cbe7be870 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 14:42:05 -0700 Subject: [PATCH 7/9] fix(stats)!: name the resolved epoch on a requested broadcast An unpinned stats reader retrying on a known lineage could resolve a replacement publisher before its announcement drained, and attribute the newcomer's counters to the outgoing epoch. The origin now stamps the epoch of the route it resolved through on the handed-out broadcast, atomically with the result: Rust `broadcast::Consumer::epoch()` and JS readonly `broadcast.Consumer.epoch`. The aggregator refuses a known replacement on an unpinned retry, leaving it to that epoch's own entry, so the strict unknown-to-known regression passes. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 2 -- doc/concept/stats.md | 3 +++ doc/lib/js/net.md | 2 +- doc/lib/rs/moq-net.md | 2 +- js/net/src/broadcast.ts | 20 ++++++++++++++++- js/net/src/internal.ts | 6 +++++ js/net/src/origin.test.ts | 20 ++++++++++++++--- js/net/src/origin.ts | 11 +++++---- rs/moq-net/src/model/broadcast.rs | 22 ++++++++++++++++++ rs/moq-net/src/model/origin.rs | 37 ++++++++++++++++++++++++++----- rs/moq-stats/src/aggregate.rs | 34 ++++++++++++++++++---------- 11 files changed, 129 insertions(+), 30 deletions(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index e00f4e74a8a..ff5e6d433c1 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -177,8 +177,6 @@ link (no epoch) and a moq-lite 07 link (with one), the epoch-carrying route supersedes the other each time it appears, so a flapping moq-lite 07 link cuts the viewers resolved through the older one. -Stats aggregation retains each known publisher epoch separately. When epoch metadata disappears or returns, it carries only the immediately outgoing same-path contribution across that transition, retaining the last snapshot until a fresh snapshot replaces it. If recovery names a different known epoch, the outgoing known lineage stays in grace and the incoming epoch resumes its own contribution. Older unversioned routes cannot distinguish a reconnect from a restart, so that fresh snapshot may regress counters; other known epochs retained during grace are never chosen as substitutes. - ### Hidden broadcasts A path segment starting with `.` hides a route from discovery, the way a diff --git a/doc/concept/stats.md b/doc/concept/stats.md index a68f5e576c8..d37d6844b51 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -39,6 +39,9 @@ broadcast at the same path: it replaces the old one, and no relay serves it groups cached under the old epoch. A reader treats each epoch as its own set of counters. The epoch rides moq-lite 07 announcements; over older versions and moq-transport a reader sees the same change as an end and a start at the path. +An aggregating reader pins each node's subscription to its epoch, and when one +publisher is reached through both kinds of link (with and without an epoch), it +carries only that path's outgoing counters across the change. At depth 0 the broadcast stays announced for the producer's life. At depth 1 or more, a group's broadcast is announced while that group has entries, and diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index 265c5668b81..57a2f6f7a34 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -49,7 +49,7 @@ for (;;) { ``` - **Origins** hold the broadcasts, not the connection: closing a session unannounces them but leaves them created for the next one. `origin.request(path)` resolves an announced local broadcast with no round trip, so a page that watches what it publishes reads its own copy, unless a cheaper route announces the same path. A refusal from the most specific route ends the request: `request.closed` settles with the handler's error and no broader route is asked. Create, populate, then `announce()` for an exact path; use `dynamic(prefix, route)` when the set of paths is not known: an exact-path subscribe before the tracks exist is refused, and nobody, local or remote, can see or reach a broadcast until it announces. -- **Epoch pinning**: `consumer.request(path, { epoch })` pins an announced publisher instance. The request stays unresolved and reports `unroutable` when the winning route names a different or missing epoch, including while an asynchronous handler answers. Omit `epoch` for unpinned lookup. +- **Epoch pinning**: `consumer.request(path, { epoch })` pins an announced publisher instance. The request stays unresolved and reports `unroutable` when the winning route names a different or missing epoch, including while an asynchronous handler answers. Omit `epoch` for unpinned lookup. The resolved broadcast's `epoch` names the route it actually resolved through, pinned or not. - **Connections** race WebTransport against WebSocket. `new Connection({ url })` pools one connection per relay URL and reconnects with backoff, which the elements use. Supplying WebTransport/WebSocket options, discovery, delay, `goaway`, or a caller-owned origin selects a private loop; explicit `share: true` refuses those options. On a relay's GOAWAY the connection dials the replacement at once while the old session keeps serving its groups until it closes or the `goaway.handover` cap (default 10s, lowered to the relay's deadline) passes; the handle and its origin stay the same. An empty GOAWAY redials the same URL through a fresh DNS resolve. A redirect is followed onto the same host by default (`goaway.redirect: "follow"` lets the relay pick the host, `"ignore"` never moves), and moves the pooled entry to the new URL; a malformed or refused one, including a host change under a pinned certificate, ends the connection with `Error.RefusedRedirect` instead of reconnecting. `closed` settles when the handle is released (`null` on a clean close); the failure that stopped retrying the current URL is `error`, and a new URL recovers the same handle. A connection owns one send-rate sampler and one `Bandwidth.Allocator`; publishers reserve against it so their encoder targets sum to the estimate instead of each matching it. - **Bandwidth** (`Bandwidth.Allocator`) divides the connection's send-rate estimate by track priority, max-min fair within a tier. An idle track claims nothing. The receive side is untouched. - **Discovery** by any pattern scope (`origin.announced(scope)`, such as `room/*/chat`; default everything). A route event's `kind` says whether the route started, updated, or ended; its `prefix` is the covered prefix relative to the origin, and `captures` reports what the scope's wildcards matched when the prefix pins them. The consumer is an async iterable. `origin.broadcasts(scope)` is a live `Getter>` of the same covered prefixes for UIs that need the current set. A borrowed `Connection.origin` also exposes `dynamic(prefix, route)` for serving paths on demand. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index c23f41ab833..b5d38df653f 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -17,7 +17,7 @@ above ([hang](/lib/rs/hang)); relays and CDNs implement only this. - **Origins** scope what a session can see, and merge duplicate subscriptions so a broadcast is pulled upstream once no matter how many local readers. - **Broadcasts** are created unannounced and invisible to everyone, then announced as an exact route, or served below a prefix with `dynamic`. A consumer of the same origin sees exactly what a peer sees. Discovery accepts pattern unions; events carry the advertised prefix and captures for a complete match. -- **Resolution** uses `consumer.request_broadcast(path, epoch)` with an owned `Epoch` to pin a publisher instance, or `None` for unpinned lookup. A pinned request refuses a different or missing epoch at request time and asynchronous resolution. +- **Resolution** uses `consumer.request_broadcast(path, epoch)` with an owned `Epoch` to pin a publisher instance, or `None` for unpinned lookup. A pinned request refuses a different or missing epoch at request time and asynchronous resolution. The resolved broadcast's `epoch()` names the route it actually resolved through, pinned or not. - **Epochs** identify publisher instances with a canonical UUIDv7 carried on each route (`Route::epoch`): the newest wins a path, and only routes with the same epoch resume a subscription; see [publisher epochs](/concept/moq-lite#publisher-epochs). - **Patterns** (`Pattern`, `Patterns`) are re-exported from [`moq-pattern`](https://docs.rs/moq-pattern). Literal `Path` stays a coordinate. - **Tracks** carry groups with a priority, an optional publisher retention window (`Info::max_age`), and a timescale. Subscribers set their own priority and max delay and can change them live. diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 369c481b060..9946e8132ac 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -4,6 +4,7 @@ * @module */ import { type Dispose, type GetPromise, type Getter, Once, Signal } from "@moq/signals"; +import type * as Epoch from "./epoch.ts"; import { NotFound } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; import { Route } from "./hop.ts"; @@ -415,6 +416,7 @@ export class Producer { interface Shared { state: BroadcastState; path: Path.Valid; + epoch?: Epoch.Valid; } // Constructs a Consumer from within this module without exposing a public constructor @@ -432,6 +434,7 @@ let makeConsumer: (shared: Shared) => Consumer; export class Consumer { #state: BroadcastState; #path: Path.Valid; + #epoch?: Epoch.Valid; // Guards against a double close() on this handle over-decrementing the consumer count. #closed = false; @@ -439,6 +442,7 @@ export class Consumer { protected constructor(shared?: never); protected constructor(shared?: Shared) { this.#path = shared?.path ?? Path.empty(); + this.#epoch = shared?.epoch; if (shared) { this.#state = shared.state; } else { @@ -461,6 +465,9 @@ export class Consumer { if (target instanceof Consumer) target.#path = path; else stampProducer(target, path); }; + hooks.stampEpoch = (target, epoch) => { + target.#epoch = epoch; + }; } /** @@ -475,6 +482,17 @@ export class Consumer { return this.#path; } + /** + * The publisher epoch of the route an origin resolved this handle through, if it has one. + * + * An origin stamps it with each request's result, so it names the publisher + * instance actually serving this handle even when the request named none. `undefined` for a + * route without an epoch or a handle not handed out by an origin. + */ + get epoch(): Epoch.Valid | undefined { + return this.#epoch; + } + /** * Settles with `null` once the broadcast closes; a broadcast end carries no cause. * Peek it synchronously (`undefined` while open), observe it reactively, or `await` it. @@ -501,7 +519,7 @@ export class Consumer { // Hand this consumer's backing state and path to a clone. Opaque (`never`) so the state type // stays unexported; a subclass passes it straight back into its own `super(...)`. protected shareState(): never { - return { state: this.#state, path: this.#path } satisfies Shared as never; + return { state: this.#state, path: this.#path, epoch: this.#epoch } satisfies Shared as never; } /** Get a lazy handle for a track on this broadcast. Repeat subscriptions dedupe onto one upstream subscription. */ diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 153fe38a997..03ef99e9f2a 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -7,6 +7,7 @@ */ import type { Dispose, Getter } from "@moq/signals"; import type { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; +import type * as Epoch from "./epoch.ts"; import type { Frame, Consumer as GroupConsumer } from "./group.ts"; import type { Route } from "./hop.ts"; import * as Path from "./path.ts"; @@ -181,6 +182,8 @@ export const hooks: { ) => void; /** Name a broadcast handle by the path an origin created or resolved it at. */ stampPath: (target: BroadcastProducer | BroadcastConsumer, path: Path.Valid) => void; + /** Name the epoch of the route an origin resolved a broadcast handle through. */ + stampEpoch: (target: BroadcastConsumer, epoch: Epoch.Valid | undefined) => void; } = { makeRequest: () => { throw new Error("track.ts not loaded"); @@ -224,6 +227,9 @@ export const hooks: { stampPath: () => { throw new Error("broadcast.ts not loaded"); }, + stampEpoch: () => { + throw new Error("broadcast.ts not loaded"); + }, }; /** diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index d7acecd348a..ad092b65453 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -1910,13 +1910,15 @@ test("request pins its epoch while an unpinned sibling follows the newest publis const reader = origin.consume(); const pinned = reader.request(path, { epoch: EPOCH }); const unpinned = reader.request(path); - expect(pinned.active.peek()).toBeDefined(); + expect(pinned.active.peek()?.epoch).toBe(EPOCH); + expect(unpinned.active.peek()?.epoch).toBe(EPOCH); const newerEpoch = Epoch.parse("01900000-0000-7000-8000-000000000002"); const second = origin.createBroadcast(path); second.announce({ epoch: newerEpoch }); expect(pinned.active.peek()).toBeUndefined(); expect(pinned.unroutable.peek()).toBe(true); - expect(unpinned.active.peek()).toBeDefined(); + // The unpinned handle names the publisher it actually resolved. + expect(unpinned.active.peek()?.epoch).toBe(newerEpoch); const stale = reader.request(path, { epoch: EPOCH }); expect(stale.active.peek()).toBeUndefined(); expect(stale.unroutable.peek()).toBe(true); @@ -1943,7 +1945,7 @@ test("request never accepts an old handler after its epoch changes asynchronousl expect(current.active.peek()).toBeUndefined(); const next = (await requests.next()).value; next?.accept(new BroadcastProducer().consume()); - expect(current.active.peek()).toBeDefined(); + expect(current.active.peek()?.epoch).toBe(epoch); expect(pending.active.peek()).toBeUndefined(); await requests.return?.(); pending.close(); @@ -1951,3 +1953,15 @@ test("request never accepts an old handler after its epoch changes asynchronousl dynamic.close(); origin.close(); }); + +test("a route without an epoch resolves a handle without one", () => { + const origin = new Producer(); + const path = Path.from("room"); + const broadcast = origin.createBroadcast(path); + broadcast.announce(); + const request = origin.consume().request(path); + expect(request.active.peek()).toBeDefined(); + expect(request.active.peek()?.epoch).toBeUndefined(); + request.close(); + origin.close(); +}); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 7f3b6472adb..e6ff63e54ca 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -1323,16 +1323,19 @@ export class Consumer { // we already made, and only a real swap clones a new one (cloning before closing the // old, so a broadcast that both routes share never briefly loses its last handle). let released = false; - let source: broadcast.Consumer | undefined; + let source: Resolution | undefined; let handle: broadcast.Consumer | undefined; const own = (resolution: Resolution | undefined): broadcast.Consumer | undefined => { if (released) return undefined; const front = epoch === undefined || resolution?.epoch === epoch ? resolution?.front : undefined; - if (front !== source) { + if (front !== source?.front || resolution?.epoch !== source?.epoch) { const previous = handle; - source = front; + source = front ? { front, epoch: resolution?.epoch } : undefined; handle = front?.clone(); - if (handle) hooks.stampPath(handle, relative); + if (handle) { + hooks.stampPath(handle, relative); + hooks.stampEpoch(handle, resolution?.epoch); + } previous?.close(); } return handle; diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index d4a97e9734a..11d8fbfd3c9 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -352,6 +352,7 @@ impl Producer { alive: self.alive.token.consume(), state: self.state.clone(), stats: stats::Scope::default(), + epoch: None, } } @@ -573,6 +574,7 @@ impl Dynamic { alive: self.alive.token.consume(), state: self.state.clone(), stats: stats::Scope::default(), + epoch: None, } } @@ -625,6 +627,8 @@ pub struct Consumer { // handoff. Inherited by the tracks subscribed through this handle. Empty (no-op) // for an untagged broadcast. stats: stats::Scope, + // The epoch of the route an origin resolved this handle through. + epoch: Option, } impl Clone for Consumer { @@ -634,6 +638,7 @@ impl Clone for Consumer { alive: self.alive.clone(), state: self.state.clone(), stats: self.stats.clone(), + epoch: self.epoch.clone(), } } } @@ -662,6 +667,22 @@ impl Consumer { self } + /// Stamp the epoch of the route an origin resolved this handle through. + pub(crate) fn with_epoch(mut self, epoch: Option) -> Self { + self.epoch = epoch; + self + } + + /// The publisher epoch of the route this handle was resolved through, if it has one. + /// + /// Set by [`origin::Consumer::request_broadcast`](crate::origin::Consumer::request_broadcast) + /// together with the result, so it names the publisher instance actually serving this + /// handle even when the request was unpinned. `None` for a route without an epoch or a + /// handle taken straight from a producer. + pub fn epoch(&self) -> Option<&crate::Epoch> { + self.epoch.as_ref() + } + /// The broadcast's metadata, as reached through this handle. pub fn info(&self) -> &Info { &self.info @@ -804,6 +825,7 @@ impl WeakConsumer { alive: self.alive.consume(), state: self.state.clone(), stats: stats::Scope::default(), + epoch: None, } } } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index a4218351b94..d8417bcb0c1 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3800,9 +3800,13 @@ impl Requesting { self } - /// Stamp a resolved broadcast with the path this cursor asked for and its egress scope. - fn hand_out(&self, broadcast: broadcast::Consumer) -> broadcast::Consumer { - broadcast.with_path(self.path.clone()).with_stats(self.stats.clone()) + /// Stamp a resolved broadcast with the path this cursor asked for, its egress scope, + /// and the epoch of the route it resolved through. + fn hand_out(&self, broadcast: broadcast::Consumer, epoch: Option) -> broadcast::Consumer { + broadcast + .with_path(self.path.clone()) + .with_stats(self.stats.clone()) + .with_epoch(epoch) } /// Poll for the requested broadcast without blocking. @@ -3816,7 +3820,7 @@ impl Requesting { })) { // Another instance than the one named: never hand it out. Ok((Ok(_), epoch)) if self.epoch.is_some() && epoch != self.epoch => Err(Error::Unroutable), - Ok((result, _)) => result.map(|broadcast| self.hand_out(broadcast)), + Ok((result, epoch)) => result.map(|broadcast| self.hand_out(broadcast, epoch)), // Every handler dropped without resolving: nobody could route it. Err(_closed) => Err(Error::Unroutable), }, @@ -7686,11 +7690,17 @@ mod tests { async fn a_request_racing_a_newer_epoch_skips_the_old_front() { let producer = origin(1).produce(); let consumer = producer.consume(); + let first = crate::Epoch::mint(); let old = producer - .publish("room/alice", Route::default().with_epoch(crate::Epoch::mint())) + .publish("room/alice", Route::default().with_epoch(first.clone())) .unwrap(); let _old_track = old.create_track("video", None).unwrap(); let resolved = consumer.request_broadcast("room/alice", None).await.expect("resolves"); + assert_eq!( + resolved.epoch(), + Some(&first), + "an unpinned request names what it resolved" + ); let epoch = crate::Epoch::mint(); let new = producer @@ -7708,6 +7718,7 @@ mod tests { ); assert!(!bare.is_clone(&resolved), "a bare request joined the superseded front"); assert!(named.is_clone(&bare), "both share the new front"); + assert_eq!(bare.epoch(), Some(&epoch), "an unpinned request names the new epoch"); let mut subscription = named.track("video").unwrap().subscribe(None).await.unwrap(); deliver(&new_track, &mut subscription, b"new").await; } @@ -7780,10 +7791,24 @@ mod tests { let other: crate::Epoch = "01900000-0000-7000-8000-000000000002".parse().unwrap(); let refused = consumer.request_broadcast("room/alice", other).await; assert!(matches!(refused, Err(Error::Unroutable))); - consumer + let named = consumer .request_broadcast("room/alice", epoch()) .await .expect("the named epoch resolves"); + assert_eq!(named.epoch(), Some(&epoch())); + } + + /// A route without an epoch resolves a handle without one. + #[moq_net_sim::test] + async fn a_route_without_an_epoch_resolves_without_one() { + let producer = origin(1).produce(); + let _broadcast = producer.publish("room/alice", Route::default()).unwrap(); + let resolved = producer + .consume() + .request_broadcast("room/alice", None) + .await + .expect("resolves"); + assert_eq!(resolved.epoch(), None); } /// The driver's completion contract: it resolves once every producer handle diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index df710185554..21ba2886e30 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -106,18 +106,15 @@ impl Default for Config { /// every node ever seen. Each route epoch is its own node: a restarted /// producer, or a group returning from idle, announces a new epoch whose /// counters add to the old epoch's kept contribution instead of regressing it. -/// Losing or recovering epoch metadata carries only the immediately outgoing -/// same-path contribution across the transition, avoiding duplicate counters for -/// one publisher reached through mixed-version links. The latest unversioned -/// snapshot replaces it; that snapshot cannot identify a reconnect or restart. -/// Metadata loss carries only the immediately outgoing same-path contribution. -/// Its known lineage remains in grace if recovery identifies a different epoch. -/// A fresh unversioned snapshot replaces the outgoing snapshot, but cannot -/// distinguish a reconnect from a restart. -/// Only a route without an epoch (an older producer, or a session older than -/// moq-lite 07) returning within the grace with a lower counter regresses the -/// merged counter, the same reset contract a single node's own restart -/// follows. Presence is not sticky: a departed node stops counting sessions +/// A reader stays pinned to its epoch, so a replacement never feeds it. +/// When epoch metadata disappears or returns at a path (one publisher reached +/// through mixed-version links), only the immediately outgoing contribution +/// carries across; a different known epoch keeps its own entry, even when an +/// unversioned reader resolves it first. A route without an epoch (an older +/// producer, or a session older than moq-lite 07) cannot tell a reconnect from +/// a restart, so its return within the grace with a lower counter regresses the +/// merged counter, the same reset contract a single node's own restart follows. +/// Presence is not sticky: a departed node stops counting sessions /// immediately. pub struct Consumer { origin: origin::Consumer, @@ -527,6 +524,19 @@ fn advance( loop { match &mut node.reader { Reader::Resolving { pending, queued } => match pending.poll_ok(waiter) { + // An unpinned retry on a known lineage can reach a replacement publisher + // before its announcement drains here. Its counters belong to its own entry, + // which that announcement creates, so this lineage keeps its last frame. + Poll::Ready(Ok(broadcast)) + if node.epoch.is_none() + && node.identity.is_some() + && broadcast + .epoch() + .is_some_and(|epoch| node.identity.as_ref() != Some(epoch)) => + { + tracing::debug!(name, "stats: node resolved another publisher"); + return changed | node.depart(); + } Poll::Ready(Ok(broadcast)) => match broadcast.track(name) { Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), Err(err) => { From 29974030e8d0e1e385a4853ea69590e328d5be1b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 16:38:35 -0700 Subject: [PATCH 8/9] fix(net): a dynamic route's answer keeps the epoch it was asked under Changing a dynamic route's epoch refuses the requests its handler still holds and forgets its served answers, as JS does. The front stamps the epoch the route had when asked, and asks again when an answer lands after the route moved to another instance. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 150 ++++++++++++++++++++++++++------- 1 file changed, 119 insertions(+), 31 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index d8417bcb0c1..3a1c4c2ad65 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1991,6 +1991,14 @@ impl AnnounceProducer { let Some(entry) = shared.routes.entry_mut(prefix, *id) else { return Err(Error::Closed); }; + // Another publisher instance: what its server was asked, or answered, for the + // old one is not its own. Under the table lock, so no front sees the new epoch + // while an old answer can still land. + if entry.epoch != route.epoch + && let Some(server) = &entry.server + { + server.lock().renew(); + } entry.hops = route.hops.clone(); entry.epoch = route.epoch.clone(); entry.stale = stale; @@ -2413,9 +2421,11 @@ async fn serve_front(task: FrontTask) -> Vec { let mut dynamic = broadcast.dynamic(); let mut front = Front::new(track::IDLE_LINGER); let mut sources: HashMap = HashMap::new(); + // The epoch each source's route had when it was asked: the instance its content is from. + let mut epochs: HashMap> = HashMap::new(); let mut next_source = 0u64; - // The in-flight upstream request: the route and its pending channel. - let mut upstream: Option<(u64, kio::Consumer)> = None; + // The in-flight upstream request: the route, its epoch when asked, and its pending channel. + let mut upstream: Option<(u64, Option, kio::Consumer)> = None; let mut tracks: HashMap, TrackIo> = HashMap::new(); let mut deadline = crate::time::Deadline::new(&timers); // The watch generation the last selection saw. @@ -2463,16 +2473,18 @@ async fn serve_front(task: FrontTask) -> Vec { ) }; - // Whether resolving through `route` would serve an epoch a newer one already beat. - let superseded = |front: &Front, route: u64| { + // Whether an answer `route` gave for `asked` would serve an epoch a newer one already + // beat, or one the route itself has since moved off. + let superseded = |front: &Front, route: u64, asked: &Option| { let table = shared.read(); let Some(entry) = table.routes.covering(&path.as_path()).find(|entry| entry.id == route) else { return false; }; - matches!( - pick(&table, &path.as_path(), horizon, front, Some(&entry.epoch)), - Pick::Superseded - ) + entry.epoch != *asked + || matches!( + pick(&table, &path.as_path(), horizon, front, Some(&entry.epoch)), + Pick::Superseded + ) }; events.push_back(select(&mut front, &sources, &mut seen, &resolved)); @@ -2494,9 +2506,9 @@ async fn serve_front(task: FrontTask) -> Vec { .routes .covering(&path.as_path()) .find(|entry| entry.id == route && entry.live()) - .map(|entry| (entry.source.clone(), entry.server.clone())) + .map(|entry| (entry.source.clone(), entry.server.clone(), entry.epoch.clone())) }; - let Some((source, server)) = found else { + let Some((source, server, epoch)) = found else { events.push_back(Event::Resolved { route, result: Err(Refusal { @@ -2510,6 +2522,7 @@ async fn serve_front(task: FrontTask) -> Vec { let id = next_source; next_source += 1; sources.insert(id, source); + epochs.insert(id, epoch); events.push_back(Event::Resolved { route, result: Ok(id) }); continue; } @@ -2546,6 +2559,7 @@ async fn serve_front(task: FrontTask) -> Vec { let id = next_source; next_source += 1; sources.insert(id, weak.consume()); + epochs.insert(id, epoch); events.push_back(Event::Resolved { route, result: Ok(id) }); continue; } @@ -2572,13 +2586,14 @@ async fn serve_front(task: FrontTask) -> Vec { } } }; - upstream = Some((route, pending)); + upstream = Some((route, epoch, pending)); } Action::Detach { source } => { // Readers keep reading the source's copy until a replacement is // spliced, so a route being beaten keeps serving until then; a // route that died ends its copy, which says so. sources.remove(&source); + epochs.remove(&source); for io in tracks.values_mut() { if io.query.as_ref().is_some_and(|(asked, _)| asked.source == source) { io.query = None; @@ -2589,19 +2604,12 @@ async fn serve_front(task: FrontTask) -> Vec { } } Action::Resolve => { - // The broadcast is whatever its first route serves. - let table = shared.read(); + // The broadcast is whatever its first source serves, from the instance + // its route had when asked. let epoch = front - .serving_route() - .and_then(|route| { - table - .routes - .covering(&path.as_path()) - .find(|entry| entry.id == route) - .map(|entry| entry.epoch.clone()) - }) + .serving() + .and_then(|source| epochs.get(&source).cloned()) .flatten(); - drop(table); resolved = Some(epoch.clone()); if let Ok(mut pending) = request.write() && pending.resolved.is_none() @@ -2744,7 +2752,7 @@ async fn serve_front(task: FrontTask) -> Vec { if let Poll::Ready(Ok(request)) = dynamic.poll_requested_track(waiter) { return Poll::Ready(Step::Assigned(request)); } - if let Some((route, pending)) = &upstream + if let Some((route, _, pending)) = &upstream && let Poll::Ready(result) = pending.poll(waiter, |p| match &p.resolved { Some(result) => Poll::Ready(result.clone()), None => Poll::Pending, @@ -2832,12 +2840,12 @@ async fn serve_front(task: FrontTask) -> Vec { } } Step::Resolved(route, result) => { - upstream = None; + let asked = upstream.take().and_then(|(_, epoch, _)| epoch); match result { - // A newer epoch won the path while the first request was in flight: - // the route was beaten, so ask the winner rather than resolve - // requesters onto a broadcast that is already replaced. - Ok(_) if resolved.is_none() && superseded(&front, route) => Event::Resolved { + // A newer epoch won the path, or the route moved to another instance, + // while the first request was in flight: ask the winner rather than + // resolve requesters onto a broadcast that is already replaced. + Ok(_) if resolved.is_none() && superseded(&front, route, &asked) => Event::Resolved { route, result: Err(Refusal { err: Error::Unroutable, @@ -2848,6 +2856,7 @@ async fn serve_front(task: FrontTask) -> Vec { let id = next_source; next_source += 1; sources.insert(id, source); + epochs.insert(id, asked); Event::Resolved { route, result: Ok(id) } } Err(err) => Event::Resolved { @@ -3573,8 +3582,9 @@ impl Dynamic { /// At the same [`Route::epoch`], consumers observe another active update for /// the same prefix and sessions forward it as a restart, so re-pricing never /// looks like new content. Another epoch, or none, names another publisher - /// instance: consumers see the old broadcast end and a new one start, and the - /// broadcasts served through the old one end. To re-price, start from the + /// instance: consumers see the old broadcast end and a new one start, the + /// broadcasts served through the old one end, and requests still waiting on it + /// are refused with [`Error::Unroutable`]. To re-price, start from the /// current route: `dynamic.update(dynamic.route().with_cost(cost))`. /// /// The prefix is fixed at announce time: to move a route, drop this and call @@ -3647,6 +3657,11 @@ impl ServeState { if state.closed { return; } + // Refused already (the route changed instance): that answer stands and nothing + // is cached for the next request. + if state.requests.remove_if(path, |p| p.same_channel(producer)).is_none() { + return; + } let resolved = match result { Ok(broadcast) => { // If a live broadcast was already served for this path while we were @@ -3657,13 +3672,23 @@ impl ServeState { } Err(err) => Err(err), }; - state.requests.remove_if(path, |p| p.same_channel(producer)); if let Ok(mut pending) = producer.write() { pending.resolved.get_or_insert(resolved); drop(state); } } + /// Refuse every request, queued or handed to a handler, and forget the answers served, + /// because the route now names another publisher instance. + fn renew(&mut self) { + for producer in self.requests.drain_all() { + if let Ok(mut request) = producer.write() { + request.resolved.get_or_insert(Err(Error::Unroutable)); + } + } + self.served = WeakCache::default(); + } + /// Drop the still-pending entry, if it is still ours. fn forget(shared: &kio::Shared, path: &PathOwned, producer: &kio::Producer) { shared.lock().requests.remove_if(path, |p| p.same_channel(producer)); @@ -7749,6 +7774,69 @@ mod tests { deliver(&new_track, &mut subscription, b"new").await; } + /// A handler asked under one publisher instance does not answer for the next: the + /// route's epoch changing while the request waits refuses it, and its late answer + /// is neither handed out under the new epoch nor cached for the next request. + #[moq_net_sim::test] + async fn a_pending_answer_does_not_outlive_its_epoch() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let dynamic = producer + .dynamic("room", Route::default().with_epoch(crate::Epoch::mint())) + .unwrap(); + let pending = consumer.request_broadcast("room/alice", None); + let request = queued(&dynamic).await; + + let epoch = crate::Epoch::mint(); + dynamic.update(dynamic.route().with_epoch(epoch.clone())).unwrap(); + let stale = broadcast::Info::new().produce(); + let _stale_track = stale.create_track("video", None).unwrap(); + request.accept(&stale); + assert!( + matches!(pending.await, Err(Error::Unroutable)), + "an answer for the old instance resolved" + ); + + let retry = consumer.request_broadcast("room/alice", None); + let request = queued(&dynamic).await; + let fresh = broadcast::Info::new().produce(); + let fresh_track = fresh.create_track("video", None).unwrap(); + request.accept(&fresh); + let resolved = retry.await.expect("the new instance resolves"); + assert_eq!(resolved.epoch(), Some(&epoch)); + let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); + deliver(&fresh_track, &mut subscription, b"new").await; + } + + /// An answer that lands just before its route changes instance, but that the front + /// has not taken yet, is not stamped with the new epoch: the front asks again. + #[moq_net_sim::test] + async fn an_answer_before_an_epoch_change_is_asked_again() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let dynamic = producer + .dynamic("room", Route::default().with_epoch(crate::Epoch::mint())) + .unwrap(); + let pending = consumer.request_broadcast("room/alice", None); + let request = queued(&dynamic).await; + + // The answer and the change land in the same tick. + let stale = broadcast::Info::new().produce(); + let _stale_track = stale.create_track("video", None).unwrap(); + request.accept(&stale); + let epoch = crate::Epoch::mint(); + dynamic.update(dynamic.route().with_epoch(epoch.clone())).unwrap(); + + let request = queued(&dynamic).await; + let fresh = broadcast::Info::new().produce(); + let fresh_track = fresh.create_track("video", None).unwrap(); + request.accept(&fresh); + let resolved = pending.await.expect("the new instance resolves"); + assert_eq!(resolved.epoch(), Some(&epoch)); + let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); + deliver(&fresh_track, &mut subscription, b"new").await; + } + /// Announce cursors follow the newest epoch over a cheaper one, and deliver a /// change of epoch as an end and a fresh start: a different broadcast. #[moq_net_sim::test] From feb3018b4dfd0fa72fa4856792e35f69de972e8e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 7 Oct 2026 19:22:56 -0700 Subject: [PATCH 9/9] docs: upgrade note for the request_broadcast epoch argument Co-Authored-By: Claude Opus 5.5 --- doc/setup/upgrade.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index cfe32310732..a8fa63255f3 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -47,6 +47,13 @@ These land with the next breaking release, not the 2026-09-23 train. `producer.demand().used` and `.unused()`, as are `Group.Producer`'s, and `Allocator.reserve` takes `producer.demand()`, replacing the `Bandwidth.Demand` interface. +- **`request_broadcast` takes an epoch.** In Rust, + `origin::Consumer::request_broadcast(path)` is + `request_broadcast(path, None)`; pass an `Epoch` instead to refuse any other + publisher instance. In TypeScript, `RequestOptions.epoch` does the same. A + dynamic route updated to another epoch now refuses the requests its handler + still holds with `Unroutable`, so a handler answering them late sees its + answer dropped. - **`moq --hop` is removed; `--epoch` replaces it.** A redundant pair shares an epoch, a UUIDv7 such as `uuidgen -7` prints, instead of a Hop ID: pass the same `--epoch` (or `MOQ_EPOCH`) to both. Unlike a rename, `--hop` is now an