diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 178dd833d06..ea41494ea88 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -147,6 +147,12 @@ its counters run from its start, and its first publish carries them. A reader treats each instance as its own set of counters. The instance rides moq-lite 07 announcements; over anything else the receiving session mints one per announcement, so a reader sees the same change as an end and a start at the path. +A minted instance also changes on every reconnect, so it cannot tell a restart from a +redial. When a minted instance follows a minted one while the departed one is still +within the aggregating reader's grace ([`Config::grace`](https://docs.rs/moq-stats/latest/moq_stats/aggregate/struct.Config.html#structfield.grace)), +its first frame decides: counters that continue are the same node, and a new bucket +epoch or a counter going down is a restart, which adds to the old counters. A return +after the grace starts over, so its earlier traffic counts twice. An aggregating reader pins each node's subscription to its instance. Every bucket shares the node's broadcast, so a token scoped to the stats prefix diff --git a/quest/m1/precut-fixes/stats-minted-reconnect.md b/quest/m1/precut-fixes/stats-minted-reconnect.md deleted file mode 100644 index debf8b7db12..00000000000 --- a/quest/m1/precut-fixes/stats-minted-reconnect.md +++ /dev/null @@ -1,43 +0,0 @@ -# [S] The stats aggregator counts a reconnected node once - -## Goal - -`moq-stats`'s aggregator keeps one entry per node across a reconnect on a -route whose instance a session minted (every version before lite-07, and -moq-transport), so a cluster link redial, a route failover, or a relay -restart within the aggregator's grace (`Config::grace`, 10 minutes) never adds a node's cumulative counters on top of its own retained -contribution. A real node restart is still detected by its counters going -down, as `doc/concept/stats.md` says for minted routes. - -## Plan - -Found by the 2026-11-17 pre-cut correctness audit; confirmed from source. -`Merged::apply_announce` (`rs/moq-stats/src/aggregate.rs`) keys nodes by -`(path, instance)`. Below lite-07 the receiving session mints a fresh -instance per session for each advertised path (`Minted::acquire` in -`rs/moq-net/src/lite/subscriber.rs`), and lite-06 is the default version. -After a redial the same node arrives under a new key, its cumulative totals -count from the start again, and once the old entry's 10-minute grace ends -its last frame folds into `retired`: the node is counted twice for good. -`release` keyed nodes by path only, so it does not have this. - -Recommended: when the new instance `is_minted()` and the path's previous -entry is also minted, carry the previous entry over to the new key, and let -the existing `regressed` check treat a counter going down as a restart. -A lite-07 instance a publisher named keeps today's per-instance entry. -A return after the grace starts over from zero, as `Merged`'s docs already -say for every route: the old entry has folded into `retired` and its path -lookup is gone, and bounding a per-node ledger of retired nodes would cost -memory for every departed node. This quest keeps that documented limit. - -Test: a simulated-clock aggregator test that unannounces and re-announces a -node path under a fresh minted instance, with counters continuing, and -asserts the merged total after the grace equals the node's last frame; and -the boundary, a re-announce just after the grace, which starts over as -documented. - -Public API: none. Wire: none. - -Caused by, for the fix PR's `## Caused by`: - -- [moq-dev/moq#7489](https://github.com/moq-dev/moq/pull/7489) - keyed aggregator nodes by announced instance diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 6301850b0c1..1e2d98ab94a 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -38,8 +38,9 @@ pub struct Config { pub prefix: Path, /// How long a departed node's traffic waits for its path to return before /// folding into the retired total (default 10 minutes). A return within it - /// under the same instance resumes the node's own counters; a return after it with counters intact counts its earlier - /// traffic twice, so size this above reconnect times. + /// under the same instance, or a minted one after a minted one with its + /// counters intact, resumes the node's own counters; a return after it with counters intact counts its + /// earlier traffic twice, so size this above reconnect times. pub grace: Duration, } @@ -94,8 +95,11 @@ impl Default for Config { /// track's new [`BucketFrame::epoch`] adds to the old epoch's counters the same /// way. A reader stays pinned to its broadcast instance, so a /// replacement never feeds it. A wire that carries no instance has its relay -/// mint one per announcement, so a node reached through one announces anew each -/// time it is announced again. +/// mint one per announcement. A minted instance following a minted one at the +/// same path, while the departed one is still within [`Config::grace`], takes +/// over its entry, and its first frame decides: counters that continue are the +/// same node reached anew, while a new bucket epoch or a counter going down is +/// a restart whose new counters add to the old ones. /// /// A bucket's per-label sessions are the exception: they are a gauge, so a /// departed node stops counting them immediately. @@ -109,8 +113,10 @@ impl Default for Config { /// rollup. A node that resumes after its departure closed its open pairs /// reopens them as new starts rather than lowering their ends. A bucket's /// per-path traffic also reopens pairs when a node resumes without regressing, -/// but keeps the reset contract of a single node: one that returns within the -/// grace with a lower counter regresses the merged counter. +/// but keeps the reset contract of a single node: a new counter epoch on a +/// node it keeps reading regresses the merged counter. A minted node that +/// returns within the grace from a restart is the exception: its first frame +/// folds the earlier run into the rollup, as an expired grace would. pub struct Consumer { origin: origin::Consumer, config: Config, @@ -510,6 +516,10 @@ trait Shape: Default { /// Forget the contribution, once it has folded into the retired total. fn clear(&mut self); + + /// Whether `frame` comes from a restarted producer rather than the one this + /// contribution was read from. + fn restarted(&self, frame: &Self::Frame) -> bool; } /// One node's fold of a keyed frame of [`Mergeable`] values. @@ -600,6 +610,14 @@ impl Keyed { self.base.clear(); self.last = None; } + + /// Whether any counter in `frame` went down from the last one read. + fn regressed(&self, frame: &BTreeMap) -> bool { + self.last + .iter() + .flatten() + .any(|(key, prev)| frame.get(key).is_some_and(|next| next.regressed(prev))) + } } impl Shape for Keyed { @@ -622,6 +640,10 @@ impl Shape for Keyed { fn clear(&mut self) { self.forget(); } + + fn restarted(&self, frame: &Self::Frame) -> bool { + self.regressed(frame) + } } /// One node's fold of a bucket track: the tiers' totals, each `(tier, path)`'s @@ -696,6 +718,11 @@ impl Shape for Bucketed { self.paths.forget(); self.labels.forget(); } + + /// A restarted node mints a new epoch for every bucket track. + fn restarted(&self, frame: &BucketFrame) -> bool { + self.epoch.as_ref().is_some_and(|epoch| *epoch != frame.epoch) + } } /// One node's subscription to the merged track. @@ -733,6 +760,9 @@ struct Node { /// Only an unannounced node folds, so a still-announced node whose reader /// failed keeps its own counters. departed: Option, + /// The shape was taken over from the path's previous minted instance, which + /// may be the same node reached anew or a restart: the first frame decides. + adopted: bool, } impl Node { @@ -753,6 +783,16 @@ impl Node { self.shape.observe(frame); } + /// Settle an adopted contribution on the new instance's first frame: one the + /// frame shows came from before a restart folds into `retired`, as a + /// departed instance's would, instead of resetting under the new counters. + fn settle_adopted(&mut self, frame: &S::Frame, retired: &mut S::Merged) { + if std::mem::take(&mut self.adopted) && self.shape.restarted(frame) { + self.shape.contribute(retired); + self.shape.clear(); + } + } + /// Fold the contribution into `retired` if the node departed at least /// `grace` before `now`, leaving the merged view unchanged. Returns whether /// it folded. A grace too long to represent never folds. @@ -778,13 +818,16 @@ struct Merged { name: String, config: moq_json::snapshot::consumer::Config, /// One entry per live or recently departed node broadcast, keyed by - /// absolute announced path and route instance, so each instance is its own entry. + /// absolute announced path and route instance, so each named instance is its + /// own entry; a minted one takes over the path's previous minted entry until + /// its first frame shows a restart. nodes: HashMap<(Path, Instance), Node>, /// The instance last announced at each path, including while it remains in grace. last_announced: HashMap, /// How long a departed node waits before folding into `retired`. grace: Duration, - /// The summed last frames of every node whose grace elapsed. + /// The summed last frames of every node whose grace elapsed, plus each + /// minted takeover's earlier run once its first frame shows a restart. retired: S::Merged, } @@ -860,7 +903,7 @@ impl Merged { let retired = &mut self.retired; let last_announced = &mut self.last_announced; self.nodes.retain(|(path, instance), node| { - changed |= advance(node, origin, config, name, waiter); + changed |= advance(node, origin, config, name, waiter, retired); if node.fold_expired(retired, grace, now) { if last_announced.get(path) == Some(instance) { last_announced.remove(path); @@ -922,7 +965,25 @@ impl Merged { }; } - self.last_announced.insert(key.0.clone(), key.1.clone()); + let mut changed = false; + let previous = self.last_announced.insert(key.0.clone(), key.1.clone()); + // Below lite-07 each session mints a fresh instance per announcement, so + // a minted instance after a minted one is likely the same node reached + // anew (a redial, a failover, a relay restart): it takes over the + // previous entry until its first frame shows whether the node restarted. + if let Some(previous) = previous + && previous != key.1 + && previous.is_minted() + && key.1.is_minted() + && !self.nodes.contains_key(&key) + && let Some(mut node) = self.nodes.remove(&(key.0.clone(), previous)) + { + // Stop reading the old instance; the arm below re-resolves the new one. + changed = node.depart(); + node.instance = key.1.clone(); + node.adopted = true; + self.nodes.insert(key.clone(), node); + } // A route update on a node already tracked (a reprice, or another route // of the same instance) keeps the live reader: existing // subscriptions survive it, and the reader re-resolves through @@ -951,10 +1012,11 @@ impl Merged { path: path.clone(), shape: S::default(), departed: None, + adopted: false, }); } } - false + changed } /// Sum the retired total and every node's contribution, per key. @@ -975,6 +1037,7 @@ fn advance( config: &moq_json::snapshot::consumer::Config, name: &str, waiter: &Waiter, + retired: &mut S::Merged, ) -> bool { let mut changed = false; // At most one re-resolve per call: a subscription that terminates @@ -1090,6 +1153,7 @@ fn advance( }, Reader::Active(reader) => match reader.poll_latest(waiter) { Poll::Ready(Ok(Some(frame))) => { + node.settle_adopted(&frame.value, retired); node.observe(frame.value); changed = true; } @@ -1377,6 +1441,7 @@ mod tests { bucket: moq_json::snapshot::Producer, track: track::Producer, frame: TrafficFrame, + epoch: Epoch, } impl NodeBroadcast { @@ -1385,13 +1450,24 @@ mod tests { } fn with_instance(origin: &origin::Producer, bucket: &str, node: &str, instance: moq_net::Instance) -> Self { + Self::with_epoch(origin, bucket, node, instance, bucket_epoch()) + } + + /// A node publishing a bucket run of its own, as a restarted node does. + fn with_epoch( + origin: &origin::Producer, + bucket: &str, + node: &str, + instance: moq_net::Instance, + epoch: Epoch, + ) -> Self { let path = format!(".stats/node/{node}"); let options = origin::RouteOptions::default().with_instance(instance, 0); let source = origin.route(path.as_str(), options).expect("announce"); let track = source .broadcast() .create_track( - crate::bucket_track(moq_net::Path::new(bucket).unwrap(), &bucket_epoch()), + crate::bucket_track(moq_net::Path::new(bucket).unwrap(), &epoch), timed(), ) .expect("create track"); @@ -1401,6 +1477,7 @@ mod tests { track, source, frame: TrafficFrame::new(), + epoch, } } @@ -1414,7 +1491,7 @@ mod tests { bucket.paths.entry(path.clone()).or_default().publisher = *traffic; } let frame = BucketFrame { - epoch: bucket_epoch(), + epoch: self.epoch.clone(), tiers: BTreeMap::from([(String::new(), bucket)]), }; self.bucket.update(&frame, next_stamp(&self.track)).expect("publish"); @@ -1714,6 +1791,98 @@ mod tests { assert_eq!(frame.get("acme/room").expect("entry").bytes, 170, "100 + 30 + 40"); } + /// Node A (100, under a minted instance) and node B (40) merged to 140, then A + /// departs and returns `after` later through a new session, which mints a new + /// instance, with its counters intact plus 20. + async fn reconnect_minted(after: Duration) -> (TrafficConsumer, NodeBroadcast) { + let origin = produce_origin(); + let agg = Consumer::new(origin.consume(), Config::new().with_grace(GRACE)); + let mut traffic = traffic(&agg); + let mut node_a = NodeBroadcast::with_instance(&origin, "acme", "a", moq_net::fuzz::minted_instance()); + let mut node_b = NodeBroadcast::new(&origin, "acme", "b"); + node_a.publish("acme/room", 100); + node_b.publish("acme/room", 40); + read_until_bytes(&mut traffic, "acme/room", 140).await; + + let frame = node_a.frame.clone(); + drop(node_a); + settle(&mut traffic).await; + + tokio::time::advance(after).await; + let mut node_a = NodeBroadcast::with_instance(&origin, "acme", "a", moq_net::fuzz::minted_instance()); + node_a.frame = frame; + node_a.publish("acme/room", 20); + // The node outlives the test's reads, like node B. + std::mem::forget(node_a); + (traffic, node_b) + } + + #[tokio::test(start_paused = true)] + async fn minted_reconnect_within_grace_counts_once() { + // A session mints a fresh instance per announcement below lite-07, so a + // reconnected node arrives under a new instance with its counters intact. + // It continues its own entry: once the grace passes, nothing it reported + // before the reconnect folds into the retired total on top of it. + let (mut traffic, _node_b) = reconnect_minted(GRACE - Duration::from_secs(1)).await; + let frame = read_monotonic_until(&mut traffic, "acme/room", 140, 160).await; + assert_eq!(frame.get("acme/room").expect("entry").bytes, 160, "100 + 20 + 40"); + + tokio::time::advance(GRACE).await; + settle(&mut traffic).await; + assert_eq!(traffic.inner.nodes.len(), 2, "one entry per node"); + assert!(traffic.inner.retired.is_empty(), "nothing folded"); + assert_eq!( + publisher(&traffic.inner.merged()) + .get("acme/room") + .expect("entry") + .bytes, + 160 + ); + } + + #[tokio::test(start_paused = true)] + async fn minted_reconnect_after_grace_starts_over() { + // Past the grace the departed entry has folded, so a minted return starts + // over and counts its earlier traffic twice, as for any route. + let (mut traffic, _node_b) = reconnect_minted(GRACE).await; + let frame = read_monotonic_until(&mut traffic, "acme/room", 140, 260).await; + assert_eq!( + frame.get("acme/room").expect("entry").bytes, + 260, + "retired 100 + returned 120 + 40" + ); + } + + #[tokio::test(start_paused = true)] + async fn minted_restart_within_grace_keeps_path_traffic() { + // A restarted node reached through an older wire also returns under a + // new minted instance within the grace, but with a new bucket epoch and + // fresh counters: its earlier per-path traffic stays counted, as for a + // restart under a named instance, instead of resetting under the new run. + let origin = produce_origin(); + let agg = Consumer::new(origin.consume(), Config::new().with_grace(GRACE)); + let mut traffic = traffic(&agg); + let mut node_a = NodeBroadcast::with_instance(&origin, "acme", "a", moq_net::fuzz::minted_instance()); + let mut node_b = NodeBroadcast::new(&origin, "acme", "b"); + node_a.publish("acme/room", 100); + node_b.publish("acme/room", 40); + read_until_bytes(&mut traffic, "acme/room", 140).await; + + drop(node_a); + settle(&mut traffic).await; + + let epoch: Epoch = "01900000-0000-7000-8000-000000000100".parse().unwrap(); + let mut node_a = NodeBroadcast::with_epoch(&origin, "acme", "a", moq_net::fuzz::minted_instance(), epoch); + node_a.publish("acme/room", 30); + let frame = read_monotonic_until(&mut traffic, "acme/room", 140, 170).await; + assert_eq!(frame.get("acme/room").expect("entry").bytes, 170, "100 + 30 + 40"); + + tokio::time::advance(GRACE).await; + node_a.publish("acme/room", 5); + let frame = read_monotonic_until(&mut traffic, "acme/room", 170, 175).await; + assert_eq!(frame.get("acme/room").expect("entry").bytes, 175, "100 + 35 + 40"); + } + #[tokio::test(start_paused = true)] async fn failed_reader_does_not_fold_while_announced() { // A node whose reader failed under a still-announced broadcast keeps its @@ -2006,11 +2175,20 @@ mod tests { impl TotalsNode { fn new(origin: &origin::Producer, node: &str) -> Self { + Self::from_options(origin, node, moq_net::origin::RouteOptions::default()) + } + + fn with_instance(origin: &origin::Producer, node: &str, instance: moq_net::Instance) -> Self { + Self::from_options( + origin, + node, + moq_net::origin::RouteOptions::default().with_instance(instance, 0), + ) + } + + fn from_options(origin: &origin::Producer, node: &str, options: moq_net::origin::RouteOptions) -> Self { let source = origin - .route( - format!(".stats/node/{node}").as_str(), - moq_net::origin::RouteOptions::default(), - ) + .route(format!(".stats/node/{node}").as_str(), options) .expect("create broadcast"); let track = source .broadcast() @@ -2059,6 +2237,33 @@ mod tests { read_until(&mut totals.inner, grew, |frame| frame[""].publisher.bytes == 175).await; } + #[tokio::test(start_paused = true)] + async fn minted_reconnect_detects_a_restart() { + // A restarted node reached through an older wire comes back under a new + // minted instance, which takes over its entry; its counter going down + // shows the restart, so the old readout folds and the new one adds to + // it, through and past the grace. + let origin = produce_origin(); + let mut node_a = TotalsNode::with_instance(&origin, "a", moq_net::fuzz::minted_instance()); + node_a.publish(100); + + let agg = Consumer::new(origin.consume(), Config::new().with_grace(GRACE)); + let mut totals = agg.totals(); + let grew = |before: &Totals, after: &Totals| after.publisher.bytes >= before.publisher.bytes; + read_until(&mut totals.inner, grew, |frame| frame[""].publisher.bytes == 100).await; + + drop(node_a); + let mut node_a = TotalsNode::with_instance(&origin, "a", moq_net::fuzz::minted_instance()); + node_a.publish(30); + read_until(&mut totals.inner, grew, |frame| frame[""].publisher.bytes == 130).await; + assert_eq!(totals.inner.nodes.len(), 1, "the reconnect continues the node's entry"); + + tokio::time::advance(GRACE).await; + node_a.publish(35); + read_until(&mut totals.inner, grew, |frame| frame[""].publisher.bytes == 135).await; + assert_eq!(totals.inner.retired[""].publisher.bytes, 100, "the first run folded"); + } + /// A node announced at `.stats/node/a`, ready to observe frames by hand. fn hand_merged() -> (Merged>, (Path, Instance)) { let origin = produce_origin();