diff --git a/rs/moq-net/benches/routes.rs b/rs/moq-net/benches/routes.rs index 034a7a30051..48aea45c498 100644 --- a/rs/moq-net/benches/routes.rs +++ b/rs/moq-net/benches/routes.rs @@ -11,6 +11,11 @@ //! - lagging: a relay hears the origin on two streams, the second some one-claim //! batches behind the first, and each hears its next batch. Swept over the claims //! the origin holds and how far the second lags: neither axis is expected to grow. +//! - confirmed: a relay holds back every claim from the streams it answers until it +//! confirms serving one, all of them at one instance and epoch. One claim restarts at +//! another epoch, then back at the shared one, each confirmed. Swept over the claims +//! held and the streams answered: the streams axis is expected to grow, since each is +//! sent the claim; the claims axis is not. //! //! The route layer is crate-private, so this reaches it through the `fuzz` module. //! @@ -25,6 +30,9 @@ const SHAPES: [(usize, usize); 4] = [(10, 10), (10_000, 10), (10, 1_000), (10_00 /// `(claims, changes)` shapes for a relay hearing a batch. const RECEIVED: [(usize, usize); 4] = [(100, 1), (10_000, 1), (100, 100), (10_000, 100)]; +/// `(claims, streams)` shapes for a relay holding back claims. +const HELD: [(usize, usize); 4] = [(10, 10), (10_000, 10), (10, 100), (10_000, 100)]; + /// `(claims, lag)` shapes for a relay hearing a stream that lags another. const LAGGING: [(usize, usize); 4] = [(100, 1), (10_000, 1), (100, 100), (10_000, 10_000)]; @@ -89,5 +97,26 @@ fn bench_lagging(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_watched, bench_unwatched, bench_received, bench_lagging); +fn bench_confirmed(c: &mut Criterion) { + let mut group = c.benchmark_group("routes/confirmed"); + for (claims, streams) in HELD { + let id = BenchmarkId::from_parameter(format!("{claims}c_{streams}s")); + let mut fixture = None; + group.bench_function(id, |b| { + let relay = fixture.get_or_insert_with(|| fuzz::RouteRelay::held(claims, streams)); + // Each change's serving change, then one batch per stream answered. + b.iter(|| assert_eq!(relay.restart(), 2 * (1 + streams))); + }); + } + group.finish(); +} + +criterion_group!( + benches, + bench_watched, + bench_unwatched, + bench_received, + bench_lagging, + bench_confirmed +); criterion_main!(benches); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 67b42249ac8..dbbffbb4948 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -643,11 +643,11 @@ impl RouteNode { } /// A relay's lite-07 route node hearing one origin's claims on streams it asked for, -/// and forwarding them on a stream it answers, for the routes bench. +/// and forwarding them on streams it answers, for the routes bench. pub struct RouteRelay { // Dropped first: a stream disconnects from its node on drop. receive: Vec, - answer: crate::route::node::Stream, + answer: Vec, #[allow(dead_code, reason = "held so the streams keep a node to talk to")] node: crate::route::Node, instance: crate::Instance, @@ -663,11 +663,13 @@ impl RouteRelay { /// The origin's identity as the relay hears it. const ORIGIN: u64 = 1; - /// A relay that already heard and forwarded the origin's `claims` claims under - /// `live/`, on each of `streams` streams. - fn new(claims: usize, streams: usize) -> Self { + /// A relay that already heard the origin's `claims` claims under `live/` on each of + /// `streams` streams, and forwarded them on each of `answers` streams. + fn new(claims: usize, streams: usize, answers: usize) -> Self { let node = crate::route::Node::new(2, 2); - let answer = node.answer(crate::route::Link::default(), Path::empty()).unwrap(); + let answer = (0..answers) + .map(|_| node.answer(crate::route::Link::default(), Path::empty()).unwrap()) + .collect(); let instance = crate::Instance::random(); let rev = claims as u64; let advert = crate::route::Advert { @@ -703,13 +705,20 @@ impl RouteRelay { /// A relay hearing the origin's `claims` claims on one stream. pub fn heard(claims: usize) -> Self { - Self::new(claims, 1) + Self::new(claims, 1, 1) + } + + /// A relay hearing the origin's `claims` claims on one stream and answering + /// `answers` streams, holding every claim back from them since none is confirmed + /// served. + pub fn held(claims: usize, answers: usize) -> Self { + Self::new(claims, 1, answers) } /// A relay hearing the origin's `claims` claims on two streams, the second `lag` /// one-claim batches behind the first. pub fn lagging(claims: usize, lag: usize) -> Self { - let mut relay = Self::new(claims, 2); + let mut relay = Self::new(claims, 2, 1); for _ in 0..lag { relay.advance(); } @@ -742,6 +751,29 @@ impl RouteRelay { self.drain() } + /// The first claim restarts at another epoch, then back at the one every other + /// claim shares, the relay confirming each, returning how many outputs it queued. + pub fn restart(&mut self) -> usize { + self.confirm(1) + self.confirm(0) + } + + /// The first claim changes to `epoch`, and the relay confirms serving it. + fn confirm(&mut self, epoch: u64) -> usize { + self.rev += 1; + let claim = crate::route::node::Claim { + epoch, + ..Self::claim(&self.instance, 0) + }; + self.receive[0].batch(Self::ORIGIN, vec![(Self::path(0), Some(claim))], Some(self.rev)); + let serves: Vec<_> = std::iter::from_fn(|| self.receive[0].try_output()).collect(); + for serve in &serves { + if let crate::route::node::Output::Serve { id, claim, served, .. } = serve { + self.receive[0].serving(*id, claim, &served.claim); + } + } + serves.len() + self.drain() + } + /// The first stream hears the origin's next revision, which changes one claim. fn advance(&mut self) { self.rev += 1; @@ -762,7 +794,7 @@ impl RouteRelay { } fn drain(&self) -> usize { - (self.receive.iter().chain([&self.answer])) + (self.receive.iter().chain(&self.answer)) .map(|stream| std::iter::from_fn(|| stream.try_output()).count()) .sum() } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index cbaa658c125..4a6ca7cf19a 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -481,7 +481,7 @@ impl Routed { if same { entry.named = named; entry.route.update(route); - self.stream.serving(id, &key.1); + self.stream.serving(id, &key.1, &served.claim); return; } } @@ -493,7 +493,7 @@ impl Routed { let entry = AnnouncedRoute::new(route, dynamic, wake); entry.waker.wake_by_ref(); self.served.insert(key.clone(), ServedClaim { named, route: entry }); - self.stream.serving(id, &key.1); + self.stream.serving(id, &key.1, &served.claim); } Err(_) => { // Outside our origin's scope. diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 0aa3005b988..297418992a8 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -13,6 +13,11 @@ //! own metric, so the origin ranks them and fails over at once when one goes; any //! feasible route forwards without a loop. A stream that stops carrying one lets go //! only once another serves it, so the origin always holds a route for it. +//! +//! A claim of another identity reaches the other streams only once a stream serving +//! it confirmed that instance, so a peer never names an instance this node's origin +//! cannot route yet. Until then its batches hold the claim back and their floors +//! certify nothing. use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}, @@ -24,7 +29,10 @@ use crate::{Instance, Path}; use rand::RngExt; -use super::{Error, GENERATED, Id, Link, Metric, Out, QUALIFIED, Reach, Rev, Router, Seqno}; +use super::{ + Error, GENERATED, Id, Link, Metric, Out, QUALIFIED, Reach, Rev, Router, Seqno, + view::{ancestors, below}, +}; /// One announce stream in one direction. pub(crate) type Key = u64; @@ -132,6 +140,11 @@ struct Outbox { serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, + /// Claim starts and restarts held back until a stream serving them confirms their + /// instance, by identity, with the identity's last floor, which certifies nothing + /// while any is held. Bounded like `pending`: one per claim. The node's `waiting` + /// indexes them. + withheld: HashMap)>, /// ROUTE_REQUESTs to send, by the identity as the peer advertised it. A higher /// seqno subsumes a lower one, so a writer that stalls holds one per identity. requests: BTreeMap, @@ -159,6 +172,13 @@ type Batch = (Changes, Option); /// Each claim's last change, by path. type Changes = BTreeMap>; +/// Claims held back, by path. +type Held = BTreeMap; + +/// The streams holding back a claim of each identity, by the path each holds it at: +/// one entry per held claim. A change to a home is looked up as a range of paths. +type Waiting = HashMap>>; + impl Outbox { fn is_empty(&self) -> bool { self.ready.is_empty() && self.order.is_empty() && self.serves.is_empty() @@ -176,37 +196,15 @@ impl Outbox { entry } - /// Queue `output`. Returns the identity of a ROUTE_END the reader will never read, - /// since it never read the route it ends, for the caller to ack in its place. - fn push(&mut self, output: Output) -> Option { + /// Queue `output`. + fn push(&mut self, output: Output) { match output { - Output::Route { id, reach: None } => { - let holds = self.delivered.contains_key(&id); - let entry = self.entry(id); - // The reader holds the route once it reads what is pending: an end of - // that keeps any end before, an end of a start it never read is acked. - let ended = entry.ended; - let acked = !holds || ended; - let closing = match ended { - true => entry.closing.take(), - false => entry.batch.take().filter(|_| holds), - }; - *entry = Pending { - ended: holds, - closing, - ..Pending::default() - }; - if !holds && let Some((seq, _)) = self.pending.remove(&id) { - self.order.remove(&seq); - } - acked.then_some(id) - } + Output::Route { reach: None, .. } => unreachable!("a route end is queued with Outbox::end"), Output::Route { id, reach: Some(Reach::Held), } => { self.entry(id).held = true; - None } Output::Route { id, @@ -215,24 +213,8 @@ impl Outbox { let entry = self.entry(id); entry.finite = Some(finite); entry.held = false; - None - } - Output::Batch { id, changes, floor } => { - let ended = self.entry(id).ended; - let (_, entry) = self.pending.get_mut(&id).expect("just queued"); - let sent = self.delivered.get(&id); - let (merged, last) = entry.batch.get_or_insert_default(); - for (path, claim) in changes { - // The reader holds nothing here once the entry reaches it. - let unsent = ended || sent.is_none_or(|sent| !sent.contains(&path)); - match claim { - None if unsent => merged.remove(&path), - claim => merged.insert(path, claim), - }; - } - *last = floor; - None } + Output::Batch { .. } => unreachable!("a batch is queued with Outbox::batch"), // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim, .. } => { @@ -244,7 +226,7 @@ impl Outbox { && queued.claim.epoch == served.claim.epoch { self.serves.insert(seq, output); - return None; + return; } let seq = self.next; self.next += 1; @@ -252,13 +234,115 @@ impl Outbox { self.serves.remove(&old); } self.serves.insert(seq, output); - None } - other => { - self.ready.push_back(other); - None + Output::Echo { .. } => { + self.ready.push_back(output); + } + } + } + + /// Queue the end of `id`'s route, for the stream `key`. Returns `id` when the reader + /// will never read it, since it never read the route it ends, for the caller to ack + /// in its place. + fn end(&mut self, key: Key, id: Id, waiting: &mut Waiting) -> Option { + // The peer drops the route's claims, so nothing held back is owed. + let held = self.withheld.remove(&id); + for path in held.iter().flat_map(|(held, _)| held.keys()) { + unwait(waiting, id, path, key); + } + let holds = self.delivered.contains_key(&id); + let entry = self.entry(id); + // The reader holds the route once it reads what is pending: an end of that keeps + // any end before, an end of a start it never read is acked. + let ended = entry.ended; + let acked = !holds || ended; + let closing = match ended { + true => entry.closing.take(), + // Its floor certifies nothing if a claim was held back. + false => { + (entry.batch.take().filter(|_| holds)).map(|(merged, floor)| (merged, floor.filter(|_| held.is_none()))) + } + }; + *entry = Pending { + ended: holds, + closing, + ..Pending::default() + }; + if !holds && let Some((seq, _)) = self.pending.remove(&id) { + self.order.remove(&seq); + } + acked.then_some(id) + } + + /// Queue a batch of `id`'s claim changes, closed by `floor`, for the stream `key`, + /// holding back each start or restart `awaits` a confirmation. + fn batch( + &mut self, + key: Key, + id: Id, + (changes, floor): (Vec<(Path, Option)>, Option), + awaits: impl Fn(&Path, &Claim) -> Option, + waiting: &mut Waiting, + ) { + let ended = self.entry(id).ended; + let Self { + pending, + delivered, + withheld, + .. + } = self; + let (_, entry) = pending.get_mut(&id).expect("just queued"); + let sent = delivered.get(&id); + let (merged, last) = entry.batch.get_or_insert_default(); + for (path, claim) in changes { + // A newer change replaces one held back. + if let Some((held, _)) = withheld.get_mut(&id) + && held.remove(&path).is_some() + { + unwait(waiting, id, &path, key); + } + if let Some(item) = &claim + && awaits(&path, item) == Some(true) + { + // The reader keeps what it was sent until then. + merged.remove(&path); + waiting + .entry(id) + .or_default() + .entry(path.clone()) + .or_default() + .insert(key); + withheld.entry(id).or_default().0.insert(path, item.clone()); + continue; } + // The reader holds nothing here once the entry reaches it. + let unsent = ended || sent.is_none_or(|sent| !sent.contains(&path)); + match claim { + None if unsent => merged.remove(&path), + claim => merged.insert(path, claim), + }; + } + *last = floor; + if withheld.get(&id).is_some_and(|(held, _)| held.is_empty()) { + withheld.remove(&id); + } + } + + /// Queue the claim of `id` held back at `path`, now confirmed. + fn release(&mut self, id: Id, path: &Path) { + let Some((held, floor)) = self.withheld.get_mut(&id) else { + return; + }; + let Some(item) = held.remove(path) else { + return; + }; + let floor = *floor; + if held.is_empty() { + self.withheld.remove(&id); } + // A batch still queued closes with a newer floor. + let (merged, _) = self.entry(id).batch.get_or_insert_with(|| (Changes::new(), floor)); + merged.insert(path.clone(), Some(item)); } fn pop(&mut self) -> Option { @@ -304,6 +388,14 @@ impl Outbox { }); } if let Some((merged, floor)) = pending.batch { + // What is held back leaves the floor certifying nothing until it goes. + let floor = match self.withheld.get_mut(&id) { + Some((_, last)) => { + *last = floor; + None + } + None => floor, + }; let changes = ordered(merged); if let Some(sent) = self.delivered.get_mut(&id) { for (path, claim) in &changes { @@ -318,6 +410,68 @@ impl Outbox { } } +/// Whether a claim of `id` at `instance` and `epoch`, presented at `path`, waits before +/// it goes to other streams: on the shown claim there or, for a claim presented at a +/// narrower interest, the one covering it that serves the same instance, until a stream +/// serving it confirms that instance. `None` when none serves it here to wait on. +fn awaits(homes: &Homes, id: Id, path: &Path, instance: &Instance, epoch: u64) -> Option { + let same = |claim: &Claim| claim.instance == *instance && claim.epoch == epoch; + let home = ancestors(path).rev().find_map(|shown| { + let home = homes.get(&(id, shown))?; + home.keys.values().any(|slot| same(&slot.served.claim)).then_some(home) + })?; + let confirmed = + (home.keys.values()).any(|slot| (slot.confirmed.as_ref()).is_some_and(|(i, e)| i == instance && *e == epoch)); + Some(!confirmed) +} + +/// The stream `key` no longer holds back a claim of `id` at `path`. +fn unwait(waiting: &mut Waiting, id: Id, path: &Path, key: Key) { + let Some(paths) = waiting.get_mut(&id) else { + return; + }; + if let Some(keys) = paths.get_mut(path) { + keys.remove(&key); + if keys.is_empty() { + paths.remove(path); + } + } + if paths.is_empty() { + waiting.remove(&id); + } +} + +/// Send on what the streams hold back of `id` at or below each of `changed`, the shown +/// claims whose homes changed, once a home serving it confirmed it: only a home at or +/// above a path decides whether a claim there waits. One no home serves any more stays +/// until the router changes or ends it. +fn recheck(homes: &Homes, queues: &Queues, waiting: &mut Waiting, id: Id, changed: &[Path]) { + let Some(paths) = waiting.get(&id) else { + return; + }; + let mut held = BTreeSet::new(); + for claim in changed { + let at = paths.get_key_value(claim).into_iter(); + for (path, keys) in at.chain(below(paths, claim)) { + held.extend(keys.iter().map(|key| (path.clone(), *key))); + } + } + for (path, key) in held { + let Some(queue) = queues.get(&key) else { + continue; + }; + let mut outbox = queue.lock(); + let confirmed = (outbox.withheld.get(&id)) + .and_then(|(held, _)| held.get(&path)) + .and_then(|item| awaits(homes, id, &path, &item.instance, item.epoch)) + == Some(false); + if confirmed { + outbox.release(id, &path); + unwait(waiting, id, &path, key); + } + } +} + /// A batch's changes as the router orders them: what starts or changes, then what ends. fn ordered(changes: Changes) -> Vec<(Path, Option)> { let (ends, starts): (Vec<_>, Vec<_>) = changes.into_iter().partition(|(_, claim)| claim.is_none()); @@ -351,7 +505,9 @@ struct State { next_key: Key, queues: Queues, /// Each shown claim of another identity, and the streams serving it. - homes: BTreeMap<(Id, Path), Home>, + homes: Homes, + /// What the streams hold back, by the confirmation it waits on. + waiting: Waiting, /// The usable feasible routes each identity's claims were last served through, /// with the name each stream's peer gives it: while they stay the same, only a /// claim whose shown state changed is served differently. @@ -366,6 +522,9 @@ struct State { origins: HashSet, } +/// Each shown claim of another identity, and the streams serving it. +type Homes = BTreeMap<(Id, Path), Home>; + /// An identity's usable feasible routes: each stream, its metric and RTT, and the /// peer's name. type Routes = Vec<(Key, Metric, Option, Id)>; @@ -381,7 +540,8 @@ struct Slot { served: Served, /// The identity as the stream's peer names it, kept while no route to it is left. named: Id, - confirmed: bool, + /// The instance and epoch the stream last confirmed serving. + confirmed: Option<(Instance, u64)>, /// No longer a route to the claim's identity: let go once a wanted stream confirms. leaving: bool, } @@ -389,7 +549,7 @@ struct Slot { impl Home { /// Unserve every leaving stream once a wanted one serves the claim. fn release(&mut self, queues: &Queues, id: Id, claim: &Path) { - if !self.keys.values().any(|slot| !slot.leaving && slot.confirmed) { + if !self.keys.values().any(|slot| !slot.leaving && slot.confirmed.is_some()) { return; } self.keys.retain(|key, slot| { @@ -425,6 +585,7 @@ impl Node { next_key: 1, queues: HashMap::new(), homes: BTreeMap::new(), + waiting: HashMap::new(), routes: HashMap::new(), links: HashSet::new(), origins: HashSet::new(), @@ -632,29 +793,41 @@ impl Stream { self.with(|router, key| router.rtt(key, rtt)); } - /// This stream's session now serves `claim` of `id`, so a stream that no longer - /// carries a route to it lets go. - pub fn serving(&self, id: Id, claim: &Path) { + /// This stream's session now serves `claim` of `id` at `served`'s instance, so the + /// other streams are sent that instance, and a stream that no longer carries a route + /// to it lets go. + pub fn serving(&self, id: Id, claim: &Path, served: &Claim) { let mut state = self.node.lock(); - let State { homes, queues, .. } = &mut *state; + let State { + homes, queues, waiting, .. + } = &mut *state; let Some(home) = homes.get_mut(&(id, claim.clone())) else { return; }; let Some(slot) = home.keys.get_mut(&self.key) else { return; }; - if slot.confirmed { + let confirmed = (served.instance.clone(), served.epoch); + if slot.confirmed.as_ref() == Some(&confirmed) { return; } - slot.confirmed = true; + slot.confirmed = Some(confirmed.clone()); home.release(queues, id, claim); + recheck(homes, queues, waiting, id, std::slice::from_ref(claim)); } } impl Drop for Stream { fn drop(&mut self) { let mut state = self.node.lock(); - state.queues.remove(&self.key); + let state = &mut *state; + if let Some(queue) = state.queues.remove(&self.key) { + for (id, (held, _)) in &queue.read().withheld { + for path in held.keys() { + unwait(&mut state.waiting, *id, path, self.key); + } + } + } state.router.disconnect(self.key); // Its session is gone, and what it served with it. for home in state.homes.values_mut() { @@ -668,26 +841,46 @@ impl Drop for Stream { } } -/// Queue `output` for the stream `key`. Returns the identity of a ROUTE_END its -/// reader will never read, whose echo the router is owed. -fn push(queues: &Queues, key: Key, output: Output) -> Option { - queues.get(&key).and_then(|queue| queue.lock().push(output)) +/// Queue `output` for the stream `key`. +fn push(queues: &Queues, key: Key, output: Output) { + if let Some(queue) = queues.get(&key) { + queue.lock().push(output); + } } impl State { - /// Route the router's messages and shown claims to the streams. + /// Route the router's messages and shown claims to the streams. Claims are served + /// before the batches carrying them go, so a batch knows what is not served yet. fn flush(&mut self) { loop { + let outs = self.router.drain(); + self.serve(); let mut acked = Vec::new(); - for out in self.router.drain() { + for out in outs { let (key, output) = match out { + Out::Route { to, id, reach: None } => { + if let Some(queue) = self.queues.get(&to) + && let Some(id) = queue.lock().end(to, id, &mut self.waiting) + { + acked.push((to, id)); + } + continue; + } Out::Route { to, id, reach } => { - if reach.is_some() && Some(id) == self.router.declared() && self.origins.contains(&to) { + if Some(id) == self.router.declared() && self.origins.contains(&to) { self.links.insert(to); } (to, Output::Route { id, reach }) } - Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), + Out::Batch { to, id, changes, floor } => { + if let Some(queue) = self.queues.get(&to) { + let homes = &self.homes; + let awaits = + |path: &Path, item: &Claim| awaits(homes, id, path, &item.instance, item.epoch); + queue.lock().batch(to, id, (changes, floor), awaits, &mut self.waiting); + } + continue; + } Out::Request { to, id, seqno } => { if let Some(queue) = self.queues.get(&to) { let mut outbox = queue.lock(); @@ -698,9 +891,7 @@ impl State { } Out::Echo { to, id } => (to, Output::Echo { id }), }; - if let Some(id) = push(&self.queues, key, output) { - acked.push((key, id)); - } + push(&self.queues, key, output); } if acked.is_empty() { break; @@ -710,6 +901,10 @@ impl State { self.router.echo(key, id); } } + } + + /// Serve the shown claims that changed through the streams. + fn serve(&mut self) { // The shown claims are read in place; the events only say which changed, and // whether a claim that went was ended by its origin or let go here. let mut ended = BTreeMap::new(); @@ -749,6 +944,7 @@ impl State { claims } }; + let changed: Vec = claims.iter().cloned().collect(); let (shown, gone): (Vec, Vec) = (claims.into_iter()).partition(|claim| self.router.shown().contains_key(&(id, claim.clone()))); // Most specific first: a job continuing its claim's instance is served before @@ -795,7 +991,7 @@ impl State { let slot = Slot { served: served.clone(), named, - confirmed: false, + confirmed: None, leaving: false, }; home.keys.insert(key, slot); @@ -828,6 +1024,8 @@ impl State { ); } } + // A claim held back may now wait on another home, or on none. + recheck(&self.homes, &self.queues, &mut self.waiting, id, &changed); #[cfg(test)] self.check(id, &routes); // Nothing left to recover: let the identity go. @@ -1017,8 +1215,8 @@ mod tests { }; assert_eq!(metric(outputs(&far)), Some(3)); assert_eq!(metric(outputs(&near)), Some(1)); - far.serving(1, &p("live/x")); - near.serving(1, &p("live/x")); + far.serving(1, &p("live/x"), &claim(&instance, 0)); + near.serving(1, &p("live/x"), &claim(&instance, 0)); // A stream whose route ends lets go, since the other still serves it. far.route(1, None); @@ -1353,6 +1551,123 @@ mod tests { assert_eq!(read.len(), 2, "the route, then its claim"); } + /// A relay sends another stream a claim only once a stream serving it confirmed + /// its instance, so a request naming that instance always finds a route: first when + /// the claim starts, again when it restarts at a new instance, and also where a + /// narrower interest is shown the claim covering it. + #[test] + fn a_claim_goes_on_only_once_served() { + let (old, new) = (Instance::random(), Instance::random()); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let edge = relay.answer(Link::default(), Path::empty()).unwrap(); + let narrow = relay.answer(Link::default(), p("live/x")).unwrap(); + let both = || [instances(&edge), instances(&narrow)]; + + origin.own(p("live"), Some(claim(&old, 0))); + pump(&answer, &receive); + assert_eq!(both(), [[(vec![], None)], [(vec![], None)]], "held until served"); + receive.serving(1, &p("live"), &claim(&old, 0)); + assert_eq!( + both(), + [ + [(vec![(p("live"), Some(old.clone()))], Some(1))], + [(vec![(p("live/x"), Some(old.clone()))], Some(1))] + ] + ); + + origin.own(p("live"), Some(claim(&new, 0))); + pump(&answer, &receive); + assert_eq!( + both(), + [[(vec![], None)], [(vec![], None)]], + "the old instance stays until the new one is served, the floor certifying nothing" + ); + receive.serving(1, &p("live"), &claim(&old, 0)); + assert_eq!(both(), [[], []], "a stale confirmation releases nothing"); + receive.serving(1, &p("live"), &claim(&new, 0)); + assert_eq!( + both(), + [ + [(vec![(p("live"), Some(new.clone()))], Some(2))], + [(vec![(p("live/x"), Some(new.clone()))], Some(2))] + ] + ); + + // A route that ends with a claim held back certifies nothing on its way out. + origin.own(p("live"), Some(claim(&old, 1))); + pump(&answer, &receive); + receive.route(1, None); + assert!(matches!( + &outputs(&edge)[..], + [Output::Batch { floor: None, .. }, Output::Route { reach: None, .. }] + )); + } + + /// A claim presented at a narrower interest waits on the shown claim covering it, + /// whichever that is by the time it is confirmed: a confirmed one that covers it + /// again once the claim between goes, and one that moves to a narrower path before + /// it is confirmed. + #[test] + fn a_held_claim_follows_the_claim_covering_it() { + let (first, second) = (Instance::random(), Instance::random()); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&first, 0))); + pump(&answer, &receive); + receive.serving(1, &p("live"), &claim(&first, 0)); + origin.own(p("live/x"), Some(claim(&first, 0))); + pump(&answer, &receive); + let narrow = relay.answer(Link::default(), p("live/x/y")).unwrap(); + assert_eq!(instances(&narrow), [(vec![], None)], "held on the unconfirmed live/x"); + origin.own(p("live/x"), None); + pump(&answer, &receive); + assert_eq!( + instances(&narrow), + [(vec![(p("live/x/y"), Some(first.clone()))], Some(3))], + "covered by the confirmed live again" + ); + + origin.own(p("live"), Some(claim(&second, 0))); + pump(&answer, &receive); + assert_eq!(instances(&narrow), [(vec![], None)]); + origin.own(p("live/x"), Some(claim(&second, 0))); + origin.own(p("live"), None); + pump(&answer, &receive); + assert_eq!( + instances(&narrow), + [(vec![], None)], + "now held on the unconfirmed live/x" + ); + receive.serving(1, &p("live/x"), &claim(&second, 0)); + assert_eq!( + instances(&narrow), + [(vec![(p("live/x/y"), Some(second.clone()))], Some(6))] + ); + } + + /// A batch's changes, each claim by its instance, and its floor. + type Instances = (Vec<(Path, Option)>, Option); + + /// The batches for identity 1 a stream would send. + fn instances(stream: &Stream) -> Vec { + (outputs(stream).into_iter()) + .filter_map(|output| match output { + Output::Batch { id: 1, changes, floor } => { + let changes = (changes.into_iter()) + .map(|(path, claim)| (path, claim.map(|claim| claim.instance))) + .collect(); + Some((changes, floor)) + } + _ => None, + }) + .collect() + } + /// The seqnos of the finite ROUTE messages for `id` a stream would send. fn routes(stream: &Stream, id: Id) -> Vec { (outputs(stream).into_iter()) @@ -1459,7 +1774,7 @@ mod tests { pump(&second_answer, &second); outputs(&first); assert!(matches!(served(outputs(&second))[..], [.., Output::Serve { .. }])); - second.serving(1, &p("live/x")); + second.serving(1, &p("live/x"), &claim(&instance, 0)); drop(first); assert!(served(outputs(&second)).is_empty(), "already served where it still is"); @@ -1481,7 +1796,7 @@ mod tests { receive.route(1, Some(Reach::Finite(advert))); receive.batch(1, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); assert!(matches!(served(outputs(&receive))[..], [Output::Serve { .. }])); - receive.serving(1, &p("live/x")); + receive.serving(1, &p("live/x"), &claim(&instance, 0)); receive.route(1, Some(Reach::Held)); assert!(matches!(