diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 6b922e590d2..7b52fae52f6 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -143,7 +143,9 @@ Routing prefers the longest covering prefix, then a fully identified hop list over one holding an anonymous hop (0), then the lowest cost, then the shortest hop list. Remaining ties hash the requested path, so equal-cost advertisers of one prefix, such as a transcode pool, split its paths, and every relay picks -the same one for a given path. +the same one for a given path. A publisher connected straight to the relay is +not hashed at its own path: its newest session wins, so a publisher that +restarts takes over from its lingering old session at once. ```toml [cluster] diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 08890fd1b7e..fc0c768f7e5 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -488,6 +488,7 @@ When resolving a path covered by several routes (across any number of streams), The Spread Hash is the 64-bit FNV-1a hash, with offset basis `0x420C0DECB00B` and the standard FNV-64 prime, of the requested path's UTF-8 bytes followed by each Hop ID of the route's path, oldest first, as 8 little-endian bytes. It is keyed on the requested path rather than the route's prefix, so equal-cost advertisers of one prefix share its paths instead of the first one taking them all, while one path resolves to the same advertiser on every relay that holds the same routes. When choosing which route to advertise for a prefix, the requested path is the prefix itself. +A relay skips the Spread Hash for a route a publisher announced to it directly at the requested path, which no other relay ranks, so a publisher reconnecting under a new Hop ID wins at once. Routes with the same Epoch serve the same Broadcast, so a relay MAY move a live subscription between them, continuing from the first frame the subscriber lacks, and a route change (a reconnect, a cheaper path, a draining session) is invisible to the subscriber. A relay MUST NOT move a subscription to a route with a different Epoch, or between routes without one: it stays on its route and ends with it. @@ -1818,6 +1819,7 @@ The `Message Length` describes the payload size on the wire. - SUBSCRIBE_DROP's Error Code is a stream error code, which ends the dropped groups' readers, including one still arriving. In lite-05 and lite-06 it is application-specific, so a subscriber there reads every drop as NOT_FOUND. - SUBSCRIBE_END declares the final boundary while lower groups can still be in flight; the Subscribe Stream FIN follows once every Group Stream has finished or been reset. - Added the Spread Hash tie-break after the shortest path: a hash of the requested path and the route's Hop IDs, so equal-cost advertisers of one prefix share its paths. +- Skipped the Spread Hash for a route a publisher announced directly at the requested path. - Added a local link preference to route selection, after the newest Epoch and before the 0 Hop ID check: the higher preference a subscriber assigns to the arriving link wins, and an unassigned link is neutral. Not carried on the wire. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live advertisement's suffix, and ANNOUNCE_START and ANNOUNCE_UPDATE gain `Hop Base` and `Hop Keep` to copy the tail of a live advertisement's Hop ID list. - Capped the Message Length of every message except FRAME at 65,535 bytes. diff --git a/js/net/src/consume.ts b/js/net/src/consume.ts index d03828a6e12..d216712a534 100644 --- a/js/net/src/consume.ts +++ b/js/net/src/consume.ts @@ -69,7 +69,7 @@ export class BroadcastCache { if (!entries) continue; let best: { identity: object; route: Route } | undefined; for (const entry of entries.values()) { - if (!best || compareRouteCandidates(path, entry.route, best.route) < 0) best = entry; + if (!best || compareRouteCandidates(path, prefix, entry.route, best.route) < 0) best = entry; } if (best) { const cached = this.#cache.get(path)?.get(best.route.epoch ?? best.identity); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 778c9720170..d3b3ba5a5ab 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -810,10 +810,16 @@ export function compareRoutes(a: Route, b: Route): number { ); } -/** Order same-prefix routes by preference, hop count, and the origin's stable spread hash. */ -export function compareRouteCandidates(path: Path.Valid, a: Route, b: Route): number { +/** + * Order routes at `prefix` resolving `path` by preference, hop count, and the stable spread + * hash. The hash only spreads a prefix's paths: at the prefix itself a tie is left for the + * newest to win, so a publisher that reconnects under a new hop takes over from its lingering + * session at once, matching `route_order` in rs/moq-net for routes a node did not get from a + * cluster peer, which is every route here. + */ +export function compareRouteCandidates(path: Path.Valid, prefix: Path.Valid, a: Route, b: Route): number { const order = compareRoutes(a, b) || a.hops.length - b.hops.length; - if (order !== 0) return order; + if (order !== 0 || path === prefix) return order; const ha = spreadHash(path, a.hops); const hb = spreadHash(path, b.hops); return ha < hb ? -1 : ha > hb ? 1 : 0; diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 23ffd232ab3..e266ffa6733 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -2000,16 +2000,32 @@ test("an equal-cost pool spreads its paths the same way on every node", async () } }); -test("an equal-cost pool advertises the same member whatever order it arrived in", async () => { - const prefix = Path.from("pool"); - const workers = [10n, 11n].map((id) => HopSchema.parse(id)); - const expected = spreadHash(prefix, [workers[0]]) < spreadHash(prefix, [workers[1]]) ? workers[0] : workers[1]; - - for (const order of [workers, [...workers].reverse()]) { +// A publisher that reconnects under a new hop while its old session lingers takes over at once, +// whichever hop hashes lower: only a prefix's paths are spread by the hash. +test("an equal-cost replacement at its own path wins whatever its hop", async () => { + const path = Path.from("pool/job-0"); + for (const order of [ + [11n, 10n], + [10n, 11n], + ].map((hops) => hops.map((id) => HopSchema.parse(id)))) { const origin = new Producer(); - const handles = order.map((hop) => wireOf(origin).receive(prefix, { hops: [hop], cost: 3n })); - await new Promise((resolve) => setImmediate(resolve)); - expect((await announcedView(origin.consume())).peek().get(prefix)?.hops).toEqual([expected]); + const served: bigint[] = []; + const handles = order.map((hop) => { + const handle = wireOf(origin).receive(path, { hops: [hop], cost: 3n }); + void (async () => { + for await (const request of handle.requested()) { + served.push(hop); + request.accept(new BroadcastProducer()); + } + })(); + return handle; + }); + await settle(); + expect((await announcedView(origin.consume())).peek().get(path)?.hops).toEqual([order[1]]); + const request = origin.request(path); + await settle(); + expect(served).toEqual([order[1]]); + request.close(); for (const handle of handles) handle.close(); origin.close(); } diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index c42bbc876bb..8606a3a4c4e 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -26,7 +26,6 @@ import { scopeCaptures, scopeHead, scopeOverlaps, - spreadHash, } from "./internal.ts"; import * as Path from "./path.ts"; import { Front } from "./resume.ts"; @@ -315,19 +314,17 @@ interface Candidate extends Advertised { } /** - * Orders advertisements at `prefix`: the better route, then a local broadcast on a tie, then - * fewer hops, then the lower {@link spreadHash} of the prefix, matching `route_order` in rs/moq-net. + * Orders advertisements at a prefix: the better route, then a local broadcast on a tie, then + * fewer hops. A remaining tie is left to the table's newest-first order; see + * {@link compareRouteCandidates}. */ -function compareCandidates(prefix: Path.Valid, a: Candidate, b: Candidate): number { - const order = +function compareCandidates(a: Candidate, b: Candidate): number { + return ( compareLinks(a.route, b.route, a.preference, b.preference) || compareRoutes(a.route, b.route) || Number(b.exact) - Number(a.exact) || - a.route.hops.length - b.route.hops.length; - if (order !== 0) return order; - const ha = spreadHash(prefix, a.route.hops); - const hb = spreadHash(prefix, b.route.hops); - return ha < hb ? -1 : ha > hb ? 1 : 0; + a.route.hops.length - b.route.hops.length + ); } /** @@ -340,12 +337,12 @@ function compareLinks(a: Route, b: Route, aPreference: Preference, bPreference: } /** - * The preferred of `entries` (newest first) for resolving `path`, not skipped: the best route, - * then fewest hops, then the lowest {@link spreadHash}, then newest. `path` is the requested - * path for a request, or the prefix itself for an advertisement. + * The preferred of `entries` (newest first, at `prefix`) for resolving `path`, not skipped: + * see {@link compareRouteCandidates}, then newest. */ function preferredEntry( path: Path.Valid, + prefix: Path.Valid, entries: readonly RouteEntry[], skip?: (entry: RouteEntry) => boolean, ): RouteEntry | undefined { @@ -358,7 +355,8 @@ function preferredEntry( } const a = entry.route.peek(); const b = best.route.peek(); - const order = compareLinks(a, b, entry.preference, best.preference) || compareRouteCandidates(path, a, b); + const order = + compareLinks(a, b, entry.preference, best.preference) || compareRouteCandidates(path, prefix, a, b); if (order < 0) best = entry; } return best; @@ -577,9 +575,9 @@ class OriginState { if (list) list.push(local); else out.set(path, [local]); } - for (const [prefix, list] of out) { + for (const list of out.values()) { // Stable, so equal routes keep the table's newest-first order. - if (list.length > 1) list.sort((a, b) => compareCandidates(prefix, a, b)); + if (list.length > 1) list.sort(compareCandidates); } return out; } @@ -1012,6 +1010,7 @@ class OriginState { if (!entries) continue; const entry = preferredEntry( path, + prefix, entries, (candidate) => !candidate.scope.matches(path) || (skip?.(candidate) ?? false), ); diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index defd571a6d6..eec2d1f3967 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -15,10 +15,11 @@ forwards the `Restart`. A restart is a new epoch at the same path: the newest epoch wins new requests and announce consumers see that `Restart`, so viewers re-request rather than stall on a replaced broadcast. Subscriptions already on the old one stay until the -application drops them or its route goes. Without an epoch, a restarted -publisher on the same hop chain as its lingering old session wins at once -(the newest announcement breaks the tie), but one on a different chain of -the same length and cost that loses the routing hash is reached only once +application drops them or its route goes. Without an epoch, a publisher +that restarts on a relay while its old session lingers there wins at once, +whatever its new hop: the relay ranks its own sessions' routes newest first +and hashes only peer routes. Through a peer, a restart on a different chain +of the same length and cost that loses the routing hash is reached only once the old session closes and its route is withdrawn. The epoch rides moq-lite 07 announcements and requests as metadata, so the diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 0b4a3aa7909..2662f85fc38 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3133,7 +3133,7 @@ mod announce_test { for version in [Version::Lite06, Version::Lite07] { let mut h = harness_on(version).await; let route = crate::origin::Route::default().with_hops(pub_hops()).with_cost(7); - let peer = h.origin.clone().peer().announce("cam", route.clone()).unwrap(); + let newer = h.origin.announce("cam", route.clone()).unwrap(); // A restart waits out the update hold like an update. moq_clock::sleep(crate::origin::DEFAULT_UPDATE_HOLD).await; settle().await; @@ -3164,7 +3164,7 @@ mod announce_test { }; restarted(h.wire.take_announces()); - drop(peer); + drop(newer); moq_clock::sleep(crate::origin::DEFAULT_UPDATE_HOLD).await; settle().await; restarted(h.wire.take_announces()); @@ -3224,11 +3224,11 @@ mod announce_test { // Same chain and cost, so only the instance changed. Both land before the loop polls. let route = crate::origin::Route::default().with_hops(pub_hops()).with_cost(7); let epoch = (!anonymous).then(crate::Epoch::mint); - let _newer = match &epoch { - Some(epoch) => h.origin.announce("cam", route.with_epoch(epoch.clone())), - None => h.origin.clone().peer().announce("cam", route), - } - .unwrap(); + let route = match &epoch { + Some(epoch) => route.with_epoch(epoch.clone()), + None => route, + }; + let _newer = h.origin.announce("cam", route).unwrap(); auth.authorize(&grant(&["cam", "other"])); moq_clock::sleep(crate::origin::DEFAULT_UPDATE_HOLD).await; settle().await; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index d59a83a9456..c438b19adb3 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -461,8 +461,9 @@ pub struct Route { /// What pulling content via this route costs, accumulated per link: lower wins /// among routes of the same anonymity, with ties broken by a broadcast published - /// on this origin, then hop length, then a deterministic hash, and finally the - /// most recently announced route. See [`Cost`]. + /// on this origin, then hop length, then a deterministic hash (except for a route + /// that entered here, at its own path), and finally the most recently announced + /// route. See [`Cost`]. pub cost: Cost, /// The credential the request advertising this route carries, from @@ -618,13 +619,18 @@ fn fnv_key(name: &str, origins: impl IntoIterator) -> u64 { /// cost, then a broadcast published on this origin (it serves what is here, not a /// claim that has to ask), then the shortest hop chain, then a deterministic hash /// of `path` and the chain so every node converges on the same winner, and finally -/// the newest announcement, so a reconnect under an otherwise identical route wins -/// the moment it lands instead of after the transport retires the old session. +/// the newest announcement, so a reconnect wins the moment it lands instead of after +/// the transport retires the old session. /// /// `path` is what is being resolved: the requested path for a request, the prefix /// itself for an advertisement. Keying the hash on the requested path is what /// spreads equal-cost advertisers of one prefix: each path picks its own winner /// from the pool, rather than every path under the prefix hashing alike. +/// +/// A route that entered here for exactly `path` is not hashed: no other node ranks +/// it, so convergence does not need the hash, and a restarted publisher announces +/// under a new first hop, so a hash would keep its dead session winning half the +/// time until the transport times out. #[allow(clippy::type_complexity)] fn route_order<'a>( path: &Path, @@ -646,7 +652,10 @@ fn route_order<'a>( entry.cost, !entry.local, entry.hops.len(), - fnv_key(path.as_str(), entry.hops.iter().copied()), + match entry.link == Link::Local && entry.prefix == *path { + true => 0, + false => fnv_key(path.as_str(), entry.hops.iter().copied()), + }, Reverse(entry.id), ) } @@ -7999,13 +8008,13 @@ mod tests { } } - /// An epochless replacement at equal cost and chain length on a chain whose hash - /// loses competes with the lingering old route: nothing restarts and requests stay - /// on the old route until it is withdrawn, then consumers restart and a re-request - /// lands on the replacement. + /// An epochless replacement from a cluster peer at equal cost and chain length, on a + /// chain whose hash loses, competes with the lingering old route: nothing restarts and + /// requests stay on the old route until it is withdrawn, then consumers restart and a + /// re-request lands on the replacement. Every node ranks peer routes alike. #[moq_clock::test] #[cfg(feature = "async")] - async fn an_equal_cost_replacement_waits_for_the_old_route() { + async fn an_equal_cost_peer_replacement_waits_for_the_old_route() { // Pinned in `spread_hash_matches_js`: hop 11 hashes below hop 10 for this path. // Restarts are delivered at once: the hold is not what this checks. let producer = Config { @@ -8015,12 +8024,13 @@ mod tests { .produce(); let consumer = producer.consume(); let mut announced = consumer.announced(); - let old = producer + let peer = producer.clone().peer(); + let old = peer .dynamic_route("pool/job-0", Route::default().with_hops(hops(&[11]))) .unwrap(); announced.expect_next_active("pool/job-0").await; - let new = producer + let new = peer .dynamic_route("pool/job-0", Route::default().with_hops(hops(&[10]))) .unwrap(); announced.assert_next_wait(); @@ -8043,6 +8053,39 @@ mod tests { assert!(!replaced.is_clone(&resolved)); } + /// A publisher that restarts while its old session lingers announces its own path under + /// a new first hop. The restart takes over at once whichever chain hashes lower, so new + /// requests never wait on the old session. + #[moq_clock::test] + #[cfg(feature = "async")] + async fn an_equal_cost_local_replacement_takes_over_at_once() { + for (old_hop, new_hop) in [(11, 10), (10, 11)] { + let producer = Config { + update_hold: Duration::ZERO, + ..Config::new(origin(1)) + } + .produce(); + let consumer = producer.consume(); + let mut announced = consumer.announced(); + let _old = producer + .dynamic_route("pool/job-0", Route::default().with_hops(hops(&[old_hop]))) + .unwrap(); + announced.expect_next_active("pool/job-0").await; + + let new = producer + .dynamic_route("pool/job-0", Route::default().with_hops(hops(&[new_hop]))) + .unwrap(); + assert_eq!( + announced.expect_next_restarted("pool/job-0").await.hops, + hops(&[new_hop]) + ); + + let pending = consumer.request_broadcast("pool/job-0", None); + queued(&new).await.accept(broadcast::Info::new().produce()); + pending.await.expect("the replacement resolves"); + } + } + /// An origin delivering restarts at once, and a way to claim `prefix` with a /// producer scoped to `scope`, from `hop` at `cost`. #[cfg(feature = "async")] diff --git a/rs/moq-net/tests/integration/replacement.rs b/rs/moq-net/tests/integration/replacement.rs index 12e851fc2a4..98c33b1bc87 100644 --- a/rs/moq-net/tests/integration/replacement.rs +++ b/rs/moq-net/tests/integration/replacement.rs @@ -268,3 +268,49 @@ async fn route_death_past_hold_errors() { } } } + +/// A publisher restarts as a new process (a new hop) while its old session lingers, +/// dead but not yet idled out. The restart wins at once, whichever chain hashes lower, +/// so a viewer's first request reaches it instead of waiting on the dead session. +async fn restart_while_the_old_session_lingers(version: &str, old_hop: u64, new_hop: u64) { + let version: Version = version.parse().unwrap(); + let viewer = produce_origin(3); + + let old = produce_origin(old_hop); + let old_cam = old.publish("cam", origin::Route::default()).unwrap(); + let _old_video = old_cam.create_track("video", timed()).unwrap(); + let old_session = link(version, &old, &viewer).await; + let mut announced = viewer.consume().announced(); + next_event(&mut announced).await; + + // The old process dies without a close: its session falls silent. + old_session.client_transport.set_latency(Duration::from_secs(3600)); + old_session.server_transport.set_latency(Duration::from_secs(3600)); + + let new = produce_origin(new_hop); + let new_cam = new.publish("cam", origin::Route::default()).unwrap(); + let new_video = new_cam.create_track("video", timed()).unwrap(); + let _new_session = link(version, &new, &viewer).await; + moq_clock::sleep(PAST_HOLD).await; + + let cam = viewer.consume().request_broadcast("cam", None).await.unwrap(); + let mut video = subscribe(&cam, "video") + .await + .unwrap_or_else(|err| panic!("{version} ({old_hop} -> {new_hop}): {err}")); + write(&new_video, "new:0"); + assert_eq!(read(&mut video).await, "new:0", "{version} ({old_hop} -> {new_hop})"); +} + +#[moq_clock::test] +async fn restart_while_the_old_session_lingers_wins() { + for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { + for (old_hop, new_hop) in [(1, 4), (4, 1)] { + moq_clock::timeout( + TEST_TIMEOUT, + restart_while_the_old_session_lingers(version, old_hop, new_hop), + ) + .await + .expect("timed out"); + } + } +}