From 5c22634a668e81e0e468eb1f46f3e8b3ab886edc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 13:47:30 -0800 Subject: [PATCH 01/16] chore(quest): claim quest/m1/refuse-route.md Quest-Claim: b79bfac48452360c From 9681b3a99c38000ef29f566c1ee913d92b60659d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:03:55 -0800 Subject: [PATCH 02/16] wip(net): a refused forwarded route withdraws the route, not the session Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/fixtures/lite.json | 44 ++- rs/moq-net/src/fuzz.rs | 9 +- rs/moq-net/src/lite/announce.rs | 397 +++++++++++++++++------ rs/moq-net/src/lite/compress.rs | 134 +++++--- rs/moq-net/src/lite/corpus.rs | 13 +- rs/moq-net/src/lite/instance.rs | 33 +- rs/moq-net/src/lite/subscriber.rs | 8 +- rs/moq-net/src/lite/subscriber/routed.rs | 252 ++++++++++---- rs/moq-net/src/stats.rs | 21 ++ 9 files changed, 671 insertions(+), 240 deletions(-) diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 87ad0e9b28a..6e97f943b2d 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -63,18 +63,18 @@ "expect": "refused" }, { - "name": "a ROUTE_START flag no version defines is refused", + "name": "a ROUTE_START flag no version defines is withdrawn", "version": "moq-lite-07-wip", "message": "announce", "hex": "06059234030504", - "expect": "refused" + "expect": "withdrawn" }, { - "name": "a ROUTE_START with more than 32 hops is refused", + "name": "a ROUTE_START with more than 32 hops is withdrawn", "version": "moq-lite-07-wip", "message": "announce", "hex": "0606923403050121", - "expect": "refused" + "expect": "withdrawn" }, { "name": "a ROUTE_START missing a hop it counts is refused", @@ -84,18 +84,18 @@ "expect": "refused" }, { - "name": "a ROUTE_START naming Node ID 0 is refused", + "name": "a ROUTE_START naming Node ID 0 is withdrawn", "version": "moq-lite-07-wip", "message": "announce", "hex": "06050003050100", - "expect": "refused" + "expect": "withdrawn" }, { - "name": "a ROUTE_START advertising the last seqno is refused", + "name": "a ROUTE_START advertising the last seqno is withdrawn", "version": "moq-lite-07-wip", "message": "announce", "hex": "060d01ffffffffffffffffff000000", - "expect": "refused" + "expect": "withdrawn" }, { "name": "a ROUTE_UPDATE references its Route ID", @@ -139,6 +139,34 @@ "hex": "000a00000178020161000100", "expect": "decoded" }, + { + "name": "a ROUTE_UPDATE flag no version defines is withdrawn", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "0705010305802a", + "expect": "withdrawn" + }, + { + "name": "a lite-07 ANNOUNCE_START with an invalid path is withdrawn", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "000c0000032f2f78020161000100", + "expect": "withdrawn" + }, + { + "name": "a lite-07 ANNOUNCE_START with an invalid instance is withdrawn", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "0009000001780200000100", + "expect": "withdrawn" + }, + { + "name": "a lite-07 ANNOUNCE_START cut short is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "0003000003", + "expect": "refused" + }, { "name": "a ROUTE_REQUEST names an origin node and seqno", "version": "moq-lite-07-wip", diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index c4c58e9722a..d1829b10df4 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -972,8 +972,8 @@ pub fn encode_announces(announced: &[Announced], compress: bool) -> Vec { } /// Decode and resolve an announce stream, as [`encode_announces`] writes it, stopping -/// at the first message that fails to decode or resolve: a subscriber closes the -/// session there. +/// at the first message that fails to decode or resolve, where a subscriber closes the +/// session, or that it refuses. pub fn decode_announces(data: &[u8], compress: bool) -> Vec { let version = announce_version(compress); let mut data = Decoder::new(data, version.into()); @@ -982,8 +982,11 @@ pub fn decode_announces(data: &[u8], compress: bool) -> Vec { while let Ok(msg) = lite::AnnounceBroadcast::decode(&mut data, version) { let announced = match msg { lite::AnnounceBroadcast::Active { suffix, hops, .. } => decoder - .start(suffix) + .start(Some(suffix)) + .and_then(|suffix| suffix.ok_or(crate::Error::ProtocolViolation)) .map(|suffix| Announced::Start(suffix, hops.into_ids())), + // A refused message assigns ids this harness does not model. + lite::AnnounceBroadcast::Refused { .. } => break, lite::AnnounceBroadcast::Update { id, hops, .. } => { decoder.get(id).map(|_| Announced::Update(id, hops.into_ids())) } diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 0405a720205..061f6556a11 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -131,6 +131,60 @@ pub enum AnnounceBroadcast { WarmUpdate { id: u64, warm: Warm }, /// WARM_END (lite-07+): the sender withdrew the warm route with this id. WarmEnd { id: u64 }, + /// A route message (lite-07+) whose content, which the sender only forwarded, this + /// node refuses. The receiver withdraws what it names and keeps the session. Decoded + /// only as far as keeps the stream's ids in step; never sent. + Refused { message: Refusable, reason: Refusal }, +} + +/// What a refused route message still tells the receiver, so the ids that follow it +/// resolve. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Refusable { + /// ROUTE_START: assigns the next Route ID to the origin node `node`. + RouteStart { node: route::Id }, + /// ROUTE_UPDATE of the route with this Route ID. + RouteUpdate { id: u64 }, + /// ANNOUNCE_START under the route with Route ID `route`: assigns the next Announce + /// ID, to `suffix`, or to no path when the path itself was refused. + Start { route: u64, suffix: Option }, + /// ANNOUNCE_UPDATE or ANNOUNCE_RESTART of the claim with this Announce ID. + Change { id: u64 }, + /// WARM_START under the route with Route ID `route`: assigns the next Warm ID. + WarmStart { route: u64 }, + /// WARM_UPDATE of the warm route with this id. + WarmUpdate { id: u64 }, +} + +/// Why a node refuses content a peer forwarded: well framed, but nothing it can use. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub(crate) enum Refusal { + #[error("the reserved origin node")] + Node, + #[error("the reserved seqno")] + Seqno, + #[error("unknown flags {0:#04x}")] + Flags(u8), + #[error("{0} hops, past the limit")] + Hops(u64), + #[error("a carrier not among its hops")] + Carrier, + #[error("an invalid path")] + Path, + #[error("an invalid instance")] + Instance, + #[error("an invalid lite-06 chain")] + Chain, +} + +/// Content a peer forwarded: `Ok(Err(..))` when it is well framed but refused. +type Content = Result, DecodeError>; + +/// Refuse the content at `r`, skipping the rest of its message's body: the refused field +/// is always the body's last, or leaves what follows unreadable. +fn refuse(r: &mut Decoder<'_>, reason: Refusal) -> Content { + r.rest(); + Ok(Err(reason)) } /// A copy of a broadcast a warm route offers. @@ -161,18 +215,30 @@ fn encode_warm(w: &mut Encoder<'_>, warm: &Warm) -> Result<(), EncodeError> { Ok(()) } -fn decode_warm(r: &mut Decoder<'_>) -> Result { +fn decode_warm(r: &mut Decoder<'_>) -> Content { let cost = r.varint()?; let carrier = r.varint()?; let count = r.varint()?; if count > route::MAX_HOPS as u64 { - return Err(DecodeError::TooMany); + return refuse(r, Refusal::Hops(count)); } if carrier >= count { - return Err(DecodeError::InvalidValue); + return refuse(r, Refusal::Carrier); } let hops = (0..count).map(|_| r.varint()).collect::>()?; - Ok(Warm { cost, carrier, hops }) + Ok(Ok(Warm { cost, carrier, hops })) +} + +/// Decode a WARM_START's body after its Route ID. +fn decode_warm_start(r: &mut Decoder<'_>) -> Content<(Path, Instance, Warm)> { + let suffix = match decode_path(r)? { + Ok(suffix) => suffix, + Err(reason) => return refuse(r, reason), + }; + let Some(instance) = super::instance::decode_copy(r)? else { + return refuse(r, Refusal::Instance); + }; + Ok(decode_warm(r)?.map(|warm| (suffix, instance, warm))) } /// ROUTE_START and ROUTE_UPDATE flag: the route is down-only. @@ -209,15 +275,20 @@ fn encode_advert(w: &mut Encoder<'_>, advert: &Advert) -> Result<(), EncodeError Ok(()) } -fn decode_advert(r: &mut Decoder<'_>) -> Result { +/// Decode an advertisement, refusing one whose seqno no origin advertises, whose flags +/// this node does not know, or that crossed too many hops. +/// +/// An unknown flag refuses the route rather than being ignored: a newer flag may change +/// what the route means, or add a field ahead of the hops that would be misread. +fn decode_advert(r: &mut Decoder<'_>) -> Content { let seqno = r.varint()?; if seqno == route::MAX { - return Err(DecodeError::InvalidValue); + return refuse(r, Refusal::Seqno); } let metric = r.varint()?; let flags = r.u8()?; if flags & !(DOWN_ONLY | RTT) != 0 { - return Err(DecodeError::InvalidValue); + return refuse(r, Refusal::Flags(flags)); } let rtt = match flags & RTT { 0 => None, @@ -225,18 +296,36 @@ fn decode_advert(r: &mut Decoder<'_>) -> Result { }; let count = r.varint()?; if count > route::MAX_HOPS as u64 { - return Err(DecodeError::TooMany); + return refuse(r, Refusal::Hops(count)); } let hops = (0..count).map(|_| r.varint()).collect::>()?; - Ok(Advert { + Ok(Ok(Advert { seqno, metric, down_only: flags & DOWN_ONLY != 0, rtt, hops, - }) + })) +} + +/// Decode a path a peer forwarded, refusing one this node cannot hold. +fn decode_path(r: &mut Decoder<'_>) -> Content { + let bytes = r.bytes()?; + let path = std::str::from_utf8(bytes).ok().and_then(|path| Path::new(path).ok()); + Ok(path.filter(|path| path.parts().count() <= Path::MAX_PARTS).ok_or(Refusal::Path)) +} + +/// Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded. +fn decode_suffix(r: &mut Decoder<'_>) -> Content { + let base = r.varint()?; + let keep = r.varint()?; + if base == 0 && keep != 0 { + return Err(DecodeError::InvalidValue); + } + Ok(decode_path(r)?.map(|rest| PathRef { base, keep, rest })) } + /// Encode an ANNOUNCE_FLOOR body: the revision plus one, or 0 to certify nothing. fn encode_floor(w: &mut Encoder<'_>, id: u64, revision: Option) -> Result<(), EncodeError> { w.varint(id)?; @@ -310,14 +399,6 @@ fn encode_route_id(w: &mut Encoder<'_>, version: Version, route: u64) -> Result< } } -/// Decode the Route ID a claim travels under, 0 on a version without one. -fn decode_route_id(r: &mut Decoder<'_>, version: Version) -> Result { - match version.has_routes() { - true => r.varint(), - false => Ok(0), - } -} - /// Encode a lite-06 announcement's hop chain, which lite-07 replaces with its Route ID. fn encode_chain(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Result<(), EncodeError> { match version.has_routes() { @@ -343,17 +424,97 @@ fn encode_lite06_chain(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Re } } +/// A lite-07 ANNOUNCE_START: its suffix, Route ID, instance and epoch, cost, and +/// Lite-06 Chain. +fn decode_start(r: &mut Decoder<'_>, version: Version) -> Result { + let suffix = decode_suffix(r)?; + let route = r.varint()?; + let refused = |r: &mut Decoder<'_>, suffix, reason| { + r.rest(); + AnnounceBroadcast::Refused { + message: Refusable::Start { route, suffix }, + reason, + } + }; + let suffix = match suffix { + Ok(suffix) => suffix, + Err(reason) => return Ok(refused(r, None, reason)), + }; + let Some((instance, epoch)) = super::instance::decode_forwarded(r)? else { + return Ok(refused(r, Some(suffix), Refusal::Instance)); + }; + let cost = Cost::decode(r, version)?; + Ok(match decode_lite06_chain(r)? { + Ok(hops) => AnnounceBroadcast::Active { + suffix, + route, + instance: Some(instance), + epoch, + hops, + cost, + }, + Err(reason) => refused(r, Some(suffix), reason), + }) +} + +/// A lite-07 ANNOUNCE_UPDATE: its Announce ID, cost, and Lite-06 Chain. +fn decode_update(r: &mut Decoder<'_>, version: Version) -> Result { + let id = r.varint()?; + let cost = Cost::decode(r, version)?; + Ok(match decode_lite06_chain(r)? { + Ok(hops) => AnnounceBroadcast::Update { id, hops, cost }, + Err(reason) => AnnounceBroadcast::Refused { + message: Refusable::Change { id }, + reason, + }, + }) +} + +/// A lite-07 ANNOUNCE_RESTART: its Announce ID, instance and epoch, cost, and Lite-06 +/// Chain. +fn decode_restart(r: &mut Decoder<'_>, version: Version) -> Result { + let id = r.varint()?; + let refused = |r: &mut Decoder<'_>, reason| { + r.rest(); + AnnounceBroadcast::Refused { + message: Refusable::Change { id }, + reason, + } + }; + let Some((instance, epoch)) = super::instance::decode_forwarded(r)? else { + return Ok(refused(r, Refusal::Instance)); + }; + let cost = Cost::decode(r, version)?; + Ok(match decode_lite06_chain(r)? { + Ok(hops) => AnnounceBroadcast::Restart { + id, + instance: Some(instance), + epoch, + cost, + hops, + }, + Err(reason) => refused(r, reason), + }) +} + /// Decode lite-07's trailing Lite-06 Chain: present only when bytes remain, and never -/// empty, so an empty chain has one spelling. -fn decode_lite06_chain(r: &mut Decoder<'_>, version: Version) -> Result { - if !version.has_routes() || r.is_empty() { - return Ok(Hops::default()); +/// empty, so an empty chain has one spelling. A chain no lite-06 relay would send, empty, +/// too long, or revisiting a hop, is refused. +fn decode_lite06_chain(r: &mut Decoder<'_>) -> Content { + if r.is_empty() { + return Ok(Ok(Hops::default())); } - let hops = Hops::decode(r, version)?; - match hops.is_empty() { - true => Err(DecodeError::InvalidValue), - false => Ok(hops), + let count = r.varint()?; + if count == 0 { + return refuse(r, Refusal::Chain); } + let mut hops = Hops::new(); + for _ in 0..count { + if hops.push(Hop::decode(r, ())?).is_err() { + return refuse(r, Refusal::Chain); + } + } + Ok(Ok(hops)) } impl Encode for Cost { @@ -419,8 +580,8 @@ impl Encode for AnnounceBroadcast { | Self::WarmEnd { .. } => return Err(EncodeError::Version), // The pre-lite-06 path-form retraction has no place on lite-06. Self::Ended { .. } => return Err(EncodeError::Version), - // Decode-only: an unknown type is never sent. - Self::Skipped => return Err(EncodeError::Unsupported), + // Decode-only: an unknown type or refused content is never sent. + Self::Skipped | Self::Refused { .. } => return Err(EncodeError::Unsupported), }; w.varint(typ)?; @@ -490,7 +651,7 @@ impl Encode for AnnounceBroadcast { encode_warm(w, warm)?; } Self::WarmEnd { id } => w.varint(*id)?, - Self::Ended { .. } | Self::Skipped => unreachable!("refused above"), + Self::Ended { .. } | Self::Skipped | Self::Refused { .. } => unreachable!("refused above"), } if w.since(&prefix) > MAX_MESSAGE_SIZE { w.discard(prefix); @@ -528,6 +689,7 @@ impl Encode for AnnounceBroadcast { | Self::Update { .. } | Self::Restart { .. } | Self::Skipped + | Self::Refused { .. } | Self::Synced | Self::RouteStart { .. } | Self::RouteUpdate { .. } @@ -558,21 +720,21 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { let mut body = buf.sub(size)?; let msg = (|| { Ok(match typ { + // Lite-07's claims and routes carry what the sender forwarded, which + // a receiver may refuse without failing the stream. + ANNOUNCE_START if version.has_routes() => decode_start(&mut body, version)?, + ANNOUNCE_UPDATE if version.has_routes() => decode_update(&mut body, version)?, + ANNOUNCE_RESTART if version.has_announce_restart() => decode_restart(&mut body, version)?, ANNOUNCE_START => { let suffix = PathRef::decode(&mut body, version)?; - let route = decode_route_id(&mut body, version)?; - let (instance, epoch) = super::instance::decode_route(&mut body, version)?; let hops = decode_chain(&mut body, version)?; let cost = Cost::decode(&mut body, version)?; Self::Active { suffix, - route, - instance, - epoch, - hops: match version.has_routes() { - true => decode_lite06_chain(&mut body, version)?, - false => hops, - }, + route: 0, + instance: None, + epoch: 0, + hops, cost, } } @@ -582,51 +744,60 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { let id = body.varint()?; let hops = decode_chain(&mut body, version)?; let cost = Cost::decode(&mut body, version)?; - Self::Update { - id, - hops: match version.has_routes() { - true => decode_lite06_chain(&mut body, version)?, - false => hops, - }, - cost, - } - } - ANNOUNCE_RESTART if version.has_announce_restart() => { - let id = body.varint()?; - let (instance, epoch) = super::instance::decode_route(&mut body, version)?; - Self::Restart { - id, - instance, - epoch, - cost: Cost::decode(&mut body, version)?, - hops: decode_lite06_chain(&mut body, version)?, - } + Self::Update { id, hops, cost } } ROUTE_START if version.has_routes() => { let node = body.varint()?; - let advert = decode_advert(&mut body)?; - if node == 0 { - return Err(DecodeError::InvalidValue); + let refused = |reason| Self::Refused { + message: Refusable::RouteStart { node }, + reason, + }; + match node { + 0 => { + body.rest(); + refused(Refusal::Node) + } + _ => decode_advert(&mut body)?.map_or_else(refused, |advert| Self::RouteStart { node, advert }), } - Self::RouteStart { node, advert } } - ROUTE_UPDATE if version.has_routes() => Self::RouteUpdate { - id: body.varint()?, - advert: decode_advert(&mut body)?, - }, + ROUTE_UPDATE if version.has_routes() => { + let id = body.varint()?; + decode_advert(&mut body)?.map_or_else( + |reason| Self::Refused { + message: Refusable::RouteUpdate { id }, + reason, + }, + |advert| Self::RouteUpdate { id, advert }, + ) + } ROUTE_HOLD if version.has_routes() => Self::RouteHold { id: body.varint()? }, ROUTE_END if version.has_routes() => Self::RouteEnd { id: body.varint()? }, ANNOUNCE_FLOOR if version.has_routes() => decode_floor(&mut body)?, - WARM_START if version.has_warm() => Self::WarmStart { - route: body.varint()?, - suffix: Path::decode(&mut body, version)?, - instance: super::instance::decode_copy(&mut body)?, - warm: decode_warm(&mut body)?, - }, - WARM_UPDATE if version.has_warm() => Self::WarmUpdate { - id: body.varint()?, - warm: decode_warm(&mut body)?, - }, + WARM_START if version.has_warm() => { + let route = body.varint()?; + decode_warm_start(&mut body)?.map_or_else( + |reason| Self::Refused { + message: Refusable::WarmStart { route }, + reason, + }, + |(suffix, instance, warm)| Self::WarmStart { + route, + suffix, + instance, + warm, + }, + ) + } + WARM_UPDATE if version.has_warm() => { + let id = body.varint()?; + decode_warm(&mut body)?.map_or_else( + |reason| Self::Refused { + message: Refusable::WarmUpdate { id }, + reason, + }, + |warm| Self::WarmUpdate { id, warm }, + ) + } WARM_END if version.has_warm() => Self::WarmEnd { id: body.varint()? }, // Unknown types are skipped by length so an earlier Lite06 build // negotiating the same ALPN does not kill the announce stream. @@ -1317,7 +1488,8 @@ mod tests { } } - /// A warm route's carrier is one of its hops, of which there are at most 32. + /// A warm route's carrier is one of its hops, of which there are at most 32. A + /// receiver refuses one that breaks either, without failing the stream. #[test] fn lite07_warm_route_names_its_carrier_among_its_hops() { let outside = AnnounceBroadcast::WarmUpdate { @@ -1336,14 +1508,19 @@ mod tests { let decode = |wire: &[u8]| { crate::coding::decode_buf(&mut &wire[..], Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)) }; - assert!(decode(&[13, 4, 7, 4, 0, 0]).is_err(), "no hops, so no carrier"); - assert!( - decode(&[13, 7, 7, 4, 3, 3, 5, 9, 6]).is_err(), + let refused = |reason| AnnounceBroadcast::Refused { + message: Refusable::WarmUpdate { id: 7 }, + reason, + }; + assert_eq!(decode(&[13, 4, 7, 4, 0, 0]).unwrap(), refused(Refusal::Carrier), "no hops"); + assert_eq!( + decode(&[13, 7, 7, 4, 3, 3, 5, 9, 6]).unwrap(), + refused(Refusal::Carrier), "a carrier past the hops" ); let mut long = vec![13, 37, 7, 4, 0, 33]; long.extend([1; 33]); - assert!(decode(&long).is_err(), "33 hops"); + assert_eq!(decode(&long).unwrap(), refused(Refusal::Hops(33))); long[1] = 36; long[5] = 32; long.pop(); @@ -1395,10 +1572,28 @@ mod tests { assert_eq!(broadcast_round_trip(msg, Version::Lite07), *msg); } - // An ANNOUNCE_UPDATE whose chain is present but empty. - let mut empty: &[u8] = &[2, 3, 7, 6, 0]; - let res = crate::coding::decode_buf(&mut empty, Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)); - assert!(res.is_err(), "{res:?}"); + // A chain present but empty, or revisiting a hop, is refused, as is the claim it + // came with. + let decode = |mut wire: &[u8]| { + crate::coding::decode_buf(&mut wire, Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)).unwrap() + }; + let refused = AnnounceBroadcast::Refused { + message: Refusable::Change { id: 7 }, + reason: Refusal::Chain, + }; + assert_eq!(decode(&[2, 3, 7, 6, 0]), refused); + assert_eq!(decode(&[2, 5, 7, 6, 2, 9, 9]), refused); + assert_eq!(decode(&[3, 9, 7, 1, b'a', 0, 5, 6, 2, 9, 9]), refused); + assert_eq!( + decode(&[0, 13, 0, 0, 1, b'c', 3, 1, b'a', 0, 5, 6, 2, 9, 9]), + AnnounceBroadcast::Refused { + message: Refusable::Start { + route: 3, + suffix: Some(PathRef::literal(Path::from_static("c"))), + }, + reason: Refusal::Chain, + } + ); } /// Only lite-07 carries a Route ID: a non-zero one has nowhere to go earlier, and an @@ -1956,18 +2151,36 @@ mod tests { } }; let over = route::MAX_HOPS as u64 + 1; + // What the sender only forwarded is refused, the message decoded all the same, so + // the receiver withdraws the route and keeps the stream. + let start = |node| Refusable::RouteStart { node }; + let refused = [ + (forge(ROUTE_START, hops(over, over)), start(1), Refusal::Hops(over)), + (forge(ROUTE_START, advert(0, 0, 1, 0)), start(0), Refusal::Node), + (forge(ROUTE_START, advert(1, route::MAX, 1, 0)), start(1), Refusal::Seqno), + // A flag no version defines, which may change what follows it. + (forge(ROUTE_START, advert(1, 0, 1, 4)), start(1), Refusal::Flags(4)), + (forge(ROUTE_START, advert(1, 0, 1, 0x86)), start(1), Refusal::Flags(0x86)), + // ROUTE_UPDATE's leading field is the Route ID, so 0 is a valid id here. + ( + forge(ROUTE_UPDATE, advert(0, route::MAX, 1, 0)), + Refusable::RouteUpdate { id: 0 }, + Refusal::Seqno, + ), + ]; + for (wire, message, reason) in refused { + assert_eq!( + decode_broadcast(&wire, Version::Lite07).unwrap(), + AnnounceBroadcast::Refused { message, reason }, + "{wire:?}" + ); + } let malformed = [ - // More hops than a route may cross, hops missing, and one too many. - forge(ROUTE_START, hops(over, over)), + // Hops missing, and one too many. forge(ROUTE_START, hops(2, 1)), forge(ROUTE_START, hops(1, 2)), - forge(ROUTE_START, advert(0, 0, 1, 0)), - forge(ROUTE_START, advert(1, route::MAX, 1, 0)), - // A flag no version defines, and an RTT flag without its field. - forge(ROUTE_START, advert(1, 0, 1, 4)), + // An RTT flag without its field. forge(ROUTE_START, advert(1, 0, 1, 2)), - // ROUTE_UPDATE's leading field is the Route ID, so 0 is a valid id here. - forge(ROUTE_UPDATE, advert(0, route::MAX, 1, 0)), // A hold carries only its Route ID. forge(ROUTE_HOLD, |w| { w.varint(0u64).unwrap(); diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index dfa8f6c370c..80e0654146f 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -6,7 +6,7 @@ //! decoder here resolves them against the stream's live announcements, which the //! subscriber has to track anyway, and the encoder picks them. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap, HashSet}; use crate::{ Error, Path, @@ -29,32 +29,29 @@ impl Live { self.suffixes.get(&id).ok_or(Error::ProtocolViolation) } - /// The live announcement `distance` back from the next id, or `None` for 0. - fn base(&self, distance: u64) -> Result, Error> { + /// The id `distance` back from the next, or `None` for 0. + fn base(&self, distance: u64) -> Result, Error> { if distance == 0 { return Ok(None); } - let id = self.next.checked_sub(distance).ok_or(Error::ProtocolViolation)?; - self.suffix(id).map(Some) + self.next.checked_sub(distance).map(Some).ok_or(Error::ProtocolViolation) } +} - fn resolve(&self, path: PathRef) -> Result { - let Some(base) = self.base(path.base)? else { - return Ok(path.rest); - }; - let keep = usize::try_from(path.keep).map_err(|_| Error::ProtocolViolation)?; - let head = head(base, keep).ok_or(Error::ProtocolViolation)?; - let suffix = match (head.is_empty(), path.rest.is_empty()) { - (true, _) => Ok(path.rest), - (false, true) => Path::new(head), - (false, false) => Path::concat(head, path.rest.as_str()), - } - .map_err(|_| Error::ProtocolViolation)?; - if suffix.parts().count() > Path::MAX_PARTS { - return Err(Error::ProtocolViolation); - } - Ok(suffix) - } +/// Resolve `path` against the live suffix `base` names, or `None` when the result is +/// no path this node can hold. Naming a head `base` lacks is the sender's own fault. +fn resolve(base: Option<&Path>, path: PathRef) -> Result, Error> { + let Some(base) = base else { + return Ok(Some(path.rest)); + }; + let keep = usize::try_from(path.keep).map_err(|_| Error::ProtocolViolation)?; + let head = head(base, keep).ok_or(Error::ProtocolViolation)?; + let suffix = match (head.is_empty(), path.rest.is_empty()) { + (true, _) => Ok(path.rest), + (false, true) => Path::new(head), + (false, false) => Path::concat(head, path.rest.as_str()), + }; + Ok(suffix.ok().filter(|suffix| suffix.parts().count() <= Path::MAX_PARTS)) } /// The first `keep` segments of `path`, or `None` if it has fewer. @@ -75,15 +72,32 @@ fn head(path: &Path, keep: usize) -> Option<&str> { #[derive(Debug, Default)] pub(crate) struct AnnounceDecoder { live: Live, + /// The live ids whose path was refused, which a base names only to be refused too. + refused: HashSet, } impl AnnounceDecoder { - /// An ANNOUNCE_START: resolve its suffix and assign it the next id. - pub fn start(&mut self, suffix: PathRef) -> Result { - let suffix = self.live.resolve(suffix)?; + /// An ANNOUNCE_START: assign it the next id, and resolve its suffix, or `None` when + /// that suffix, or the base it names, is no path this node can hold. + pub fn start(&mut self, suffix: Option) -> Result, Error> { + let suffix = match suffix { + Some(suffix) => match self.live.base(suffix.base)? { + Some(id) if self.refused.contains(&id) => None, + Some(id) => resolve(Some(self.live.suffix(id)?), suffix)?, + None => resolve(None, suffix)?, + }, + None => None, + }; let id = self.live.next; self.live.next += 1; - self.live.suffixes.insert(id, suffix.clone()); + match &suffix { + Some(suffix) => { + self.live.suffixes.insert(id, suffix.clone()); + } + None => { + self.refused.insert(id); + } + } Ok(suffix) } @@ -93,14 +107,20 @@ impl AnnounceDecoder { } /// The suffix of the live id an ANNOUNCE_UPDATE or ANNOUNCE_RESTART names, which - /// stays live. + /// stays live. An id whose path was refused has none. pub fn get(&self, id: u64) -> Result { self.live.suffix(id).cloned() } - /// An ANNOUNCE_END: retire the id, returning its suffix. - pub fn end(&mut self, id: u64) -> Result { - self.live.suffixes.remove(&id).ok_or(Error::ProtocolViolation) + /// An ANNOUNCE_END: retire the id, returning its suffix, or `None` if it was refused. + pub fn end(&mut self, id: u64) -> Result, Error> { + if let Some(suffix) = self.live.suffixes.remove(&id) { + return Ok(Some(suffix)); + } + match self.refused.remove(&id) { + true => Ok(None), + false => Err(Error::ProtocolViolation), + } } } @@ -275,76 +295,82 @@ mod tests { #[test] fn resolves_a_path_head() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("pid/private/channel_1/health-1")).unwrap(); + decoder.start(Some(literal("pid/private/channel_1/health-1"))).unwrap(); // Three leading segments of the latest START. - let suffix = decoder.start(path(1, 3, "health-2")).unwrap(); + let suffix = decoder.start(Some(path(1, 3, "health-2"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "pid/private/channel_1/health-2"); } #[test] fn keeps_the_whole_path() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("a/b")).unwrap(); + decoder.start(Some(literal("a/b"))).unwrap(); // A full keep with a rest names a route beneath the base's: not a duplicate, // since the rest extends it. - let suffix = decoder.start(path(1, 2, "c")).unwrap(); + let suffix = decoder.start(Some(path(1, 2, "c"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "a/b/c"); // And a full keep with an empty rest copies it whole. - let suffix = decoder.start(path(1, 3, "")).unwrap(); + let suffix = decoder.start(Some(path(1, 3, ""))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "a/b/c"); } #[test] fn a_new_stream_has_no_base() { let mut decoder = AnnounceDecoder::default(); - assert!(matches!(decoder.start(path(1, 0, "a")), Err(Error::ProtocolViolation))); + assert!(matches!(decoder.start(Some(path(1, 0, "a"))), Err(Error::ProtocolViolation))); } #[test] fn rejects_a_retired_or_unassigned_base() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("a")).unwrap(); - decoder.start(literal("b")).unwrap(); + decoder.start(Some(literal("a"))).unwrap(); + decoder.start(Some(literal("b"))).unwrap(); decoder.end(0).unwrap(); // Id 0 is two back, and retired. - assert!(matches!(decoder.start(path(2, 1, "x")), Err(Error::ProtocolViolation))); + assert!(matches!(decoder.start(Some(path(2, 1, "x"))), Err(Error::ProtocolViolation))); // Three back was never assigned. - assert!(matches!(decoder.start(path(3, 0, "x")), Err(Error::ProtocolViolation))); + assert!(matches!(decoder.start(Some(path(3, 0, "x"))), Err(Error::ProtocolViolation))); // Id 1 is still live, so reusing it works after the END. - let suffix = decoder.start(path(1, 1, "x")).unwrap(); + let suffix = decoder.start(Some(path(1, 1, "x"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "b/x"); } #[test] fn rejects_a_keep_longer_than_the_base() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("a/b")).unwrap(); - assert!(matches!(decoder.start(path(1, 3, "x")), Err(Error::ProtocolViolation))); + decoder.start(Some(literal("a/b"))).unwrap(); + assert!(matches!(decoder.start(Some(path(1, 3, "x"))), Err(Error::ProtocolViolation))); } + /// A resolved path past the limits is content the peer forwarded: refused, its id + /// still assigned, and a base naming it refused too. #[test] - fn rejects_a_resolved_path_too_deep() { + fn refuses_a_resolved_path_too_deep() { let mut decoder = AnnounceDecoder::default(); let deep = vec!["x"; Path::MAX_PARTS].join("/"); - decoder.start(literal(&deep)).unwrap(); - assert!(matches!( - decoder.start(path(1, Path::MAX_PARTS as u64, "y")), - Err(Error::ProtocolViolation) - )); + decoder.start(Some(literal(&deep))).unwrap(); + assert_eq!(decoder.start(Some(path(1, Path::MAX_PARTS as u64, "y"))).unwrap(), None); + assert_eq!(decoder.last(), 1); + assert_eq!(decoder.start(Some(path(1, 1, "z"))).unwrap(), None); + decoder.end(1).unwrap(); + assert!(matches!(decoder.end(1), Err(Error::ProtocolViolation))); + // A start refused before its suffix resolved takes an id all the same. + assert_eq!(decoder.start(None).unwrap(), None); + assert_eq!(decoder.last(), 3); } #[test] - fn rejects_a_resolved_path_too_long() { + fn refuses_a_resolved_path_too_long() { let mut decoder = AnnounceDecoder::default(); let head = "x".repeat(Path::MAX_BYTES - 2); - decoder.start(literal(&head)).unwrap(); + decoder.start(Some(literal(&head))).unwrap(); // `head/y` is exactly the limit, and one byte more is refused. - let suffix = decoder.start(path(1, 1, "y")).unwrap(); + let suffix = decoder.start(Some(path(1, 1, "y"))).unwrap().unwrap(); assert_eq!(suffix.len(), Path::MAX_BYTES); - assert!(matches!(decoder.start(path(1, 1, "yz")), Err(Error::ProtocolViolation))); + assert_eq!(decoder.start(Some(path(1, 1, "yz"))).unwrap(), None); } /// Encode a START with `encoder`, decode it with `decoder`, and check that the @@ -352,7 +378,7 @@ mod tests { fn start(encoder: &mut AnnounceEncoder, decoder: &mut AnnounceDecoder, suffix: &str) -> (u64, usize) { let (id, wire) = encoder.start(Path::new(suffix).unwrap()); let size = wire.encode_bytes(VERSION).unwrap().len(); - assert_eq!(decoder.start(wire).unwrap().as_str(), suffix); + assert_eq!(decoder.start(Some(wire)).unwrap().unwrap().as_str(), suffix); (id.unwrap(), size) } diff --git a/rs/moq-net/src/lite/corpus.rs b/rs/moq-net/src/lite/corpus.rs index 2e5eac24e4e..309eff3fab3 100644 --- a/rs/moq-net/src/lite/corpus.rs +++ b/rs/moq-net/src/lite/corpus.rs @@ -4,8 +4,10 @@ //! //! Each case is one complete wire input at a negotiated `version` (its handshake name), with //! the outcome both must reach: `decoded`, where our own encoding of the decoded value must -//! equal the input, so a case also pins the value, or `refused`, where the input is rejected -//! without asking for more bytes. A fuzz crash on a single moq-lite message belongs here +//! equal the input, so a case also pins the value, `refused`, where the input is rejected +//! without asking for more bytes, or `withdrawn`, where a lite-07 route message decodes but its +//! forwarded content is refused, so the receiver withdraws the route or claim and keeps the +//! session. A fuzz crash on a single moq-lite message belongs here //! rather than in `fuzz/regressions/`, so JS replays it too. use crate::coding::{Decode, DecodeError, Decoder, Encode, Form}; @@ -45,6 +47,7 @@ enum Message { enum Expect { Decoded, Refused, + Withdrawn, } /// What a decode came to: our own encoding of what it decoded, as hex, or why it did not. @@ -72,7 +75,10 @@ fn run(case: &Case, data: &[u8]) -> String { }; let decoder = || Decoder::new(data, version.into()); match case.message { - Message::Announce => outcome(data, version, super::AnnounceBroadcast::decode_slice(data, version)), + Message::Announce => match super::AnnounceBroadcast::decode_slice(data, version) { + Ok((super::AnnounceBroadcast::Refused { .. }, used)) if used == data.len() => "withdrawn".to_string(), + decoded => outcome(data, version, decoded), + }, Message::AnnounceReply => outcome(data, version, super::AnnounceReply::decode_slice(data, version)), Message::AnnounceInit => { let mut r = decoder(); @@ -114,6 +120,7 @@ fn corpus() { let want = match case.expect { Expect::Decoded => case.hex.clone(), Expect::Refused => "refused".to_string(), + Expect::Withdrawn => "withdrawn".to_string(), }; let got = run(case, &unhex(&case.hex)); if got != want { diff --git a/rs/moq-net/src/lite/instance.rs b/rs/moq-net/src/lite/instance.rs index 60687d09000..f38022cf146 100644 --- a/rs/moq-net/src/lite/instance.rs +++ b/rs/moq-net/src/lite/instance.rs @@ -71,13 +71,13 @@ pub(super) fn encode_route( w.varint(epoch) } -/// Decode the instance and epoch an announcement serves; see [`encode_route`]. -pub(super) fn decode_route(r: &mut Decoder<'_>, version: Version) -> Result<(Option, u64), DecodeError> { - if !version.has_instance() { - return Ok((None, 0)); - } - let instance = decode(r.bytes()?, r.bool()?)?; - Ok((Some(instance), r.varint()?)) +/// Decode lite-07's instance and epoch, `None` when they are well framed but the +/// instance is not one this node can use, which withdraws only the claim naming it; +/// see [`encode_route`]. +pub(super) fn decode_forwarded(r: &mut Decoder<'_>) -> Result, DecodeError> { + let instance = decode(r.bytes()?, r.bool()?).ok(); + let epoch = r.varint()?; + Ok(instance.map(|instance| (instance, epoch))) } /// Encode the instance of a copy a warm route names, which is never none. @@ -85,9 +85,10 @@ pub(super) fn encode_copy(w: &mut Encoder<'_>, instance: &Instance) -> Result<() encode(w, instance) } -/// Decode the instance of a copy a warm route names; see [`encode_copy`]. -pub(super) fn decode_copy(r: &mut Decoder<'_>) -> Result { - decode(r.bytes()?, r.bool()?) +/// Decode the instance of a copy a warm route names, `None` when it is well framed but +/// not one this node can use; see [`encode_copy`]. +pub(super) fn decode_copy(r: &mut Decoder<'_>) -> Result, DecodeError> { + Ok(decode(r.bytes()?, r.bool()?).ok()) } fn encode(w: &mut Encoder<'_>, instance: &Instance) -> Result<(), EncodeError> { @@ -163,13 +164,11 @@ mod test { ) .unwrap(); let mut decoder = Decoder::new(&buf, Version::Lite07.into()); - assert_eq!( - decode_route(&mut decoder, Version::Lite07).unwrap(), - (Some(instance), 7) - ); + assert_eq!(decode_forwarded(&mut decoder).unwrap(), Some((instance, 7))); assert!(decoder.is_empty()); - // Every lite-07 route names one: none is refused both ways. + // Every lite-07 route names one: none is never sent, and a receiver refuses an + // invalid one without failing the stream. assert!( encode_route( &mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), @@ -184,7 +183,9 @@ mod test { w.bytes(&[]).unwrap(); w.bool(false); w.varint(0).unwrap(); - assert!(decode_route(&mut Decoder::new(&buf, Version::Lite07.into()), Version::Lite07).is_err()); + let mut decoder = Decoder::new(&buf, Version::Lite07.into()); + assert_eq!(decode_forwarded(&mut decoder).unwrap(), None); + assert!(decoder.is_empty()); } #[test] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 5c3418b711f..650a155819b 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -822,7 +822,8 @@ impl Subscriber { } => { let suffix = match self.version.has_announce_id() { // Every `active` assigns the next ordinal, even ones we drop locally. - true => run.decoder.start(suffix)?, + // Only lite-07 refuses a forwarded path, and it never reaches here. + true => run.decoder.start(Some(suffix))?.ok_or(Error::ProtocolViolation)?, // Nothing references an announcement here, so there are no bases. false => suffix.rest, }; @@ -865,7 +866,7 @@ impl Subscriber { lite::AnnounceBroadcast::EndedId { id } => { // Resolve and retire the id; an unknown or already-retired id is a // protocol violation. - let path = prefix.join(&run.decoder.end(id)?)?; + let path = prefix.join(&run.decoder.end(id)?.ok_or(Error::ProtocolViolation)?)?; tracing::debug!(broadcast = %self.log_path(&path), "unannounced"); run.announced.withdraw(&path); } @@ -913,7 +914,8 @@ impl Subscriber { | lite::AnnounceBroadcast::Floor { .. } | lite::AnnounceBroadcast::WarmStart { .. } | lite::AnnounceBroadcast::WarmUpdate { .. } - | lite::AnnounceBroadcast::WarmEnd { .. } => return Err(Error::ProtocolViolation), + | lite::AnnounceBroadcast::WarmEnd { .. } + | lite::AnnounceBroadcast::Refused { .. } => return Err(Error::ProtocolViolation), } Ok(()) } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index f01d2167b9a..bde5173abc5 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -6,7 +6,7 @@ //! the origin, minting a source per request that names the claim's origin node. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, task::Poll, }; @@ -22,9 +22,13 @@ use super::{AnnouncedRoute, Subscriber}; /// One route the peer sent on the stream, by its Route ID. struct WireRoute { node: route::Id, - reach: route::Reach, + /// How the peer reaches the node, or `None` while this node refuses what the peer + /// forwarded: the node holds no such route, as if the peer had ended it. + reach: Option, /// The Announce ID of each of its live claims, by path. claims: BTreeMap, + /// The Announce IDs of its live claims whose path this node refused. + unnamed: BTreeSet, /// How its claims differ from `committed`, by path, so what the stream keeps /// between floors is bounded by its claims rather than the messages it read. pending: BTreeMap>, @@ -47,9 +51,12 @@ impl WireRoute { /// One live claim the peer sent, by its Announce ID. struct WireClaim { route: u64, - /// Absolute: the origin node keys every claim from the origin's root. - path: Path, - claim: node::Claim, + /// Absolute: the origin node keys every claim from the origin's root. `None` when this + /// node refused the path, which the claim then never has. + path: Option, + /// `None` while this node refuses what the peer forwarded, until an ANNOUNCE_RESTART + /// names all of it again. + claim: Option, } /// A claim the node serves through this stream. @@ -103,6 +110,8 @@ pub(super) struct Routed { read_rtt: Option, /// Whether the peer has a route here, so the link's RTT is read. sampling: bool, + /// Counts what this node refuses of the peer's routes and claims. + stats: crate::stats::Session, } impl Routed { @@ -160,6 +169,7 @@ impl Routed { rtt: Default::default(), read_rtt: None, sampling: false, + stats: subscriber.origin.stats().clone(), }) } @@ -172,41 +182,29 @@ impl Routed { ) -> Result<(), Error> { use lite::AnnounceBroadcast as Msg; match announce { - Msg::RouteStart { node, advert } => { - if self.routes.values().any(|route| route.node == node) { - return Err(Error::ProtocolViolation); + Msg::RouteStart { node, advert } => self.route_start(node, Some(route::Reach::Finite(advert)))?, + Msg::RouteUpdate { id, advert } => self.reach(id, Some(route::Reach::Finite(advert)))?, + Msg::RouteHold { id } => { + let route = self.routes.get(&id).ok_or(Error::ProtocolViolation)?; + // A refused route has nothing to hold. + if route.reach.is_some() { + self.reach(id, Some(route::Reach::Held))?; } - self.held.push(self.slots.acquire()?); - let id = self.next_route; - self.next_route += 1; - let reach = route::Reach::Finite(advert); - self.routes.insert( - id, - WireRoute { - node, - reach: reach.clone(), - claims: BTreeMap::new(), - pending: BTreeMap::new(), - committed: None, - }, - ); - self.stream.route(node, Some(reach)); } - Msg::RouteUpdate { id, advert } => self.reach(id, route::Reach::Finite(advert))?, - Msg::RouteHold { id } => self.reach(id, route::Reach::Held)?, Msg::RouteEnd { id } => { let route = self.routes.remove(&id).ok_or(Error::ProtocolViolation)?; - for claim in route.claims.into_values() { + for claim in route.claims.into_values().chain(route.unnamed) { decoder.end(claim)?; self.claims.remove(&claim); self.held.pop(); } self.echoing.extend(self.held.pop()); - self.stream.route(route.node, None); + if route.reach.is_some() { + self.stream.route(route.node, None); + } self.ends.push(id); } Msg::Floor { id, revision } => { - let certifies = self.certifies(); let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; let changes = std::mem::take(&mut route.pending); let (committed, floor) = route.committed.get_or_insert_default(); @@ -217,12 +215,11 @@ impl Routed { }; } *floor = revision; - // The node never held a claim the limit leaves out, so it neither starts nor ends. - let changes = (changes.into_iter()) - .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) - .collect(); - let revision = revision.filter(|_| certifies); - self.stream.batch(route.node, changes, revision); + // A refused route files its claims once the peer forwards it usably again. + if route.reach.is_some() { + let node = route.node; + self.file(node, changes, revision); + } } Msg::Active { suffix, @@ -232,16 +229,20 @@ impl Routed { hops, cost, } => { - let path = self.path(prefix, &decoder.start(suffix)?)?; + let path = decoder.start(Some(suffix))?; + let path = self.claim_path(prefix, path); let claim = claim(instance, epoch, cost, hops)?; - self.start(decoder.last(), route, path, claim)?; + self.start(decoder.last(), route, path, Some(claim))?; } Msg::Update { id, hops, cost } => { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; - claim.claim.cost = cost.value(); - claim.claim.chain = hops; - let change = (claim.path.clone(), Some(claim.claim.clone())); - self.pending(id, change)?; + // An update names none of what a refused claim lacks. + if let Some(held) = &mut claim.claim { + held.cost = cost.value(); + held.chain = hops; + let change = claim.claim.clone(); + self.pending(id, change)?; + } } Msg::Restart { id, @@ -252,14 +253,15 @@ impl Routed { } => { let fresh = self::claim(instance, epoch, cost, hops)?; let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; - claim.claim = fresh; - let change = (claim.path.clone(), Some(claim.claim.clone())); + claim.claim = Some(fresh); + let change = claim.claim.clone(); self.pending(id, change)?; } Msg::EndedId { id } => { decoder.end(id)?; self.end(id)?; } + Msg::Refused { message, reason } => self.refused(prefix, message, reason, decoder)?, // Lite-07 ends a claim by its Announce ID only. Msg::Ended { .. } => return Err(Error::ProtocolViolation), // Only a relay choosing a parent per broadcast uses warm routes, which this @@ -270,27 +272,140 @@ impl Routed { Ok(()) } - /// The absolute path of a claim under the stream's prefix. - fn path(&self, prefix: &Path, suffix: &Path) -> Result { - Ok(self.root.join(&prefix.join(suffix)?)?) + /// Apply a message whose content, which the peer only forwarded, this node refuses: + /// withdraw the route or claim it names, and keep the stream. + fn refused( + &mut self, + prefix: &Path, + message: lite::Refusable, + reason: lite::Refusal, + decoder: &mut lite::AnnounceDecoder, + ) -> Result<(), Error> { + use lite::Refusable; + match message { + Refusable::RouteStart { node } => { + tracing::warn!(node, %reason, "refusing a forwarded route"); + self.route_start(node, None)?; + } + Refusable::RouteUpdate { id } => { + let node = self.routes.get(&id).ok_or(Error::ProtocolViolation)?.node; + tracing::warn!(node, route = id, %reason, "refusing a forwarded route"); + self.reach(id, None)?; + } + Refusable::Start { route, suffix } => { + let path = decoder.start(suffix)?; + let path = self.claim_path(prefix, path); + let id = decoder.last(); + tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); + self.start(id, route, path, None)?; + } + Refusable::Change { id } => { + let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; + tracing::warn!(route = claim.route, claim = id, %reason, "refusing a forwarded claim"); + claim.claim = None; + self.pending(id, None)?; + } + // Warm routes are ignored here anyway, so refusing one changes nothing. + Refusable::WarmStart { route } => { + tracing::warn!(route, %reason, "refusing a forwarded warm route"); + } + Refusable::WarmUpdate { id } => { + tracing::warn!(warm = id, %reason, "refusing a forwarded warm route"); + } + } + self.stats.refuse_route(); + Ok(()) + } + + /// The peer started a route to `node` with the next Route ID, reaching it by `reach`, + /// or `None` when this node refuses what the peer forwarded. + fn route_start(&mut self, node: route::Id, reach: Option) -> Result<(), Error> { + // The peer keeps one route per origin node; a refused reserved node is none. + if node != 0 && self.routes.values().any(|route| route.node == node) { + return Err(Error::ProtocolViolation); + } + self.held.push(self.slots.acquire()?); + let id = self.next_route; + self.next_route += 1; + self.routes.insert( + id, + WireRoute { + node, + reach: reach.clone(), + claims: BTreeMap::new(), + unnamed: BTreeSet::new(), + pending: BTreeMap::new(), + committed: None, + }, + ); + if reach.is_some() { + self.stream.route(node, reach); + } + Ok(()) + } + + /// Hand the node a batch of the route to `node`, leaving out what the limit does not + /// admit, which the node never held, so it neither starts nor ends. + fn file(&self, node: route::Id, changes: BTreeMap>, revision: Option) { + let changes = (changes.into_iter()) + .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) + .collect(); + self.stream.batch(node, changes, revision.filter(|_| self.certifies())); + } + + /// The absolute path of a claim under the stream's prefix, or `None` when it, or the + /// suffix it joins, is no path this node can hold. + fn claim_path(&self, prefix: &Path, suffix: Option) -> Option { + let path = prefix.join(&suffix?).ok()?; + self.root.join(&path).ok() } - fn reach(&mut self, id: u64, reach: route::Reach) -> Result<(), Error> { + /// The route with Route ID `id` now reaches its node by `reach`, or `None` while this + /// node refuses what the peer forwarded. A route refused before files again once + /// usable, like a route the peer ended and started over, with the claims of its last + /// floor. + fn reach(&mut self, id: u64, reach: Option) -> Result<(), Error> { let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; + // The reserved node names no route, whatever the peer forwards for it. + let reach = reach.filter(|_| route.node != 0); + let refused = route.reach.is_none(); route.reach = reach.clone(); - self.stream.route(route.node, Some(reach)); + let node = route.node; + match (refused, reach) { + (true, None) => {} + (false, None) => self.stream.route(node, None), + (true, Some(reach)) => { + let committed = route.committed.as_ref().map(|(claims, floor)| { + let claims = claims.iter().map(|(path, claim)| (path.clone(), Some(claim.clone()))); + (claims.collect(), *floor) + }); + self.stream.route(node, Some(reach)); + if let Some((claims, floor)) = committed { + self.file(node, claims, floor); + } + } + (false, Some(reach)) => self.stream.route(node, Some(reach)), + } Ok(()) } - /// A claim the peer started under `route`, assigned Announce ID `id`. - fn start(&mut self, id: u64, route: u64, path: Path, claim: node::Claim) -> Result<(), Error> { + /// A claim the peer started under `route`, assigned Announce ID `id`, at `path` unless + /// this node refused it, and `None` while this node refuses what the peer forwarded. + fn start(&mut self, id: u64, route: u64, path: Option, claim: Option) -> Result<(), Error> { let wire = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; - if wire.claims.contains_key(&path) { - return Err(Error::ProtocolViolation); + match &path { + Some(path) if wire.claims.contains_key(path) => return Err(Error::ProtocolViolation), + Some(path) => { + wire.claims.insert(path.clone(), id); + wire.change(path.clone(), claim.clone()); + } + None => { + wire.unnamed.insert(id); + } } self.held.push(self.slots.acquire()?); - wire.claims.insert(path.clone(), id); - wire.change(path.clone(), Some(claim.clone())); + // A claim without a path stays refused, whatever the peer sends for it. + let claim = claim.filter(|_| path.is_some()); self.claims.insert(id, WireClaim { route, path, claim }); Ok(()) } @@ -300,15 +415,28 @@ impl Routed { let claim = self.claims.remove(&id).ok_or(Error::ProtocolViolation)?; self.held.pop(); let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; - route.claims.remove(&claim.path); - route.change(claim.path, None); + match claim.path { + Some(path) => { + route.claims.remove(&path); + route.change(path, None); + } + None => { + route.unnamed.remove(&id); + } + } Ok(()) } - fn pending(&mut self, id: u64, change: (Path, Option)) -> Result<(), Error> { - let route = self.claims.get(&id).ok_or(Error::ProtocolViolation)?.route; - let route = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; - route.change(change.0, change.1); + /// The claim with Announce ID `id` is now `change`, pending its route's next floor. + fn pending(&mut self, id: u64, change: Option) -> Result<(), Error> { + let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; + let Some(path) = claim.path.clone() else { + // Nothing reaches the node for a claim without a path. + claim.claim = None; + return Ok(()); + }; + let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; + route.change(path, change); Ok(()) } @@ -361,12 +489,14 @@ impl Routed { ids.sort_unstable(); for id in &ids { let route = &self.routes[id]; - stream.route(route.node, Some(route.reach.clone())); + if let Some(reach) = &route.reach { + stream.route(route.node, Some(reach.clone())); + } } let certifies = self.certifies(); for id in &ids { let route = &self.routes[id]; - if let Some((claims, floor)) = &route.committed { + if let Some((claims, floor)) = route.committed.as_ref().filter(|_| route.reach.is_some()) { let changes = (claims.iter()) .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) .map(|(path, claim)| (path.clone(), Some(claim.clone()))); diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index d5fe43ffcdf..571fe33ddd7 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -77,6 +77,9 @@ //! [`Registry::report`] and [`Registry::snapshot`] read each registered [`Wire`] //! without polling the session. A retag flushes its old tier and label; dropping the guard flushes //! its final nonnegative delta. Unsupported transports contribute zero. +//! * `routes_refused` ([`Presence`]): cumulative routes and claims that lite-07 +//! sessions under the label forwarded and this node refused, each withdrawn while +//! the session stays open. Driven by [`Session::refuse_route`]. //! * `announces_peak` / `subscriptions_peak` ([`Presence`]): the most //! announcements, or subscriptions, any one session under the label held //! at once, to compare against what bounds them. A peak never @@ -293,6 +296,7 @@ struct SessionCounters { sessions_ended: AtomicU64, announces_peak: AtomicU64, subscriptions_peak: AtomicU64, + routes_refused: AtomicU64, /// Payload bytes of the sessions recording here. One lock covers the folded totals /// and the live sessions, so a readout never goes down when a session leaves. bytes: Mutex, @@ -334,6 +338,7 @@ impl SessionCounters { received_bytes, announces_peak: self.announces_peak.load(Ordering::Relaxed), subscriptions_peak: self.subscriptions_peak.load(Ordering::Relaxed), + routes_refused: self.routes_refused.load(Ordering::Relaxed), }; (presence, activity, empty) } @@ -714,6 +719,9 @@ pub struct Presence { pub wire_tx: u64, /// Cumulative transport bytes received, including transport framing. pub wire_rx: u64, + /// Cumulative routes, warm routes, and claims a lite-07 session forwarded that this + /// node refused, withdrawing each and keeping the session. + pub routes_refused: u64, } #[derive(Serialize)] @@ -731,6 +739,7 @@ struct PresenceSer { subscriptions_peak: u64, wire_tx: u64, wire_rx: u64, + routes_refused: u64, } impl From for PresenceSer { @@ -749,6 +758,7 @@ impl From for PresenceSer { received_bytes: p.received_bytes, announces_peak: p.announces_peak, subscriptions_peak: p.subscriptions_peak, + routes_refused: p.routes_refused, } } } @@ -771,6 +781,7 @@ struct PresenceDe { wire_tx: u64, #[serde(default)] wire_rx: u64, + routes_refused: u64, } impl From for Presence { @@ -787,6 +798,7 @@ impl From for Presence { received_bytes: d.received_bytes, announces_peak: d.announces_peak, subscriptions_peak: d.subscriptions_peak, + routes_refused: d.routes_refused, } } } @@ -817,6 +829,7 @@ impl Presence { self.received_bytes += other.received_bytes; self.announces_peak = self.announces_peak.max(other.announces_peak); self.subscriptions_peak = self.subscriptions_peak.max(other.subscriptions_peak); + self.routes_refused += other.routes_refused; } /// Sessions currently connected. @@ -1859,6 +1872,14 @@ impl Session { presence.peak(cap).fetch_max(held, Ordering::Relaxed); } + /// Count a route or claim the peer forwarded that this node refused. + pub(crate) fn refuse_route(&self) { + let Some(inner) = &self.inner else { return }; + let current = inner.current.lock().unwrap_or_else(PoisonError::into_inner); + let Some(presence) = ¤t.presence else { return }; + presence.routes_refused.fetch_add(1, Ordering::Relaxed); + } + /// Egress (publisher / reads) scope for a broadcast path. The path is the /// absolute broadcast name. pub(crate) fn egress(&self, path: impl Into) -> Scope { From 1e6fec8ff360e5d152b2ef4a2119724da212db93 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:08:24 -0800 Subject: [PATCH 03/16] test(net): session fixtures for refused routes and claims Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 311 +++++++++++++++++++++++++++++- 1 file changed, 310 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 650a155819b..9ac8f25c606 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10018,7 +10018,18 @@ mod tests { version: Version, session: crate::lite::test_transport::ScriptedSession, ) -> (AnnouncePrefix, origin::Consumer) { - let origin = origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(); + root_prefix_counted(version, session, Default::default()) + } + + /// [`root_prefix`], counting the session's stats into `stats`. + fn root_prefix_counted( + version: Version, + session: crate::lite::test_transport::ScriptedSession, + stats: crate::stats::Session, + ) -> (AnnouncePrefix, origin::Consumer) { + let origin = origin::Config::new(std::num::NonZeroU64::new(1).unwrap()) + .produce() + .with_stats(stats); let consumer = origin.consume(); let subscriber = Subscriber::new(SubscriberConfig { runtime: crate::time::sim(), @@ -10183,6 +10194,304 @@ mod tests { } } + /// A lite-07 ANNOUNCE_START for `suffix` under Route ID `route`. + fn claim_on(route: u64, suffix: &'static str) -> impl Fn(&mut crate::coding::Encoder<'_>) { + move |w| { + lite::AnnounceBroadcast::Active { + route, + instance: Some(crate::Instance::random()), + epoch: 0, + suffix: lite::PathRef::literal(Path::new(suffix).unwrap()), + hops: crate::Hops::new(), + cost: crate::origin::Cost::default(), + } + .encode(w, Version::Lite07) + .unwrap() + } + } + + /// A lite-07 message of type `typ` with `body`, as a newer or broken peer might send it. + fn raw(typ: u64, body: Vec) -> impl Fn(&mut crate::coding::Encoder<'_>) { + move |w| { + w.varint(typ).unwrap(); + w.varint(body.len() as u64).unwrap(); + w.slice(&body); + } + } + + /// An advertisement body: seqno 1, metric 0, `flags`, then `hops` hops, and `rest`. + fn advert_body(lead: u64, seqno: u64, flags: u8, hops: u64, rest: &[u8]) -> Vec { + let mut body = Vec::new(); + let w = &mut crate::coding::Encoder::new(&mut body, Version::Lite07.into()); + w.varint(lead).unwrap(); + w.varint(seqno).unwrap(); + w.varint(0).unwrap(); + w.u8(flags); + w.varint(hops).unwrap(); + for hop in 0..hops { + w.varint(100 + hop).unwrap(); + } + w.slice(rest); + body + } + + const ROUTE_START: u64 = 6; + const ROUTE_UPDATE: u64 = 7; + + /// Every advertisement a relay refuses as content: a flag no version defines (with the + /// field it might announce), the last seqno, and more hops than a route may cross. + fn refused_adverts(lead: u64) -> [(&'static str, Vec); 3] { + [ + ("an unknown flag", advert_body(lead, 1, 0x80, 0, &[42])), + ("the last seqno", advert_body(lead, crate::route::MAX, 0, 0, &[])), + ("33 hops", advert_body(lead, 1, 0, crate::route::MAX_HOPS as u64 + 1, &[])), + ] + } + + /// The scripted peer's two routes, each with a claim: Route ID 0 to node 7 carrying + /// `a`, and Route ID 1 to node 8 carrying `b`. + fn two_routes() -> Vec { + announce_script( + Version::Lite07, + &[ + &announce_ok(Version::Lite07, 0), + &route_start(7), + &route_start(8), + &claim_on(0, "a"), + &claim_on(1, "b"), + &floor(0, Some(0)), + &floor(1, Some(0)), + ], + ) + } + + /// A neighbour forwarding a route this relay refuses, whatever it refuses, loses only + /// that route: its claims withdraw, while the session, the neighbour's other route, + /// and that route's claims stay, and the stream keeps reading. The route comes back + /// once the neighbour forwards it usably again. + #[kio::test] + async fn a_refused_route_update_withdraws_only_that_route() { + for (what, body) in refused_adverts(0) { + let session = crate::lite::test_transport::ScriptedSession::new(two_routes()); + let log = session.log.clone(); + let (registry, stats) = counted(); + let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); + let mut cursor = consumer.routes(); + let poll = |prefix: &mut AnnouncePrefix| { + assert!(prefix.poll(&kio::Waiter::noop()).is_pending(), "{what}"); + assert!(log.closes().is_empty(), "{what}: {:?}", log.closes()); + }; + let push = |messages: &[&dyn Fn(&mut crate::coding::Encoder<'_>)]| { + session.push(&announce_script(Version::Lite07, messages)); + }; + poll(&mut prefix); + cursor.assert_next_active("a"); + cursor.assert_next_active("b"); + + // Its claims go offline, as when the neighbour ends the route. + push(&[&raw(ROUTE_UPDATE, body)]); + poll(&mut prefix); + cursor.expect_next_offline("a").await; + cursor.assert_next_wait(); + + // The stream reads on, the other route's claims with it. + push(&[&claim_on(1, "c"), &floor(1, Some(1))]); + poll(&mut prefix); + cursor.assert_next_active("c"); + + // Forwarded usably again, the route files its last floor's claims again. + push(&[&raw(ROUTE_UPDATE, advert_body(0, 2, 0, 0, &[]))]); + poll(&mut prefix); + cursor.assert_next_active("a"); + cursor.assert_next_wait(); + assert_eq!(refusals(®istry), 1, "{what}"); + } + } + + /// A stats registry and one session counting into it. + fn counted() -> (crate::stats::Registry, crate::stats::Session) { + let registry = crate::stats::Registry::new(crate::stats::Config::new()); + let session = (registry.tier(crate::stats::Tier::default())) + .session(crate::stats::SessionInfo::new(Path::from_static("peer"))); + (registry, session) + } + + /// The routes and claims `registry`'s sessions refused. + fn refusals(registry: &crate::stats::Registry) -> u64 { + let sessions = registry.snapshot().sessions(); + sessions.iter().map(|(_, presence)| presence.routes_refused).sum() + } + + /// A ROUTE_START this relay refuses still takes its Route ID, so the claims the + /// neighbour sends under it are held but never served, and the next route's ids line + /// up. A ROUTE_UPDATE it can use files the route after all, unless the route named + /// the reserved node, and ROUTE_END retires it like any other. + #[kio::test] + async fn a_refused_route_start_holds_its_route_id() { + let mut cases = refused_adverts(7).map(|(what, body)| (what, body, true)).to_vec(); + cases.push(("node 0", advert_body(0, 1, 0, 0, &[]), false)); + for (what, body, revives) in cases { + let script = announce_script( + Version::Lite07, + &[ + &announce_ok(Version::Lite07, 0), + &raw(ROUTE_START, body), + &claim_on(0, "a"), + &floor(0, Some(0)), + &route_start(8), + &claim_on(1, "b"), + &floor(1, Some(0)), + ], + ); + let session = crate::lite::test_transport::ScriptedSession::new(script); + let log = session.log.clone(); + let (mut prefix, consumer) = root_prefix(Version::Lite07, session.clone()); + let mut cursor = consumer.routes(); + let poll = |prefix: &mut AnnouncePrefix| { + assert!(prefix.poll(&kio::Waiter::noop()).is_pending(), "{what}"); + assert!(log.closes().is_empty(), "{what}: {:?}", log.closes()); + }; + poll(&mut prefix); + cursor.assert_next_active("b"); + cursor.assert_next_wait(); + + session.push(&announce_script( + Version::Lite07, + &[&raw(ROUTE_UPDATE, advert_body(0, 2, 0, 0, &[]))], + )); + poll(&mut prefix); + match revives { + true => cursor.assert_next_active("a"), + false => { + cursor.assert_next_wait(); + continue; + } + }; + session.push(&announce_script( + Version::Lite07, + &[&|w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::RouteEnd { id: 0 } + .encode(w, Version::Lite07) + .unwrap() + }], + )); + poll(&mut prefix); + cursor.expect_next_offline("a").await; + } + } + + /// The body of a lite-07 ANNOUNCE_START under Route ID 0, its path and instance + /// written raw. + fn start_body(path: &[u8], instance: &[u8]) -> Vec { + let mut body = Vec::new(); + let w = &mut crate::coding::Encoder::new(&mut body, Version::Lite07.into()); + w.varint(0).unwrap(); + w.varint(0).unwrap(); + w.bytes(path).unwrap(); + w.varint(0).unwrap(); + w.bytes(instance).unwrap(); + w.bool(false); + w.varint(0).unwrap(); + w.varint(0).unwrap(); + body + } + + const ANNOUNCE_START: u64 = 0; + const ANNOUNCE_END: u64 = 1; + const ANNOUNCE_UPDATE: u64 = 2; + const ANNOUNCE_RESTART: u64 = 3; + + /// A claim this relay refuses withdraws only that claim. One refused before its path + /// still takes its Announce ID, so the ids after it line up and its end is no + /// violation; one refused by a later change goes until an ANNOUNCE_RESTART names + /// all of it again. + #[kio::test] + async fn a_refused_claim_withdraws_only_that_claim() { + let session = crate::lite::test_transport::ScriptedSession::new(two_routes()); + let log = session.log.clone(); + let (registry, stats) = counted(); + let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); + let mut cursor = consumer.routes(); + let poll = |prefix: &mut AnnouncePrefix| { + assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); + assert!(log.closes().is_empty(), "{:?}", log.closes()); + }; + let push = |messages: &[&dyn Fn(&mut crate::coding::Encoder<'_>)]| { + session.push(&announce_script(Version::Lite07, messages)); + }; + poll(&mut prefix); + cursor.assert_next_active("a"); + cursor.assert_next_active("b"); + + // Ids 2 and 3: an invalid path, and an invalid instance. Then `a` (id 0) changes + // to an empty Lite-06 Chain, and `c` (id 4) arrives after them. + push(&[ + &raw(ANNOUNCE_START, start_body(b"x//y", b"i")), + &raw(ANNOUNCE_START, start_body(b"d", b"")), + &raw(ANNOUNCE_UPDATE, vec![0, 0, 0]), + &claim_on(0, "c"), + &floor(0, Some(1)), + ]); + poll(&mut prefix); + cursor.assert_next_active("c"); + cursor.assert_next_ended("a"); + cursor.assert_next_wait(); + assert_eq!(refusals(®istry), 3); + + // An update names nothing a refused claim lacks; a restart names all of it. + let restart = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Restart { + id: 0, + instance: Some(crate::Instance::random()), + epoch: 1, + cost: Default::default(), + hops: Default::default(), + } + .encode(w, Version::Lite07) + .unwrap() + }; + push(&[&raw(ANNOUNCE_UPDATE, vec![3, 5]), &floor(0, Some(2))]); + poll(&mut prefix); + cursor.assert_next_wait(); + push(&[&restart, &floor(0, Some(3))]); + poll(&mut prefix); + cursor.assert_next_active("a"); + + // The refused claims end like any other, and `c`'s end still finds it. + push(&[ + &raw(ANNOUNCE_END, vec![2]), + &raw(ANNOUNCE_END, vec![3]), + &raw(ANNOUNCE_END, vec![4]), + &floor(0, Some(4)), + ]); + poll(&mut prefix); + cursor.assert_next_ended("c"); + cursor.assert_next_wait(); + } + + /// Only the neighbour's own framing and state close the session: a refused route + /// message naming a Route ID or Announce ID the neighbour never opened still does. + #[kio::test] + async fn an_unknown_id_still_closes_the_session() { + let unknown: [(&str, Vec, u64); 3] = [ + ("a refused update of an unknown route", refused_adverts(5)[0].1.clone(), ROUTE_UPDATE), + ("a refused change of an unknown claim", vec![9, 0, 0], ANNOUNCE_UPDATE), + ("a refused claim under an unknown route", { + let mut body = start_body(b"x//y", b"i"); + body[7] = 5; + body + }, ANNOUNCE_START), + ]; + for (what, body, typ) in unknown { + let mut script = two_routes(); + script.extend(announce_script(Version::Lite07, &[&raw(typ, body)])); + let (mut prefix, _consumer) = + root_prefix(Version::Lite07, crate::lite::test_transport::ScriptedSession::new(script)); + let err = kio::wait(|waiter| prefix.poll_run(waiter)).await.unwrap_err(); + assert!(matches!(err, Error::ProtocolViolation), "{what}: {err}"); + } + } + #[kio::test] async fn a_duplicate_synced_is_refused() { const VERSION: Version = Version::Lite07; From 93feb3a8372d9286c47110de859469f1df3b3326 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:11:31 -0800 Subject: [PATCH 04/16] test(net): a route at the hop limit is never forwarded Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 8b202c32a18..4a0b8a3780d 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1747,6 +1747,47 @@ mod tests { })) } + /// A route that already crossed as many nodes as one may is not forwarded, since this + /// node's hop would take it past what any peer can decode, so a session never fails to + /// encode one. One hop fewer forwards at exactly the limit. + #[test] + fn a_route_at_the_hop_limit_is_not_forwarded() { + use crate::route::MAX_HOPS; + for (hops, forwarded) in [(MAX_HOPS - 1, Some(MAX_HOPS)), (MAX_HOPS, None)] { + let relay = Node::new(2, 2); + let to_b = relay.answer(Link::default(), Path::empty()).unwrap(); + let from_a = relay.receive(Link::default(), Path::empty()).unwrap(); + from_a.route( + 5, + Some(Reach::Finite(Advert { + seqno: 1, + metric: 1, + down_only: false, + rtt: None, + hops: (100..100 + hops as u64).collect(), + })), + ); + // A route goes on the wire only with a claim it shows, once that is served. + let live = claim(&Instance::random(), 0); + from_a.batch(5, vec![(p("live"), Some(live.clone()))], Some(0)); + from_a.serving(5, &p("live"), &live); + let sent: Vec<_> = (outputs(&to_b).into_iter()) + .filter_map(|output| match output { + Output::Route { + id: 5, + reach: Some(Reach::Finite(advert)), + } => Some(advert), + _ => None, + }) + .collect(); + assert_eq!(sent.iter().map(|advert| advert.hops.len()).max(), forwarded, "{hops} hops"); + for advert in sent { + let start = crate::lite::AnnounceBroadcast::RouteStart { node: 5, advert }; + crate::coding::Encode::encode_bytes(&start, crate::lite::Version::Lite07).unwrap(); + } + } + } + /// A relay's own seqno advances only when a session over which another node /// learned a path to it closes: a client that forwards nothing costs no ROUTE. #[test] From 2d7c1f931b3cdd143d1fee305887782152b1fb13 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:11:58 -0800 Subject: [PATCH 05/16] feat(stats): count the routes and claims a node refuses Co-Authored-By: Claude Opus 5.5 --- doc/concept/stats.md | 8 +++++++- js/stats/src/frame.ts | 1 + rs/moq-stats/src/aggregate.rs | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 531c1a07874..48079a28fa8 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -411,7 +411,7 @@ should prefer the canonical name and fall back to the legacy one. "sessions_refused": { "refused": 0, "unavailable": 0, "request": 0, "forbidden": 0, "membership": 0, "expired": 0 }, "sessions_failed": { "idle_timeout": 1, "transport": 0, "application": 0, "expired": 0 }, "announces_peak": 40, "subscriptions_peak": 900, - "wire_tx": 2048, "wire_rx": 1024 + "wire_tx": 2048, "wire_rx": 1024, "routes_refused": 0 } } ``` @@ -481,6 +481,12 @@ goes down while the label has a session; in the totals and presence tracks it never goes down while the track lives. Summing nodes or labels takes the largest. +`routes_refused` counts the routes and claims that the label's moq-lite-07 +sessions forwarded and the relay refused, such as a route with a flag this +release does not know. The relay withdraws each one and keeps the session, so a +neighbour on a newer release costs only that route. A rising count during a +rollout means the releases disagree on route content. + ### Counters Every counter is a cumulative, monotonic unsigned integer. A rate is the diff --git a/js/stats/src/frame.ts b/js/stats/src/frame.ts index a9bb12aa08c..39b717e3f7e 100644 --- a/js/stats/src/frame.ts +++ b/js/stats/src/frame.ts @@ -85,6 +85,7 @@ export const PresenceSchema = legacy( subscriptions_peak: counter, wire_tx: counter, wire_rx: counter, + routes_refused: counter, }), ); diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index d4c5319ec1a..78e3a79a084 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -357,6 +357,7 @@ impl Mergeable for Presence { || self.received_bytes < prev.received_bytes || self.announces_peak < prev.announces_peak || self.subscriptions_peak < prev.subscriptions_peak + || self.routes_refused < prev.routes_refused } } From b3c58ebd705c1c0be15a127849e48ae872c33a2a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:26:07 -0800 Subject: [PATCH 06/16] feat(js/net): a refused forwarded route withdraws the route, not the session Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/announce.test.ts | 22 ++-- js/net/src/lite/announce.ts | 154 +++++++++++++++++++----- js/net/src/lite/corpus.test.ts | 10 +- js/net/src/lite/instance.ts | 22 ++++ js/net/src/lite/subscriber.test.ts | 51 +++++++- js/net/src/lite/subscriber.ts | 187 +++++++++++++++++++++-------- js/net/src/path.ts | 7 ++ js/stats/api/index.api.md | 8 ++ rs/moq-net/src/lite/announce.rs | 26 +++- rs/moq-net/src/lite/compress.rs | 25 +++- rs/moq-net/src/lite/subscriber.rs | 31 +++-- rs/moq-net/src/route/node.rs | 6 +- 12 files changed, 426 insertions(+), 123 deletions(-) diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 94f00636027..79390fe8695 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -532,23 +532,25 @@ test("AnnounceHistory rejects a keep longer than the base", () => { test("AnnounceHistory keeps segments split at an escaped slash", () => { const history = new AnnounceHistory(); history.start({ suffix: Path.from("a\\/b") }); - // The kept `a\` is half an escape, which the rest completes or nothing does. + // The kept `a\` is half an escape, which the rest completes, or nothing does and the path is + // refused, its id assigned all the same. const suffix = history.start({ suffix: Path.from("c"), pathBase: { distance: 1n, keep: 1 } }); expect(suffix).toBe(Path.from("a\\/c")); - expect(() => history.start({ suffix: Path.empty(), pathBase: { distance: 1n, keep: 1 } })).toThrow( - ProtocolViolation, - ); + expect(history.start({ suffix: Path.empty(), pathBase: { distance: 1n, keep: 1 } })).toBeUndefined(); + expect(history.next).toBe(3n); }); -test("AnnounceHistory rejects a resolved path past the byte limit", () => { +test("AnnounceHistory refuses a resolved path past the byte limit", () => { const history = new AnnounceHistory(); history.start({ suffix: Path.from("x".repeat(Path.MAX_BYTES - 2)) }); - // `head/y` is exactly the limit, and one byte more is refused. + // `head/y` is exactly the limit, and one byte more is refused as content: the id is assigned, + // a base naming it is refused too, and its end is no violation. const suffix = history.start({ suffix: Path.from("y"), pathBase: { distance: 1n, keep: 1 } }); - expect(suffix.length).toBe(Path.MAX_BYTES); - expect(() => history.start({ suffix: Path.from("yz"), pathBase: { distance: 1n, keep: 1 } })).toThrow( - ProtocolViolation, - ); + expect(suffix?.length).toBe(Path.MAX_BYTES); + expect(history.start({ suffix: Path.from("yz"), pathBase: { distance: 1n, keep: 1 } })).toBeUndefined(); + expect(history.start({ suffix: Path.from("z"), pathBase: { distance: 1n, keep: 1 } })).toBeUndefined(); + expect(history.end(2n)).toBeUndefined(); + expect(() => history.end(2n)).toThrow(ProtocolViolation); }); test("a keep without a base is a violation on draft-07", async () => { diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index a4b5bebd8c0..84995f6fce6 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -4,7 +4,7 @@ import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; -import { decodeRoute, encodeRoute } from "./instance.ts"; +import { decodeForwarded, decodeRoute, encodeRoute, utf8 } from "./instance.ts"; import * as Message from "./message.ts"; import { hasAnnounceCompression, @@ -129,7 +129,26 @@ export type AnnounceBroadcast = /** Lite07+ ANNOUNCE_FLOOR: closes the batch of claims under the route with this id, certifying * the stream's claims for it across the requested prefix as the origin's at `revision`, or * nothing when `revision` is undefined. */ - | { status: "floor"; id: bigint; revision?: bigint }; + | { status: "floor"; id: bigint; revision?: bigint } + /** Lite07+: a route message whose content, which the sender only forwarded, this node refuses. + * The receiver withdraws what it names and keeps the session. Decoded only as far as keeps the + * stream's ids in step; never sent. */ + | { status: "refused"; message: Refusable; reason: Refusal }; + +/** What a refused route message still tells the receiver, so the ids that follow it resolve. */ +export type Refusable = + /** ROUTE_START: assigns the next Route ID to the origin node `node`. */ + | { kind: "routeStart"; node: bigint } + /** ROUTE_UPDATE of the route with this Route ID. */ + | { kind: "routeUpdate"; id: bigint } + /** ANNOUNCE_START under Route ID `route`: assigns the next Announce ID, to `path`, or to no path + * when the path itself was refused. */ + | { kind: "start"; route: bigint; path?: { suffix: Path.Valid; pathBase?: Base } } + /** ANNOUNCE_RESTART of the claim with this Announce ID. */ + | { kind: "change"; id: bigint }; + +/** Why a node refuses content a peer forwarded: well framed, but nothing it can use. */ +export type Refusal = "node" | "seqno" | "flags" | "hops" | "path" | "instance"; /** * A message the subscriber sends on a lite-07 Announce Stream after its ANNOUNCE_REQUEST. @@ -271,20 +290,71 @@ async function encodeAdvert(w: Writer, advert: Advert) { for (const hop of advert.hops) await w.u62(hop); } -async function decodeAdvert(r: Reader): Promise { +/** Decode an advertisement, refusing one whose seqno no origin advertises, whose flags this node + * does not know, or that crossed too many hops. The rest of the body is skipped, since an unknown + * flag may announce a field ahead of the hops. Mirrors rs/moq-net. */ +async function decodeAdvert(r: Reader): Promise { + const refuse = async (reason: Refusal) => { + await r.readAll(); + return reason; + }; const seqno = await r.u62(); - if (seqno === ROUTE_MAX) throw new ProtocolViolation("a route advertised the last seqno"); + if (seqno === ROUTE_MAX) return refuse("seqno"); const metric = await r.u62(); const flags = await r.u8(); - if (flags & ~(DOWN_ONLY | RTT)) throw new ProtocolViolation(`unknown route flags: ${flags}`); + if (flags & ~(DOWN_ONLY | RTT)) return refuse("flags"); const rtt = flags & RTT ? await r.u62() : undefined; const count = await r.u62(); - if (count > BigInt(ROUTE_MAX_HOPS)) throw new ProtocolViolation(`a route crossed ${count} nodes`); + if (count > BigInt(ROUTE_MAX_HOPS)) return refuse("hops"); const hops: bigint[] = []; for (let i = 0n; i < count; i++) hops.push(await r.u62()); return { seqno, metric, downOnly: (flags & DOWN_ONLY) !== 0, rtt, hops }; } +/** Refuse a route to the reserved Node ID 0, skipping the rest of its body. */ +async function refuseNode(r: Reader): Promise { + await r.readAll(); + return "node"; +} + +/** Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded, or + * undefined for a path this node refuses. */ +async function decodeForwardedPath(r: Reader): Promise<{ suffix: Path.Valid; pathBase?: Base } | undefined> { + const distance = await r.u62(); + const keep = await r.u53(); + // A distance of 0 names nothing, so a keep without a base is a violation. + if (distance === 0n && keep !== 0) throw new ProtocolViolation("announce keep without a base"); + const text = utf8(await r.read(await r.u53())); + const suffix = text === undefined ? undefined : Path.tryDecode(text); + if (suffix === undefined) return undefined; + return distance === 0n ? { suffix } : { suffix, pathBase: { distance, keep } }; +} + +/** A lite-07 ANNOUNCE_START: its suffix, Route ID, instance and epoch, and cost. */ +async function decodeStart(r: Reader, version: Version): Promise { + const path = await decodeForwardedPath(r); + const route = await r.u62(); + const refused = async (reason: Refusal): Promise => { + await r.readAll(); + return { status: "refused", message: { kind: "start", route, path }, reason }; + }; + if (!path) return refused("path"); + const { instance, epoch } = await decodeForwarded(r); + if (instance === undefined) return refused("instance"); + return { status: "active", ...path, route, instance, epoch, cost: await decodeRouteCost(r, version) }; +} + +/** A lite-07 ANNOUNCE_RESTART: its Announce ID, instance and epoch, and cost. */ +async function decodeRestart(r: Reader, version: Version): Promise { + const id = await r.u62(); + const { instance, epoch } = await decodeForwarded(r); + if (instance === undefined) { + await r.readAll(); + return { status: "refused", message: { kind: "change", id }, reason: "instance" }; + } + return { status: "restart", id, instance, epoch, cost: await decodeRouteCost(r, version) }; +} + // lite-06 message body (no discriminator; the type is carried outside the length prefix). async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: Version) { switch (msg.status) { @@ -338,6 +408,7 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: // The pre-lite-06 path-form retraction has no place on lite-06. throw new Error("ended-by-path not supported for this version"); case "skipped": + case "refused": throw new Error("decode-only announce type cannot be encoded"); } } @@ -374,6 +445,7 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { case "ended": throw new Error("ended-by-path not supported for this version"); case "skipped": + case "refused": throw new Error("decode-only announce type cannot be encoded"); } } @@ -381,6 +453,8 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): Promise { switch (typ) { case ANNOUNCE_START: { + // Lite-07's claims carry what the sender forwarded, which a receiver may refuse. + if (hasRoutes(version)) return decodeStart(r, version); const path = await decodePath(r, version); const route = hasRoutes(version) ? { route: await r.u62() } : {}; const served = await decodeRoute(r, version); @@ -400,6 +474,7 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P case ANNOUNCE_RESTART: { // Unknown before lite-07: skipped below like any other. if (!hasAnnounceRestart(version)) break; + if (hasRoutes(version)) return decodeRestart(r, version); const id = await r.u62(); const served = await decodeRoute(r, version); return { status: "restart", id, ...served, cost: await decodeRouteCost(r, version) }; @@ -407,12 +482,19 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P case ROUTE_START: { if (!hasRoutes(version)) break; const node = await r.u62(); - if (node === 0n) throw new ProtocolViolation("Node ID 0 names no origin"); - return { status: "routeStart", node, advert: await decodeAdvert(r) }; + const advert = node === 0n ? await refuseNode(r) : await decodeAdvert(r); + if (typeof advert === "string") + return { status: "refused", message: { kind: "routeStart", node }, reason: advert }; + return { status: "routeStart", node, advert }; } - case ROUTE_UPDATE: + case ROUTE_UPDATE: { if (!hasRoutes(version)) break; - return { status: "routeUpdate", id: await r.u62(), advert: await decodeAdvert(r) }; + const id = await r.u62(); + const advert = await decodeAdvert(r); + if (typeof advert === "string") + return { status: "refused", message: { kind: "routeUpdate", id }, reason: advert }; + return { status: "routeUpdate", id, advert }; + } case ROUTE_HOLD: if (!hasRoutes(version)) break; return { status: "routeHold", id: await r.u62() }; @@ -540,36 +622,41 @@ export async function decodeAnnounceReplyMaybe(r: Reader, version: Version): Pro export class AnnounceHistory { #next = 0n; #live = new Map(); + // The live ids whose path was refused, which a base names only to be refused too. + #refused = new Set(); /** The Announce ID the next ANNOUNCE_START is assigned. */ get next(): bigint { return this.#next; } - /** An ANNOUNCE_START: resolve its suffix and assign it the next id. */ - start(msg: { suffix: Path.Valid; pathBase?: Base }): Path.Valid { - let suffix = msg.suffix; - if (msg.pathBase) { - const base = this.#live.get(this.#next - msg.pathBase.distance); - if (base === undefined) throw new ProtocolViolation(`announce base ${msg.pathBase.distance} is not live`); - const head = Path.parts(base); - const keep = msg.pathBase.keep; - if (keep > head.length) throw new ProtocolViolation(`announce keeps ${keep} of ${head.length} segments`); - // Joined as text before parsing: a kept segment may end in the `\` of an escaped `/`. - const kept = head.slice(0, keep); - if (suffix !== Path.empty()) kept.push(suffix); - try { - suffix = Path.from(kept.join("/")); - } catch (cause) { - throw new ProtocolViolation("announce keeps half an escape", { cause }); - } - const error = Path.check(suffix); - if (error) throw new ProtocolViolation(error); - } - this.#live.set(this.#next++, suffix); + /** An ANNOUNCE_START: assign it the next id, and resolve its suffix, or undefined when that + * suffix, or the base it names, is no path this node can hold. */ + start(msg: { suffix: Path.Valid; pathBase?: Base } | undefined): Path.Valid | undefined { + const suffix = msg && this.#resolve(msg); + const id = this.#next++; + if (suffix === undefined) this.#refused.add(id); + else this.#live.set(id, suffix); return suffix; } + #resolve(msg: { suffix: Path.Valid; pathBase?: Base }): Path.Valid | undefined { + const suffix = msg.suffix; + if (!msg.pathBase) return suffix; + const id = this.#next - msg.pathBase.distance; + if (this.#refused.has(id)) return undefined; + const base = this.#live.get(id); + if (base === undefined) throw new ProtocolViolation(`announce base ${msg.pathBase.distance} is not live`); + const head = Path.parts(base); + const keep = msg.pathBase.keep; + if (keep > head.length) throw new ProtocolViolation(`announce keeps ${keep} of ${head.length} segments`); + // Joined as text before parsing: a kept segment may end in the `\` of an escaped `/`. A path + // that does not resolve is refused as content, as rs/moq-net does. + const kept = head.slice(0, keep); + if (suffix !== Path.empty()) kept.push(suffix); + return Path.tryDecode(kept.join("/")); + } + /** The suffix of the live announcement `id`, which an update or restart references. */ get(id: bigint): Path.Valid { const live = this.#live.get(id); @@ -577,8 +664,9 @@ export class AnnounceHistory { return live; } - /** An ANNOUNCE_END: retire the id, returning its suffix. */ - end(id: bigint): Path.Valid { + /** An ANNOUNCE_END: retire the id, returning its suffix, or undefined if it was refused. */ + end(id: bigint): Path.Valid | undefined { + if (this.#refused.delete(id)) return undefined; const live = this.get(id); this.#live.delete(id); return live; diff --git a/js/net/src/lite/corpus.test.ts b/js/net/src/lite/corpus.test.ts index e10e5ae3936..70803e30476 100644 --- a/js/net/src/lite/corpus.test.ts +++ b/js/net/src/lite/corpus.test.ts @@ -26,7 +26,7 @@ interface Case { budget?: number; /** `client_setup`: the one version the server accepts. */ server?: string; - expect: "decoded" | "refused"; + expect: "decoded" | "refused" | "withdrawn"; } const VERSIONS = new Map(Object.values(Version).map((v) => [versionName(v), v])); @@ -38,10 +38,11 @@ function version(name: string): Version { } // Decode `r` the way a session does, returning how to encode what it decoded. -async function decode(c: Case, r: Reader, v: Version): Promise<(w: Writer) => Promise> { +async function decode(c: Case, r: Reader, v: Version): Promise<((w: Writer) => Promise) | "withdrawn"> { switch (c.message) { case "announce": { const msg = await decodeAnnounceBroadcast(r, v); + if (msg.status === "refused") return "withdrawn"; return (w) => encodeAnnounceBroadcast(w, msg, v); } case "announce_reply": { @@ -69,7 +70,7 @@ async function decode(c: Case, r: Reader, v: Version): Promise<(w: Writer) => Pr async function outcome(c: Case): Promise { const v = version(c.version); const r = new Reader(undefined, Buffer.from(c.hex, "hex"), v); - let encode: (w: Writer) => Promise; + let encode: ((w: Writer) => Promise) | "withdrawn"; try { encode = await decode(c, r, v); } catch (err) { @@ -77,6 +78,7 @@ async function outcome(c: Case): Promise { return err instanceof UnexpectedEnd && !isDecodeError(err) ? "short" : "refused"; } if (!(await r.done())) return "left bytes"; + if (encode === "withdrawn") return encode; return Buffer.from(await Writer.collect(v, encode)).toString("hex"); } @@ -88,6 +90,6 @@ test("the corpus has cases", () => { for (const c of cases) { test(c.name, async () => { - expect(await outcome(c)).toBe(c.expect === "decoded" ? c.hex : "refused"); + expect(await outcome(c)).toBe(c.expect === "decoded" ? c.hex : c.expect); }); } diff --git a/js/net/src/lite/instance.ts b/js/net/src/lite/instance.ts index 7c6eec902a9..1caeec38ab9 100644 --- a/js/net/src/lite/instance.ts +++ b/js/net/src/lite/instance.ts @@ -67,6 +67,28 @@ async function encode(w: Writer, instance: Instance.Valid): Promise { await w.bool(Instance.isMinted(instance)); } +/** Decode lite-07's instance and epoch, with `instance` undefined when they are well framed but + * the instance is not one this node can use, which withdraws only the claim naming it. Mirrors + * rs/moq-net. */ +export async function decodeForwarded(r: Reader): Promise<{ instance?: Instance.Valid; epoch: bigint }> { + const bytes = await r.read(await r.u53()); + const minted = await r.bool(); + const epoch = await r.u62(); + const text = utf8(bytes); + if (text === undefined || text.length === 0 || !text.isWellFormed()) return { epoch }; + const named = Instance.parse(text); + return { instance: minted ? Mint.mint(named) : named, epoch }; +} + +/** `bytes` as UTF-8 text, or undefined when they are not UTF-8. */ +export function utf8(bytes: Uint8Array): string | undefined { + try { + return decoder.decode(bytes); + } catch { + return undefined; + } +} + function decode(bytes: Uint8Array, minted: boolean): Instance.Valid { const named = Instance.parse(decoder.decode(bytes)); return minted ? Mint.mint(named) : named; diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 32929730997..546d379c367 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4005,9 +4005,11 @@ async function routedStream(options?: RouteOptions, getStats?: () => Promise encodeAnnounceBroadcast(peer, msg, version); - return { announced, send, replies }; + // Bytes as a newer or broken peer might write them. + const raw = (bytes: number[]) => stream.inbound.enqueue(new Uint8Array(bytes)); + return { announced, send, raw, replies }; }; - const { announced, send, replies } = await another(0); + const { announced, send, raw, replies } = await another(0); // The TRACK request a consume of `path` opens as its `index`th stream. const trackRequest = async (path: Path.Valid, index: number, pin?: Instance.Valid) => { const consumed = subscriber.consume(path, pin); @@ -4021,9 +4023,52 @@ async function routedStream(options?: RouteOptions, getStats?: () => Promise { + const { subscriber, announced, send, raw } = await routedStream(); + const [a, b, c, d] = ["live/a", "live/b", "live/c", "live/d"].map((path) => Path.from(path)); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: a, route: 0n, instance }); + await send({ status: "active", suffix: b, route: 1n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: b, kind: "start" }); + + // ROUTE_UPDATE of Route ID 0 with a flag no version defines, and the field it might announce. + raw([7, 5, 0, 1, 0, 0x80, 42]); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "offline" }); + + // Announce ID 2: an ANNOUNCE_START under Route ID 1 whose path has an empty segment. + raw([0, 13, 0, 0, 4, ...new TextEncoder().encode("x//y"), 1, 1, 0x69, 0, 0, 0]); + await send({ status: "endedId", id: 2n }); + await send({ status: "active", suffix: c, route: 1n, instance }); + await send({ status: "floor", id: 1n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: c, kind: "start" }); + + // Forwarded usably again, the route's claims come back. + await send({ status: "routeUpdate", id: 0n, advert: ADVERT }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + + // ROUTE_START to the reserved node takes Route ID 2, and its claims never show. + raw([6, 5, 0, 1, 0, 0, 0]); + await send({ status: "active", suffix: d, route: 2n, instance }); + await send({ status: "floor", id: 2n, revision: 0n }); + await send({ status: "endedId", id: 3n }); + await send({ status: "floor", id: 1n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: c, kind: "end" }); + + announced.close(); + subscriber.close(); +}); + // Two streams carry one origin's claims, and a withdrawal reaches one first. Of the floors that // certify a claim, the newest decides whether it exists, so the lagging stream's copy is held back // rather than selected and refused: also once the newer stream's route ends. diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 54c314bf2d2..0c6d11e0ebf 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -67,8 +67,11 @@ import { AnnounceInit, AnnounceOk, AnnounceRequest, + type Base, decodeAnnounceBroadcastMaybe, encodeAnnounceReply, + type Refusable, + type Refusal, ROUTE_MAX_HOPS, } from "./announce.ts"; import { Datagram as DatagramMessage, Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; @@ -706,6 +709,11 @@ export class Subscriber { // copy from it. Tracked even for announces we skip as reflected, since the sender // doesn't know we skipped. const history = new AnnounceHistory(); + // Only lite-07 refuses a forwarded path, and it never reaches here. + const known = (path: Path.Valid | undefined): Path.Valid => { + if (path === undefined) throw new ProtocolViolation("a refused path before lite-07"); + return path; + }; // Receive announce updates (for Draft03+, this includes initial state) for (;;) { @@ -749,7 +757,7 @@ export class Subscriber { switch (announce.status) { case "active": { - const suffix = hasAnnounceId(this.version) ? history.start(announce) : announce.suffix; + const suffix = hasAnnounceId(this.version) ? known(history.start(announce)) : announce.suffix; // The wire names the suffix beneath the interest prefix; the consumer // sees the covered path from the session root. path = Path.join(prefix, suffix); @@ -766,7 +774,7 @@ export class Subscriber { break; case "endedId": // Resolve and retire the id; an unknown or retired id is a protocol violation. - path = Path.join(prefix, history.end(announce.id)); + path = Path.join(prefix, known(history.end(announce.id))); active = false; break; case "update": @@ -964,11 +972,15 @@ export class Subscriber { type Route = { id: bigint; node: bigint; - advert: Advert; + // Undefined while this node refuses what the peer forwarded: no route at all, its claims + // offline, until a ROUTE_UPDATE it can use. Mirrors rs/moq-net. + advert: Advert | undefined; // ROUTE_HOLD: no usable path to the origin, until the next ROUTE_UPDATE. held: boolean; // The Announce ID of each live claim, by path: one per path, checked by lookup. claims: Map; + // The Announce IDs of its live claims whose path this node refused. + unnamed: Set; // How the claims differ from the view's, applied whole at the next floor. pending: Map; // The claims as of the last floor, and the revision it certified them at, if any. @@ -986,7 +998,9 @@ export class Subscriber { // Each live route and claim holds one of the session's announce slots. const routes = new SlotMap(this.#announces); - const claims = new SlotMap(this.#announces); + // A claim refused has no `claim` until a restart names all of it, and one refused before its + // path has no `path` either, and never will. + const claims = new SlotMap(this.#announces); // The origin nodes with a live route here: at most one route each. const nodes = new Set(); // The routes whose committed claims include each path. @@ -1066,12 +1080,14 @@ export class Subscriber { // A route is no route at all once it crossed as many nodes as one may, or crossed us: a copy // of our own route, come back. - const usable = (wire: Route) => - wire.advert.hops.length < ROUTE_MAX_HOPS && !wire.advert.hops.includes(this.hop); + const usable = (wire: Route): wire is Route & { advert: Advert } => + wire.advert !== undefined && + wire.advert.hops.length < ROUTE_MAX_HOPS && + !wire.advert.hops.includes(this.hop); // A claim through this session: its cost plus the route's metric, past this session's link // cost of 1 as Rust's default, and unavailable while the route is held. - const candidate = (wire: Route, claim: Claim): WireRoute => + const candidate = (wire: Route & { advert: Advert }, claim: Claim): WireRoute => this.#drained( WireRoute.normalize({ instance: claim.instance, @@ -1170,6 +1186,99 @@ export class Subscriber { } }; + // The peer started a route to `node` with the next Route ID, reaching it by `advert`, or + // undefined when this node refuses what the peer forwarded. + const startRoute = async (node: bigint, advert: Advert | undefined) => { + // The peer keeps one route per origin node; a refused reserved node is none. + if (node !== 0n && nodes.has(node)) throw new ProtocolViolation(`second route to node ${node}`); + const id = nextRoute++; + const view: Route["view"] = { prefix, hidden, revision: undefined, claims: new Map() }; + const wire: Route = { + id, + node, + advert, + held: false, + claims: new Map(), + unnamed: new Set(), + pending: new Map(), + view, + shown: new Map(), + carrier: { + view, + // Another route's floor or end runs this: a closed reader is about to let go. + reshow: () => { + if (!closing && announced.closed.peek() === undefined) show(wire, new Map(), true); + }, + }, + owner: {}, + }; + routes.set(id, wire); + const started = sample(); + if (node !== 0n) nodes.add(node); + this.#floors.join(node, wire.carrier); + // The link's first read lands before the route's first claims show, as Rust + // reads the link before it serves them: a listing reports a path's route when + // it turns present and not again for an RTT, so a read landing after would + // never reach it. + if (started) await race([firstRead, announced.closed]); + }; + + // A claim the peer started under `routeId` with the next Announce ID, at `path` unless this + // node refused it, and `claim` unless it refuses what the peer forwarded. + const startClaim = ( + routeId: bigint, + path: { suffix: Path.Valid; pathBase?: Base } | undefined, + claim: Claim | undefined, + ) => { + const wire = route(routeId); + const suffix = history.start(path); + const id = history.next - 1n; + // The interest prefix can push a joined path past the limits: refused too. + const full = suffix === undefined ? undefined : Path.join(prefix, suffix); + const named = full !== undefined && Path.check(full) === undefined ? full : undefined; + if (named === undefined) { + if (claim) console.warn(`refusing a forwarded claim: route=${routeId} claim=${id} reason=path`); + claims.set(id, { route: routeId }); + wire.unnamed.add(id); + return; + } + if (wire.claims.has(named)) throw new ProtocolViolation(`route ${routeId} claims ${named} twice`); + claims.set(id, { route: routeId, path: named, claim }); + wire.claims.set(named, id); + if (claim) change(routeId, named, claim); + }; + + // Apply a message whose content, which the peer only forwarded, this node refuses: withdraw + // the route or claim it names, and keep the stream. Mirrors rs/moq-net. + const refused = (message: Refusable, reason: Refusal) => { + switch (message.kind) { + case "routeStart": + console.warn(`refusing a forwarded route: node=${message.node} reason=${reason}`); + return startRoute(message.node, undefined); + case "routeUpdate": { + const wire = route(message.id); + console.warn(`refusing a forwarded route: node=${wire.node} route=${message.id} reason=${reason}`); + wire.advert = undefined; + refreshRoute(wire, true); + return; + } + case "start": + console.warn( + `refusing a forwarded claim: route=${message.route} claim=${history.next} reason=${reason}`, + ); + return startClaim(message.route, message.path, undefined); + case "change": { + const live = claimOf(message.id); + console.warn( + `refusing a forwarded claim: route=${live.route} claim=${message.id} reason=${reason}`, + ); + live.claim = undefined; + if (live.path) change(live.route, live.path, undefined); + return; + } + } + }; + // A draining peer ranks its routes last from the GOAWAY itself. const stopDrain = this.#goaway?.changed(() => { if (announced.closed.peek() !== undefined) return; @@ -1185,42 +1294,13 @@ export class Subscriber { if (msg instanceof Error) throw msg; switch (msg.status) { - case "routeStart": { - if (nodes.has(msg.node)) throw new ProtocolViolation(`second route to node ${msg.node}`); - const id = nextRoute++; - const view: Route["view"] = { prefix, hidden, revision: undefined, claims: new Map() }; - const wire: Route = { - id, - node: msg.node, - advert: msg.advert, - held: false, - claims: new Map(), - pending: new Map(), - view, - shown: new Map(), - carrier: { - view, - // Another route's floor or end runs this: a closed reader is about to let go. - reshow: () => { - if (!closing && announced.closed.peek() === undefined) show(wire, new Map(), true); - }, - }, - owner: {}, - }; - routes.set(id, wire); - const started = sample(); - nodes.add(msg.node); - this.#floors.join(msg.node, wire.carrier); - // The link's first read lands before the route's first claims show, as Rust - // reads the link before it serves them: a listing reports a path's route when - // it turns present and not again for an RTT, so a read landing after would - // never reach it. - if (started) await race([firstRead, announced.closed]); + case "routeStart": + await startRoute(msg.node, msg.advert); break; - } case "routeUpdate": { const wire = route(msg.id); - wire.advert = msg.advert; + // The reserved node names no route, whatever the peer forwards for it. + wire.advert = wire.node === 0n ? undefined : msg.advert; wire.held = false; refreshRoute(wire, !usable(wire)); break; @@ -1234,7 +1314,7 @@ export class Subscriber { case "routeEnd": { // Every claim under the route goes with it: offline, not withdrawn. const wire = route(msg.id); - for (const id of wire.claims.values()) { + for (const id of [...wire.claims.values(), ...wire.unnamed]) { history.end(id); claims.delete(id); } @@ -1295,21 +1375,14 @@ export class Subscriber { if (msg.route === undefined) throw new ProtocolViolation("lite-07 claim without a route"); if (msg.instance === undefined) throw new ProtocolViolation("lite-07 claim without an instance"); - const wire = route(msg.route); - const path = Path.join(prefix, history.start(msg)); - const error = Path.check(path); - if (error) throw new ProtocolViolation(error); - if (wire.claims.has(path)) - throw new ProtocolViolation(`route ${msg.route} claims ${path} twice`); - const id = history.next - 1n; const claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; - claims.set(id, { route: msg.route, path, claim }); - wire.claims.set(path, id); - change(msg.route, path, claim); + startClaim(msg.route, msg, claim); break; } case "update": { const live = claimOf(msg.id); + // An update names none of what a refused claim lacks. + if (!live.claim || !live.path) break; live.claim = { ...live.claim, cost: msg.cost ?? Cost.zero }; change(live.route, live.path, live.claim); break; @@ -1318,15 +1391,25 @@ export class Subscriber { const live = claimOf(msg.id); if (msg.instance === undefined) throw new ProtocolViolation("lite-07 restart without an instance"); + // A claim without a path stays refused, whatever the peer sends for it. + if (!live.path) break; live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; change(live.route, live.path, live.claim); break; } + case "refused": + await refused(msg.message, msg.reason); + break; case "endedId": { const live = claimOf(msg.id); history.end(msg.id); claims.delete(msg.id); - route(live.route).claims.delete(live.path); + const wire = route(live.route); + if (!live.path) { + wire.unnamed.delete(msg.id); + break; + } + wire.claims.delete(live.path); change(live.route, live.path, undefined); break; } diff --git a/js/net/src/path.ts b/js/net/src/path.ts index e0495608f3e..bd11bebd11b 100644 --- a/js/net/src/path.ts +++ b/js/net/src/path.ts @@ -108,6 +108,13 @@ export function decode(raw: string): Valid { return encode(from(raw)); } +/** Like {@link decode}, but `undefined` for an invalid path rather than a throw. @internal */ +export function tryDecode(raw: string): Valid | undefined { + if (spelling(raw) !== undefined) return undefined; + const path = raw as Valid; + return check(path) === undefined ? path : undefined; +} + /** * Validate a path before writing it to the wire, enforcing {@link MAX_PARTS} and a 4096 byte limit. * diff --git a/js/stats/api/index.api.md b/js/stats/api/index.api.md index 0e9c3b6fe99..4fa11dc3fda 100644 --- a/js/stats/api/index.api.md +++ b/js/stats/api/index.api.md @@ -26,6 +26,7 @@ export const BucketFrameSchema: ZodMiniObject<{ announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault, ZodMiniPi announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault, Zod announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault, ZodMiniPref announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault) -> Content { fn decode_path(r: &mut Decoder<'_>) -> Content { let bytes = r.bytes()?; let path = std::str::from_utf8(bytes).ok().and_then(|path| Path::new(path).ok()); - Ok(path.filter(|path| path.parts().count() <= Path::MAX_PARTS).ok_or(Refusal::Path)) + Ok(path + .filter(|path| path.parts().count() <= Path::MAX_PARTS) + .ok_or(Refusal::Path)) } /// Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded. @@ -325,7 +327,6 @@ fn decode_suffix(r: &mut Decoder<'_>) -> Content { Ok(decode_path(r)?.map(|rest| PathRef { base, keep, rest })) } - /// Encode an ANNOUNCE_FLOOR body: the revision plus one, or 0 to certify nothing. fn encode_floor(w: &mut Encoder<'_>, id: u64, revision: Option) -> Result<(), EncodeError> { w.varint(id)?; @@ -757,7 +758,8 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { body.rest(); refused(Refusal::Node) } - _ => decode_advert(&mut body)?.map_or_else(refused, |advert| Self::RouteStart { node, advert }), + _ => decode_advert(&mut body)? + .map_or_else(refused, |advert| Self::RouteStart { node, advert }), } } ROUTE_UPDATE if version.has_routes() => { @@ -1512,7 +1514,11 @@ mod tests { message: Refusable::WarmUpdate { id: 7 }, reason, }; - assert_eq!(decode(&[13, 4, 7, 4, 0, 0]).unwrap(), refused(Refusal::Carrier), "no hops"); + assert_eq!( + decode(&[13, 4, 7, 4, 0, 0]).unwrap(), + refused(Refusal::Carrier), + "no hops" + ); assert_eq!( decode(&[13, 7, 7, 4, 3, 3, 5, 9, 6]).unwrap(), refused(Refusal::Carrier), @@ -2157,10 +2163,18 @@ mod tests { let refused = [ (forge(ROUTE_START, hops(over, over)), start(1), Refusal::Hops(over)), (forge(ROUTE_START, advert(0, 0, 1, 0)), start(0), Refusal::Node), - (forge(ROUTE_START, advert(1, route::MAX, 1, 0)), start(1), Refusal::Seqno), + ( + forge(ROUTE_START, advert(1, route::MAX, 1, 0)), + start(1), + Refusal::Seqno, + ), // A flag no version defines, which may change what follows it. (forge(ROUTE_START, advert(1, 0, 1, 4)), start(1), Refusal::Flags(4)), - (forge(ROUTE_START, advert(1, 0, 1, 0x86)), start(1), Refusal::Flags(0x86)), + ( + forge(ROUTE_START, advert(1, 0, 1, 0x86)), + start(1), + Refusal::Flags(0x86), + ), // ROUTE_UPDATE's leading field is the Route ID, so 0 is a valid id here. ( forge(ROUTE_UPDATE, advert(0, route::MAX, 1, 0)), diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index 80e0654146f..1fe9bb105eb 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -34,7 +34,10 @@ impl Live { if distance == 0 { return Ok(None); } - self.next.checked_sub(distance).map(Some).ok_or(Error::ProtocolViolation) + self.next + .checked_sub(distance) + .map(Some) + .ok_or(Error::ProtocolViolation) } } @@ -318,7 +321,10 @@ mod tests { #[test] fn a_new_stream_has_no_base() { let mut decoder = AnnounceDecoder::default(); - assert!(matches!(decoder.start(Some(path(1, 0, "a"))), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(1, 0, "a"))), + Err(Error::ProtocolViolation) + )); } #[test] @@ -329,9 +335,15 @@ mod tests { decoder.end(0).unwrap(); // Id 0 is two back, and retired. - assert!(matches!(decoder.start(Some(path(2, 1, "x"))), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(2, 1, "x"))), + Err(Error::ProtocolViolation) + )); // Three back was never assigned. - assert!(matches!(decoder.start(Some(path(3, 0, "x"))), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(3, 0, "x"))), + Err(Error::ProtocolViolation) + )); // Id 1 is still live, so reusing it works after the END. let suffix = decoder.start(Some(path(1, 1, "x"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "b/x"); @@ -341,7 +353,10 @@ mod tests { fn rejects_a_keep_longer_than_the_base() { let mut decoder = AnnounceDecoder::default(); decoder.start(Some(literal("a/b"))).unwrap(); - assert!(matches!(decoder.start(Some(path(1, 3, "x"))), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(1, 3, "x"))), + Err(Error::ProtocolViolation) + )); } /// A resolved path past the limits is content the peer forwarded: refused, its id diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 9ac8f25c606..a2c0548cb42 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10244,7 +10244,10 @@ mod tests { [ ("an unknown flag", advert_body(lead, 1, 0x80, 0, &[42])), ("the last seqno", advert_body(lead, crate::route::MAX, 0, 0, &[])), - ("33 hops", advert_body(lead, 1, 0, crate::route::MAX_HOPS as u64 + 1, &[])), + ( + "33 hops", + advert_body(lead, 1, 0, crate::route::MAX_HOPS as u64 + 1, &[]), + ), ] } @@ -10474,19 +10477,29 @@ mod tests { #[kio::test] async fn an_unknown_id_still_closes_the_session() { let unknown: [(&str, Vec, u64); 3] = [ - ("a refused update of an unknown route", refused_adverts(5)[0].1.clone(), ROUTE_UPDATE), + ( + "a refused update of an unknown route", + refused_adverts(5)[0].1.clone(), + ROUTE_UPDATE, + ), ("a refused change of an unknown claim", vec![9, 0, 0], ANNOUNCE_UPDATE), - ("a refused claim under an unknown route", { - let mut body = start_body(b"x//y", b"i"); - body[7] = 5; - body - }, ANNOUNCE_START), + ( + "a refused claim under an unknown route", + { + let mut body = start_body(b"x//y", b"i"); + body[7] = 5; + body + }, + ANNOUNCE_START, + ), ]; for (what, body, typ) in unknown { let mut script = two_routes(); script.extend(announce_script(Version::Lite07, &[&raw(typ, body)])); - let (mut prefix, _consumer) = - root_prefix(Version::Lite07, crate::lite::test_transport::ScriptedSession::new(script)); + let (mut prefix, _consumer) = root_prefix( + Version::Lite07, + crate::lite::test_transport::ScriptedSession::new(script), + ); let err = kio::wait(|waiter| prefix.poll_run(waiter)).await.unwrap_err(); assert!(matches!(err, Error::ProtocolViolation), "{what}: {err}"); } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 4a0b8a3780d..2ae519452a3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1780,7 +1780,11 @@ mod tests { _ => None, }) .collect(); - assert_eq!(sent.iter().map(|advert| advert.hops.len()).max(), forwarded, "{hops} hops"); + assert_eq!( + sent.iter().map(|advert| advert.hops.len()).max(), + forwarded, + "{hops} hops" + ); for advert in sent { let start = crate::lite::AnnounceBroadcast::RouteStart { node: 5, advert }; crate::coding::Encode::encode_bytes(&start, crate::lite::Version::Lite07).unwrap(); From 2077fc84965ad8e0fb960fe2fd39e98a1d17e395 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:28:47 -0800 Subject: [PATCH 07/16] docs(draft): route content a relay refuses withdraws, and finish the quest Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 38 +++++++---- js/net/src/lite/announce.test.ts | 29 +++++--- quest/m1/refuse-route.md | 67 ------------------- quest/m2/cluster-routes/flapping-publisher.md | 1 - quest/m2/cluster-routes/request-loop.md | 1 - quest/m2/cluster-routes/switch.md | 5 +- quest/m2/lite07-finalize.md | 1 - 7 files changed, 48 insertions(+), 94 deletions(-) delete mode 100644 quest/m1/refuse-route.md diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 55bbf3b7164..0177fb3e1d3 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -135,7 +135,7 @@ Any broadcasts and subscriptions are transparently proxied by the CDN behind the A Broadcast is a collection of Tracks named by a path. This corresponds to a MoqTransport's "track namespace". A path is a sequence of non-empty segments joined by `/` (0x2F), so it never starts or ends with `/` or contains two in a row; the empty path has no segments and prefixes every path. -A `\` (0x5C) escapes the `\` or `/` that follows it, as a MoqTransport namespace field holding either is spelled; a path with any other `\` is a PROTOCOL_VIOLATION. +A `\` (0x5C) escapes the `\` or `/` that follows it, as a MoqTransport namespace field holding either is spelled; a path with any other `\` is a PROTOCOL_VIOLATION, or in a claim refused (see [Refused Content](#refused-content)). A continuous stream is named by an Instance, carried on its claims (see [ANNOUNCE_START](#announce-start)): a path and an Instance name one Broadcast, and a publisher MUST NOT serve different content under the same pair. Within one Instance, group sequences never restart and never name two different groups, so claims sharing it serve the same bytes; two Instances are independent streams, never stitched together. A publisher that starts its group sequences over, such as a restarted process, uses a new Instance, while replicas of the same content share one. @@ -443,7 +443,7 @@ An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping The subscriber MUST close the session with a PROTOCOL_VIOLATION if it receives a message referencing a Route ID or Announce ID that was never assigned or already retired, a ROUTE_START for an origin node that already has a live route on the stream, an ANNOUNCE_START for a prefix its route already claims, or any message before ANNOUNCE_OK. When the stream is closed, the subscriber MUST assume that every route and claim on it is gone. -A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, including one resolved by [Compression](#announce-compression). +A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, except in a claim, which it refuses instead (see [Refused Content](#refused-content)), including one resolved by [Compression](#announce-compression). A claim covers a path when its prefix is a leading run of the path's segments; matching is per path segment, so a prefix never matches half a segment, and equality is byte-by-byte within each segment. A publisher answering a request stream presents, per route, each claim inside the requested prefix and only the most specific claim at or above it, at the requested prefix itself (an empty suffix), which is exactly the covered set the subscriber may see. When that most specific covering claim changes, such as when it is withdrawn and a broader one still covers the request, the publisher moves the presented claim to the new one with an ANNOUNCE_UPDATE or ANNOUNCE_RESTART, and ends it only when no claim of the route covers the requested prefix. @@ -472,7 +472,20 @@ The receiver resolves the base against the claims live when the message arrives, The base need not share the new claim's route. A publisher MAY always send a `Path Base` of 0. -The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments, or the resolved path breaks its own rules. +The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, or a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments. +A resolved path that breaks its own rules, or a base whose path the subscriber refused, refuses the claim (see [Refused Content](#refused-content)). + +#### Refused Content {#refused-content} +Route messages carry content the publisher may only have forwarded: a route's Node ID, Seqno, Flags, and Hops, a claim's path, Instance, and Lite-06 Chain, and a warm route's path, Instance, Carrier Index, and Hops. +A newer version or release may send such content that an older receiver cannot use, so a receiver that refuses it withdraws only the route or claim it describes and keeps the session, as BGP treats a malformed attribute as a withdrawal ({{?RFC7606}}). +This specification marks each such value "refused"; the receiver skips the rest of the message's body, and SHOULD log and count each refusal so a refused route is never silently missing. + +- A refused ROUTE_START still assigns its Route ID, and a refused ROUTE_UPDATE keeps it; either leaves the receiver no route, so its claims go offline as on ROUTE_END. The messages naming the Route ID stay valid, and a later ROUTE_UPDATE the receiver can use makes the route usable again, with the claims its last floor certified. A route to Node ID 0 stays refused. +- A refused ANNOUNCE_START still assigns its Announce ID, and a refused ANNOUNCE_UPDATE or ANNOUNCE_RESTART keeps it; the claim is withdrawn at its route's next floor. A claim whose path was refused stays refused; any other comes back with a later ANNOUNCE_RESTART the receiver can use. +- A refused WARM_START still assigns its Warm ID. + +Only the publisher's own framing and stream state close the session: a message that does not decode within its Message Length, such as an RTT flag without its RTT, and a message naming an id that was never assigned or already retired, or breaking the stream's order, as above. +A relay never forwards content a peer would refuse for the relay's own part in it: a route that arrives with 32 Hops is unusable, so the relay's Hop never takes one past 32. #### Hidden Paths {#hidden} A claim is hidden from a request when a segment of its path below the requested prefix starts with `.` (0x2E). @@ -966,7 +979,7 @@ moq-lite-07 uses the leading-ones encoding of [moqt]: the number of leading 1 bi Most messages are prefixed with a variable-length integer indicating the number of bytes in the message payload that follows. This length field does not include the length of the varint length itself. -A message on the Setup Stream or any bidirectional stream that fails to decode, such as one whose fields disagree with its Message Length or carry a value this specification forbids, MUST close the session with a PROTOCOL_VIOLATION. +A message on the Setup Stream or any bidirectional stream that fails to decode, such as one whose fields disagree with its Message Length or carry a value this specification forbids, MUST close the session with a PROTOCOL_VIOLATION, except content a route message forwards, which the receiver refuses (see [Refused Content](#refused-content)). Resetting only the stream would let the peer repeat the violation on the next one. A stream that ends or is reset before its message completes only terminates that stream, except on the Setup and Auth Streams. A GROUP or FRAME that fails to decode resets only its Group or Fetch Stream, and a STREAM_TYPE the negotiated version does not define resets only its stream (see [STREAM_TYPE](#stream_type)). @@ -1224,18 +1237,18 @@ Set to 0x6 to indicate a ROUTE_START message. **Node ID**: The origin node, as the publisher names it on this session. -0 is a PROTOCOL_VIOLATION. +0 is refused (see [Refused Content](#refused-content)). **Seqno**: The origin's sequence number of the route, which only the origin advances. -2^64-1 is a PROTOCOL_VIOLATION. +2^64-1 is refused. **Metric**: The publisher's metric to the origin node, 0 when it is the origin. **Flags**: A bit field: 0x1 when the route is down-only (see [Routing](#routing)), and 0x2 when an RTT follows. -Any other bit set is a PROTOCOL_VIOLATION. +Any other bit set is refused, rather than ignored: a later version may give a bit a meaning that changes the route, or a field ahead of the Hops. **RTT**: The publisher's RTT to the origin node in milliseconds (see [Routing](#routing)), 0 when it is the origin. @@ -1243,7 +1256,7 @@ Present only when Flags has 0x2 set, and absent when the publisher has none. **Hop Count**: The number of Hop IDs that follow. -More than 32 is a PROTOCOL_VIOLATION. +More than 32 is refused. **Hop ID**: The route's Hops (see [Routing](#routing)), in order, the publisher's own last. @@ -1384,7 +1397,7 @@ A claim never changes its Route ID. **Instance**: The stream the claim serves (see [Broadcast](#broadcast)): non-empty UTF-8 text. -An empty or non-UTF-8 Instance is a PROTOCOL_VIOLATION. +An empty or non-UTF-8 Instance is refused (see [Refused Content](#refused-content)). A relay forwards the Instance unchanged. **Minted**: @@ -1405,7 +1418,7 @@ A relay forwards it unchanged; the cost of a path through the claim is the Annou **Lite-06 Chain**: The moq-lite-06 Hop ID list a claim arrived with where its origin node learned it over moq-lite-06, encoded as that list: a count, then each Hop ID. -A publisher sends it, as the message's last field, only to a peer that sent the [Lite-06 Chain Parameter](#lite06-chain-parameter), and only when the claim has one; a received empty chain is a PROTOCOL_VIOLATION. +A publisher sends it, as the message's last field, only to a peer that sent the [Lite-06 Chain Parameter](#lite06-chain-parameter), and only when the claim has one; a received chain that is empty, or names a Hop ID twice, is refused. A relay forwards it unchanged, and translating the claim for a moq-lite-06 peer sends it followed by a Hop ID standing for the claim's origin node. A node MUST NOT serve a claim whose chain names itself, by its own Hop ID or one standing for its Node ID, or whose translated chain would exceed its hop limit. @@ -1545,11 +1558,11 @@ What pulling the copy cost the publisher, 0 when it is the copy's carrier (see [ **Carrier Index**: The position of the copy's carrier among the Hop IDs that follow, 0 for the first. -A value not below `Hop Count` is a PROTOCOL_VIOLATION. +A value not below `Hop Count` is refused (see [Refused Content](#refused-content)). **Hop Count**: The number of Hop IDs that follow. -More than 32 is a PROTOCOL_VIOLATION. +More than 32 is refused. **Hop ID**: The copy's Hops (see [Warm Routes](#warm-routes)), in order, the publisher's own last. @@ -2485,6 +2498,7 @@ An application-specific payload. - FRAME leads with `Payload Length`, the payload size plus one, in place of the trailing `Message Length`; 0 is malformed. - A publisher SHOULD drop a datagram rather than block when the transport has no room for it, where it previously sent one only if the congestion controller could transmit it immediately. - Added an optional `Subscriber Max Age` to FETCH, encoded as milliseconds plus one with zero meaning absent. It can tighten, but never relax, `Publisher Max Age`. +- A receiver refuses forwarded route content it cannot use, such as an unknown route flag, more than 32 Hops, or an invalid claim path or Instance, withdrawing only that route or claim rather than closing the session ([Refused Content](#refused-content)). ## moq-lite-06 diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 79390fe8695..e20cbde1848 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -424,7 +424,7 @@ test("the route layer round-trips on draft-07", async () => { ).rejects.toThrow(); }); -test("the route layer refuses the last seqno and Node ID 0", async () => { +test("the route layer refuses the last seqno, Node ID 0, unknown flags, and too many hops", async () => { const v = Version.DRAFT_07; const last = { seqno: ROUTE_MAX, metric: 0n, downOnly: false, rtt: undefined, hops: [] }; await expect( @@ -453,24 +453,35 @@ test("the route layer refuses the last seqno and Node ID 0", async () => { await w.u62(0n); await w.u8(0); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, zero, v), v)).rejects.toThrow(ProtocolViolation); + // What the sender only forwarded is refused, the message decoded all the same, so the receiver + // withdraws the route and keeps the stream. + const decode = (wire: Uint8Array) => decodeAnnounceBroadcast(new Reader(undefined, wire, v), v); + expect(await decode(zero)).toEqual({ + status: "refused", + message: { kind: "routeStart", node: 0n }, + reason: "node", + }); const maxed = await forged(async (w) => { await w.u62(1n); await w.u62(ROUTE_MAX); await w.u62(0n); await w.u8(0); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, maxed, v), v)).rejects.toThrow(ProtocolViolation); - // A flag no version defines, and the RTT flag without its field. - for (const flags of [4, 2]) { - const flagged = await forged(async (w) => { + expect(await decode(maxed)).toEqual({ + status: "refused", + message: { kind: "routeStart", node: 1n }, + reason: "seqno", + }); + // A flag no version defines is refused, and the RTT flag without its field is malformed. + const flagged = (flags: number) => + forged(async (w) => { await w.u62(1n); await w.u62(0n); await w.u62(0n); await w.u8(flags); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, flagged, v), v)).rejects.toThrow(); - } + expect(await decode(await flagged(4))).toMatchObject({ status: "refused", reason: "flags" }); + await expect(decode(await flagged(2))).rejects.toThrow(); // More hops than a route may carry, refused before reading them. const overHops = await forged(async (w) => { await w.u62(1n); @@ -479,7 +490,7 @@ test("the route layer refuses the last seqno and Node ID 0", async () => { await w.u8(0); await w.u53(ROUTE_MAX_HOPS + 1); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, overHops, v), v)).rejects.toThrow(ProtocolViolation); + expect(await decode(overHops)).toMatchObject({ status: "refused", reason: "hops" }); }); test("the subscriber's replies round-trip on draft-07", async () => { diff --git a/quest/m1/refuse-route.md b/quest/m1/refuse-route.md deleted file mode 100644 index 2dcde8f008c..00000000000 --- a/quest/m1/refuse-route.md +++ /dev/null @@ -1,67 +0,0 @@ -# [M] A relay that refuses a forwarded route drops the route, not the session - -## Goal - -A lite-07 relay that refuses a route because of its content, which the -neighbour only forwarded, withdraws that route and keeps the session. Such -content includes the path, its hop count, the flags, and the claims under the -route. The neighbour's other routes and claims stay live. Only a fault in the -neighbour's own framing or per-session state closes the session, such as a -Route ID it never opened or a message out of order on its stream. A relay -that cannot encode a route for a peer, such as one past that peer's hop -limit, does not forward it, and the session stays open. - -This matters for moq.pro, which rolls out with relays on the newest and the -previous release side by side. A newer release that adds a check, raises a -limit, or sets a new flag would otherwise make older relays close sessions to -neighbours that did nothing wrong, and the closes cascade across the mesh. BGP -hit the same problem and answered with "treat as withdraw" -([RFC 7606](https://www.rfc-editor.org/rfc/rfc7606)). - -## Plan - -Decided 2026-11-13 (maintainer, /quest-plan on the BGP failure modes): - -- ✅ Drop the route and keep the session; close only on the neighbour's own - framing or state (rejected: closing as today and relying on a newer relay - to send nothing an older peer's version lacks, which misses a newer relay - adding a check against content older relays forward; skipping unknown - flags and fields and keeping the route, which accepts content the relay - cannot interpret). -- ✅ m1 (rejected: m2 beside the switch). It lands first on today's lite-07 - codec, so it waits on nothing in m2, and - [the switch](/quest/m2/cluster-routes/switch.md) keeps the rule when it - rewrites the codec. -- ✅ It gates the lite-07 cut - ([lite07-finalize](/quest/m2/lite07-finalize.md) requires it), since once - lite-07 is published, changing how it reacts to a refused route needs - lite-08 (rejected: shipping lite-07 without it). -- ✅ Test within one binary (rejected: a nightly job that builds the - `release` relay and runs it in a mesh with `main` relays, a second build - for a check the fixture already covers by injecting refusals). - -Today `decode_advert` in `rs/moq-net/src/lite/announce.rs` fails the decode, -and so closes the session, on an unknown flag bit or more than 32 hops, and -`encode_advert` errors on a route it cannot encode. Sort each check in -`lite/announce.rs` and `lite/subscriber/routed.rs` into content the -neighbour forwarded or state the neighbour owns. A refusal must still be -loud: log it with the route and the reason, and count it, so a refused route -is never silently missing. - -Decide in the PR whether an unknown flag bit refuses the route or is -ignored, given that the flags byte is the obvious place for a newer release -to extend a route. Mirror the rule in `js/net` and in the draft's error -handling for route messages. - -Fixtures on real sessions, with mocked time: a neighbour forwards a route -that the relay refuses, once for each refusal the codec and router make, -and the session, the neighbour's other routes, and their claims stay live -while the refused route's claims withdraw; a route past the next peer's hop -limit is not forwarded, and that session stays open; an unknown Route ID -still closes the session. - -## Related - -- [Routes and announces](/quest/m2/cluster-routes/README.md) - the route messages this rule covers -- [A dual-homed publisher stays reachable while one link flaps](/quest/m2/cluster-routes/flapping-publisher.md) - another failure mode from the same BGP comparison -- [A request loop heals once routes settle](/quest/m2/cluster-routes/request-loop.md) - another failure mode from the same BGP comparison diff --git a/quest/m2/cluster-routes/flapping-publisher.md b/quest/m2/cluster-routes/flapping-publisher.md index fcdc6320b8a..c8d4f8cabc0 100644 --- a/quest/m2/cluster-routes/flapping-publisher.md +++ b/quest/m2/cluster-routes/flapping-publisher.md @@ -68,5 +68,4 @@ Work: ## Related -- [A relay that refuses a forwarded route drops the route, not the session](/quest/m1/refuse-route.md) - another failure mode from the same BGP comparison - [A request loop heals once routes settle](/quest/m2/cluster-routes/request-loop.md) - another failure mode from the same BGP comparison diff --git a/quest/m2/cluster-routes/request-loop.md b/quest/m2/cluster-routes/request-loop.md index dd8fbb562db..8dd0bff1fd1 100644 --- a/quest/m2/cluster-routes/request-loop.md +++ b/quest/m2/cluster-routes/request-loop.md @@ -42,5 +42,4 @@ pending request (the front in `rs/moq-net/src/model/front.rs`, and how ## Related -- [A relay that refuses a forwarded route drops the route, not the session](/quest/m1/refuse-route.md) - another failure mode from the same BGP comparison - [A dual-homed publisher stays reachable while one link flaps](/quest/m2/cluster-routes/flapping-publisher.md) - another failure mode from the same BGP comparison diff --git a/quest/m2/cluster-routes/switch.md b/quest/m2/cluster-routes/switch.md index 692083d0496..6a28c8bfb25 100644 --- a/quest/m2/cluster-routes/switch.md +++ b/quest/m2/cluster-routes/switch.md @@ -116,9 +116,8 @@ by misparse, not by version. Relays pinned to `main` roll together, as each earlier lite-07 wire change did. A relay refuses a route its neighbour only forwarded by withdrawing that -route, not by closing the session -([refuse-route](/quest/m1/refuse-route.md)). If that rule has landed, the -rewritten codec and the path router keep it; if not, it follows this switch. +route, not by closing the session (the draft's Refused Content section). The +rewritten codec and the path router keep that rule. Public API: none expected. Wire: lite-07-wip only. diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index 25560fed44f..c198e137478 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -131,7 +131,6 @@ transcript freezes at the cut and a flaw in it can only be fixed on the wire. - [@moq/net by start](/quest/m2/time-addressing/js.md) - JS speaks the same lite-07 wire - [A group a subscriber gives up on is always reported](/quest/m2/group-drops.md) - may add SUBSCRIBE_DROP for a stopped group; leaves this list if it takes the no-wire path - [Prove cache-aware routing](/quest/m2/cache-routing-proof.md) - rewrites lite-07's one-hop warm routes and `Serves` for routes that propagate -- [A relay that refuses a forwarded route drops the route, not the session](/quest/m1/refuse-route.md) - lite-07 tolerates relays on two releases before it is published (maintainer, 2026-11-13) ## Related From 6ed657b777713fcff70fd59814e5835021cdc3ea Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:44:49 -0800 Subject: [PATCH 08/16] test(net): cover a refused ANNOUNCE_RESTART Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 8 +++++++- rs/moq-net/src/stats.rs | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a2c0548cb42..d27402f7b67 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10460,12 +10460,18 @@ mod tests { poll(&mut prefix); cursor.assert_next_active("a"); + // A restart to an empty instance is refused like the update was. + push(&[&raw(ANNOUNCE_RESTART, vec![0, 0, 0, 0, 0]), &floor(0, Some(4))]); + poll(&mut prefix); + cursor.assert_next_ended("a"); + assert_eq!(refusals(®istry), 4); + // The refused claims end like any other, and `c`'s end still finds it. push(&[ &raw(ANNOUNCE_END, vec![2]), &raw(ANNOUNCE_END, vec![3]), &raw(ANNOUNCE_END, vec![4]), - &floor(0, Some(4)), + &floor(0, Some(5)), ]); poll(&mut prefix); cursor.assert_next_ended("c"); diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index 571fe33ddd7..210f2b1812e 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -79,7 +79,7 @@ //! its final nonnegative delta. Unsupported transports contribute zero. //! * `routes_refused` ([`Presence`]): cumulative routes and claims that lite-07 //! sessions under the label forwarded and this node refused, each withdrawn while -//! the session stays open. Driven by [`Session::refuse_route`]. +//! the session stays open. //! * `announces_peak` / `subscriptions_peak` ([`Presence`]): the most //! announcements, or subscriptions, any one session under the label held //! at once, to compare against what bounds them. A peak never @@ -1873,6 +1873,7 @@ impl Session { } /// Count a route or claim the peer forwarded that this node refused. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn refuse_route(&self) { let Some(inner) = &self.inner else { return }; let current = inner.current.lock().unwrap_or_else(PoisonError::into_inner); From 86489426cdb9e2b8abb75747e81537768eca1216 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:24:09 -0800 Subject: [PATCH 09/16] fix(net): refuse root claims, refused floors, and resolved paths like the rest A root claim's empty path is a path in JS, a refused route's floors stay out of the node's shared floors as Rust's node never holds it, and a claim whose path resolves past the limits is logged and counted like any other refusal. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscriber.test.ts | 61 ++++++++++++++++++++++++ js/net/src/lite/subscriber.ts | 40 ++++++++++++---- rs/moq-net/src/lite/subscriber.rs | 42 +++++++++++++++- rs/moq-net/src/lite/subscriber/routed.rs | 9 +++- 4 files changed, 139 insertions(+), 13 deletions(-) diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 546d379c367..bcac1845b07 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4069,6 +4069,67 @@ test("a lite-07 refused route or claim withdraws only itself", async () => { subscriber.close(); }); +// A claim at the session root has the empty path, which is a path all the same: its updates, +// restarts, and end apply as any other's. +test("a lite-07 claim at the root updates and ends", async () => { + const { subscriber, announced, send } = await routedStream(); + const root = Path.empty(); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: root, route: 0n, instance: Instance.parse("i") }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "start" }); + + await send({ status: "restart", id: 0n, instance: Instance.parse("j") }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "restart" }); + + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "end" }); + + // Its path is free again, so starting it anew is no duplicate. + await send({ status: "active", suffix: root, route: 0n, instance: Instance.parse("k") }); + await send({ status: "floor", id: 0n, revision: 3n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "start" }); + + announced.close(); + subscriber.close(); +}); + +// A route this subscriber refused is no view of its node: a newer floor on it hides nothing +// another stream carries, as rs/moq-net's node never holds the route. +test("a lite-07 refused route's floors hide no other stream's claims", async () => { + const { subscriber, announced, send, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const instance = Instance.parse("i"); + // One varint byte, so the raw message below spells it. + const node = 7n; + + await send({ status: "routeStart", node, advert: ADVERT }); + await send({ status: "active", suffix: Path.from("live/x"), route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x"), kind: "start" }); + + // ROUTE_START to the node with a flag no version defines, then a newer floor without live/x. + live.raw([6, 5, 7, 1, 0, 0x80, 42]); + await live.send({ status: "floor", id: 0n, revision: 5n }); + // A claim of another node on that stream shows once it read all of the above. + await live.send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); + await live.send({ status: "floor", id: 1n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + // What the first stream shows next is its own new claim, not live/x withdrawn. + await send({ status: "active", suffix: Path.from("live/y"), route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + // Two streams carry one origin's claims, and a withdrawal reaches one first. Of the floors that // certify a claim, the newest decides whether it exists, so the lagging stream's copy is held back // rather than selected and refused: also once the newer stream's route ends. diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 0c6d11e0ebf..222615ea4be 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1215,7 +1215,8 @@ export class Subscriber { routes.set(id, wire); const started = sample(); if (node !== 0n) nodes.add(node); - this.#floors.join(node, wire.carrier); + // A refused route's floors stay its own, as Rust's node never holds the route. + if (advert !== undefined) this.#floors.join(node, wire.carrier); // The link's first read lands before the route's first claims show, as Rust // reads the link before it serves them: a listing reports a path's route when // it turns present and not again for an RTT, so a read landing after would @@ -1248,6 +1249,21 @@ export class Subscriber { if (claim) change(routeId, named, claim); }; + // The route now reaches its node by `advert`, or undefined while this node refuses what the + // peer forwarded. A refused route leaves the node's floors, as an ended one does, and joins + // again once usable, with what its last floor certified, as rs/moq-net files it again. + const reach = (wire: Route, advert: Advert | undefined) => { + const joined = wire.advert !== undefined; + // The reserved node names no route, whatever the peer forwards for it. + wire.advert = wire.node === 0n ? undefined : advert; + if (joined && wire.advert === undefined) this.#floors.leave(wire.node, wire.carrier); + if (joined || wire.advert === undefined) return; + this.#floors.join(wire.node, wire.carrier); + const others = this.#floors.floor(wire.node, wire.carrier); + show(wire, new Map(), true); + if (others) this.#consumes.revalidate(); + }; + // Apply a message whose content, which the peer only forwarded, this node refuses: withdraw // the route or claim it names, and keep the stream. Mirrors rs/moq-net. const refused = (message: Refusable, reason: Refusal) => { @@ -1258,7 +1274,7 @@ export class Subscriber { case "routeUpdate": { const wire = route(message.id); console.warn(`refusing a forwarded route: node=${wire.node} route=${message.id} reason=${reason}`); - wire.advert = undefined; + reach(wire, undefined); refreshRoute(wire, true); return; } @@ -1273,7 +1289,7 @@ export class Subscriber { `refusing a forwarded claim: route=${live.route} claim=${message.id} reason=${reason}`, ); live.claim = undefined; - if (live.path) change(live.route, live.path, undefined); + if (live.path !== undefined) change(live.route, live.path, undefined); return; } } @@ -1299,9 +1315,8 @@ export class Subscriber { break; case "routeUpdate": { const wire = route(msg.id); - // The reserved node names no route, whatever the peer forwards for it. - wire.advert = wire.node === 0n ? undefined : msg.advert; wire.held = false; + reach(wire, msg.advert); refreshRoute(wire, !usable(wire)); break; } @@ -1337,7 +1352,12 @@ export class Subscriber { const wire = route(msg.id); // A floor that certifies less keeps what the route's last one certified. const certified = wire.view.revision; - if (certified !== undefined && (msg.revision === undefined || msg.revision < certified)) + const joined = wire.advert !== undefined; + if ( + joined && + certified !== undefined && + (msg.revision === undefined || msg.revision < certified) + ) this.#floors.regress(wire.node, wire.carrier); const touched = new Map(); for (const [path, claim] of wire.pending) { @@ -1365,7 +1385,7 @@ export class Subscriber { wire.view.revision = msg.revision; // Every other route to the node is shown anew first; with one, this route's // unchanged claims may change state too. - const others = this.#floors.floor(wire.node, wire.carrier); + const others = joined && this.#floors.floor(wire.node, wire.carrier); show(wire, touched, others); // Another view of the node may now cover a consumed path differently. if (others) this.#consumes.revalidate(); @@ -1382,7 +1402,7 @@ export class Subscriber { case "update": { const live = claimOf(msg.id); // An update names none of what a refused claim lacks. - if (!live.claim || !live.path) break; + if (live.claim === undefined || live.path === undefined) break; live.claim = { ...live.claim, cost: msg.cost ?? Cost.zero }; change(live.route, live.path, live.claim); break; @@ -1392,7 +1412,7 @@ export class Subscriber { if (msg.instance === undefined) throw new ProtocolViolation("lite-07 restart without an instance"); // A claim without a path stays refused, whatever the peer sends for it. - if (!live.path) break; + if (live.path === undefined) break; live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; change(live.route, live.path, live.claim); break; @@ -1405,7 +1425,7 @@ export class Subscriber { history.end(msg.id); claims.delete(msg.id); const wire = route(live.route); - if (!live.path) { + if (live.path === undefined) { wire.unnamed.delete(msg.id); break; } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index d27402f7b67..abf6ff7b9dd 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10386,10 +10386,16 @@ mod tests { /// The body of a lite-07 ANNOUNCE_START under Route ID 0, its path and instance /// written raw. fn start_body(path: &[u8], instance: &[u8]) -> Vec { + based_start_body(0, 0, path, instance) + } + + /// An ANNOUNCE_START body on route 0 whose suffix keeps `keep` segments of the claim + /// `base` Announce IDs back. + fn based_start_body(base: u64, keep: u64, path: &[u8], instance: &[u8]) -> Vec { let mut body = Vec::new(); let w = &mut crate::coding::Encoder::new(&mut body, Version::Lite07.into()); - w.varint(0).unwrap(); - w.varint(0).unwrap(); + w.varint(base).unwrap(); + w.varint(keep).unwrap(); w.bytes(path).unwrap(); w.varint(0).unwrap(); w.bytes(instance).unwrap(); @@ -10478,6 +10484,38 @@ mod tests { cursor.assert_next_wait(); } + /// Fields valid alone can still resolve past the path limits: that claim is refused + /// and counted like any other, and the stream keeps going. + #[kio::test] + async fn a_claim_resolved_too_deep_is_refused() { + let session = crate::lite::test_transport::ScriptedSession::new(two_routes()); + let log = session.log.clone(); + let (registry, stats) = counted(); + let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); + let mut cursor = consumer.routes(); + assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); + cursor.assert_next_active("a"); + cursor.assert_next_active("b"); + + // Id 2 sits at the depth limit; id 3 keeps all of it and adds a segment. + let deep = vec!["s"; Path::MAX_PARTS].join("/"); + session.push(&announce_script( + Version::Lite07, + &[ + &raw(ANNOUNCE_START, start_body(deep.as_bytes(), b"i")), + &raw(ANNOUNCE_START, based_start_body(1, Path::MAX_PARTS as u64, b"z", b"i")), + &claim_on(0, "c"), + &floor(0, Some(1)), + ], + )); + assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); + assert!(log.closes().is_empty(), "{:?}", log.closes()); + cursor.assert_next_active(&deep); + cursor.assert_next_active("c"); + cursor.assert_next_wait(); + assert_eq!(refusals(®istry), 1); + } + /// Only the neighbour's own framing and state close the session: a refused route /// message naming a Route ID or Announce ID the neighbour never opened still does. #[kio::test] diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index bde5173abc5..8c47884604c 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -232,7 +232,14 @@ impl Routed { let path = decoder.start(Some(suffix))?; let path = self.claim_path(prefix, path); let claim = claim(instance, epoch, cost, hops)?; - self.start(decoder.last(), route, path, Some(claim))?; + let id = decoder.last(); + // Fields valid alone can still resolve, or join the prefix, past the path limits. + if path.is_none() { + let reason = lite::Refusal::Path; + tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); + self.stats.refuse_route(); + } + self.start(id, route, path, Some(claim))?; } Msg::Update { id, hops, cost } => { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; From 305da0cbe6f7197c6e83096a04a72dc05d1c6c70 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:31:43 -0800 Subject: [PATCH 10/16] fix(js/net): a route refused once live leaves its last floor as it was Floors.leave retains the route's claims by reference, so a refused route reads on into a copy of its own. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscriber.test.ts | 38 ++++++++++++++++++++++++++++++ js/net/src/lite/subscriber.ts | 6 ++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index bcac1845b07..bc3fef89b1c 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4130,6 +4130,44 @@ test("a lite-07 refused route's floors hide no other stream's claims", async () subscriber.close(); }); +// A route refused once live leaves what its last floor certified, as an ended one does: the claims +// it reads afterwards never rewrite that. +test("a lite-07 route refused once live leaves its last floor as it was", async () => { + const { subscriber, announced, send, raw, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const instance = Instance.parse("i"); + const node = 7n; + const x = Path.from("live/x"); + + // Both streams carry live/x, the first at revision 5 and the second lagging at 3. + await send({ status: "routeStart", node, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 5n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + await live.send({ status: "routeStart", node, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance }); + await live.send({ status: "floor", id: 0n, revision: 3n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start" }); + + // The first stream's route is refused, then ends live/x at a floor no view takes. + raw([7, 5, 0, 1, 0, 0x80, 42]); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 6n }); + + // The second stream's next floor still has live/x, as revision 5 did: what it shows next is + // another node's claim, not live/x withdrawn. + await live.send({ status: "floor", id: 0n, revision: 4n }); + await live.send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); + await live.send({ status: "floor", id: 1n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + // Two streams carry one origin's claims, and a withdrawal reaches one first. Of the floors that // certify a claim, the newest decides whether it exists, so the lagging stream's copy is held back // rather than selected and refused: also once the newer stream's route ends. diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 222615ea4be..85743557a45 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1256,7 +1256,11 @@ export class Subscriber { const joined = wire.advert !== undefined; // The reserved node names no route, whatever the peer forwards for it. wire.advert = wire.node === 0n ? undefined : advert; - if (joined && wire.advert === undefined) this.#floors.leave(wire.node, wire.carrier); + if (joined && wire.advert === undefined) { + this.#floors.leave(wire.node, wire.carrier); + // What its floor certified stays as it was, while the route goes on reading claims. + wire.view.claims = new Map(wire.view.claims); + } if (joined || wire.advert === undefined) return; this.#floors.join(wire.node, wire.carrier); const others = this.#floors.floor(wire.node, wire.carrier); From 441a83f951b251a823c54f6f6ebb7656699c7591 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:49:48 -0800 Subject: [PATCH 11/16] fix(js/net): a route refused once live re-checks cached consumes, as ROUTE_END does Leaving the node's floors can make a broader claim cover a path again, and show() skips the unchanged claim, so a held consume kept the validity a narrower floor had aborted until something else re-checked its path. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/subscriber.test.ts | 45 +++++++++++++++++++++++++++++- js/net/src/lite/subscriber.ts | 3 ++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index bc3fef89b1c..8ebf8b50f1d 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4023,7 +4023,7 @@ async function routedStream(options?: RouteOptions, getStats?: () => Promise await pick.info; }); +// A route refused once live stops covering, as an ended one does: a held consume a narrower floor +// of it took off a broader claim is routable again once that floor is gone. +test("a lite-07 route refused once live gives a held consume back its broader claim", async () => { + const { subscriber, announced, send, trackRequest, another, opened } = await routedStream(); + const live = Path.from("live"); + const y = Path.from("live/x/y"); + const minted = Mint.fresh(); + + // NODE_A claims live under a floor that certifies nothing, and a consume of live/x/y takes it. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: live, route: 0n, instance: minted }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: live, kind: "start" }); + const x = await another(1, Path.Pattern.subtree(Path.from("live/x"))); + const held = await trackRequest(y, 2); + expect(held.request).toMatchObject({ node: NODE_A }); + + // NODE_A's revision 1 presents only live/x/m under live/x, so live no longer covers live/x/y. + await x.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await x.send({ status: "active", suffix: Path.from("m"), route: 0n, instance: minted }); + await x.send({ status: "floor", id: 0n, revision: 1n }); + expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); + await held.info; + + // That route is refused: nothing certifies live/x/y any more, so live covers it again. + x.raw([7, 5, 0, 1, 0, 0x80, 42]); + expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "offline" }); + const info = held.consumed + .track("audio") + .info() + .then( + () => "answered", + () => "refused", + ); + expect(await Promise.race([opened(3).then(() => "opened"), info])).toBe("opened"); + + held.consumed.close(); + x.announced.close(); + announced.close(); + subscriber.close(); + await info; +}); + // A route that cannot be used for a while keeps its claims' place in a tie, so the announcements // and the consumes still agree once it recovers, as a Rust entry keeps its id. test("a lite-07 tie keeps its winner across a route's unusable spell", async () => { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 85743557a45..d74b920d459 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1278,8 +1278,11 @@ export class Subscriber { case "routeUpdate": { const wire = route(message.id); console.warn(`refusing a forwarded route: node=${wire.node} route=${message.id} reason=${reason}`); + const joined = wire.advert !== undefined; reach(wire, undefined); refreshRoute(wire, true); + // Its floors no longer cover anything, as on ROUTE_END. + if (joined) this.#consumes.revalidate(); return; } case "start": From c1d4e7b4b7d54ed1581864d84d0f71ff1577299b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 22:02:54 -0800 Subject: [PATCH 12/16] feat(net): a refused claim goes offline, its route's floors certifying nothing meanwhile A claim this node refuses was not withdrawn by its origin, so while its route holds one, the route's floors certify nothing: readers see it go offline, and a relay forwards no withdrawal the origin never made. Once the last refused claim ends, the route's floors certify again. JS now reports a claim gone where no live floor certifies its path as offline, as Rust's router already does. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 4 +- js/net/src/auth.test.ts | 3 +- js/net/src/lite/floors.ts | 11 +++ js/net/src/lite/subscriber.test.ts | 49 ++++++++++- js/net/src/lite/subscriber.ts | 25 ++++-- rs/moq-net/src/lite/subscriber.rs | 19 +++- rs/moq-net/src/lite/subscriber/routed.rs | 106 +++++++++++++++++++++++ rs/moq-net/src/model/origin.rs | 16 ++-- 8 files changed, 211 insertions(+), 22 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 0177fb3e1d3..09a99e2ff77 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -481,7 +481,7 @@ A newer version or release may send such content that an older receiver cannot u This specification marks each such value "refused"; the receiver skips the rest of the message's body, and SHOULD log and count each refusal so a refused route is never silently missing. - A refused ROUTE_START still assigns its Route ID, and a refused ROUTE_UPDATE keeps it; either leaves the receiver no route, so its claims go offline as on ROUTE_END. The messages naming the Route ID stay valid, and a later ROUTE_UPDATE the receiver can use makes the route usable again, with the claims its last floor certified. A route to Node ID 0 stays refused. -- A refused ANNOUNCE_START still assigns its Announce ID, and a refused ANNOUNCE_UPDATE or ANNOUNCE_RESTART keeps it; the claim is withdrawn at its route's next floor. A claim whose path was refused stays refused; any other comes back with a later ANNOUNCE_RESTART the receiver can use. +- A refused ANNOUNCE_START still assigns its Announce ID, and a refused ANNOUNCE_UPDATE or ANNOUNCE_RESTART keeps it; the claim goes offline at its route's next floor, which is not an ANNOUNCE_END. A claim whose path was refused stays refused; any other comes back with a later ANNOUNCE_RESTART the receiver can use. Its origin withdrew nothing, so while a route has a live claim the receiver refused, the receiver takes the route's floors as certifying nothing, and a relay forwards no withdrawal for the claim. - A refused WARM_START still assigns its Warm ID. Only the publisher's own framing and stream state close the session: a message that does not decode within its Message Length, such as an RTT flag without its RTT, and a message naming an id that was never assigned or already retired, or breaking the stream's order, as above. @@ -2498,7 +2498,7 @@ An application-specific payload. - FRAME leads with `Payload Length`, the payload size plus one, in place of the trailing `Message Length`; 0 is malformed. - A publisher SHOULD drop a datagram rather than block when the transport has no room for it, where it previously sent one only if the congestion controller could transmit it immediately. - Added an optional `Subscriber Max Age` to FETCH, encoded as milliseconds plus one with zero meaning absent. It can tighten, but never relax, `Publisher Max Age`. -- A receiver refuses forwarded route content it cannot use, such as an unknown route flag, more than 32 Hops, or an invalid claim path or Instance, withdrawing only that route or claim rather than closing the session ([Refused Content](#refused-content)). +- A receiver refuses forwarded route content it cannot use, such as an unknown route flag, more than 32 Hops, or an invalid claim path or Instance, withdrawing only that route or claim rather than closing the session ([Refused Content](#refused-content)). A refused claim goes offline, and its route's floors certify nothing while it holds one. ## moq-lite-06 diff --git a/js/net/src/auth.test.ts b/js/net/src/auth.test.ts index 54117af06ad..3e06542a2a2 100644 --- a/js/net/src/auth.test.ts +++ b/js/net/src/auth.test.ts @@ -410,7 +410,8 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { issued[0]?.revoke(SessionCode.Unauthorized, "expired"); const second = await nextRoute(announced); expect(second?.prefix).toBe(Path.from("a/x")); - expect(second?.kind).toBe("end"); + // Withdrawn under a floor that certifies nothing, so the reader sees it let go, not ended. + expect(second?.kind).toBe("offline"); // The union is empty but still a grant, and a new token restores it. const empty = await waitFor(client.auth.grant, (g) => g !== undefined && g.publish.size === 0); diff --git a/js/net/src/lite/floors.ts b/js/net/src/lite/floors.ts index 056a0493cd8..36b59e0229a 100644 --- a/js/net/src/lite/floors.ts +++ b/js/net/src/lite/floors.ts @@ -166,6 +166,17 @@ export class Floors { return theirs; } + /** + * Whether `view` or a live view of `node` certifies whether a claim at `path` exists, so a + * claim gone there was withdrawn by its origin rather than let go, as rs/moq-net's router + * tells an end from going offline. + */ + certified(node: bigint, view: View, path: Path.Valid): boolean { + if (certifies(view, path)) return true; + for (const carrier of this.#nodes.get(node)?.carriers ?? []) if (certifies(carrier.view, path)) return true; + return false; + } + /** * Whether a claim of `node` at `at` may serve a consume of `path` below it. The newest view * certifying `path`, the most specific of those alike, certifies what covers it there: a claim diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 8ebf8b50f1d..3d552c3052d 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -421,7 +421,8 @@ test("a lite-07 restart replaces the instance, and the next consume subscribes f await send(async (w) => { await encodeAnnounceBroadcast(w, { status: "endedId", id: 0n }, v); - await encodeAnnounceBroadcast(w, { status: "floor", id: 0n }, v); + // Certified, the claim's origin withdrew it: it ends rather than going offline. + await encodeAnnounceBroadcast(w, { status: "floor", id: 0n, revision: 0n }, v); }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "end" }); @@ -4069,6 +4070,52 @@ test("a lite-07 refused route or claim withdraws only itself", async () => { subscriber.close(); }); +// A claim this subscriber refuses was not withdrawn by its origin: it goes offline, and while its +// route holds one, the route's floors certify nothing, so another claim's end there goes offline +// too. Mirrors rs/moq-net. +test("a lite-07 refused claim goes offline, its route certifying nothing meanwhile", async () => { + const { subscriber, announced, send, raw } = await routedStream(); + const [a, b, e] = ["live/a", "live/b", "live/e"].map((path) => Path.from(path)); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: a, route: 0n, instance }); + await send({ status: "active", suffix: b, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: b, kind: "start" }); + + // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. JS reads no Lite-06 Chain, so an + // ANNOUNCE_UPDATE has nothing it could refuse. + const refuse = () => raw([3, 5, 0, 0, 0, 0, 0]); + refuse(); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "offline" }); + await send({ status: "endedId", id: 1n }); + await send({ status: "floor", id: 0n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: b, kind: "offline" }); + + // A usable restart brings it back, and refusing the next takes it offline again. + await send({ status: "restart", id: 0n, instance: Instance.parse("j") }); + await send({ status: "floor", id: 0n, revision: 3n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + refuse(); + await send({ status: "floor", id: 0n, revision: 4n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "offline" }); + + // Once it ends, the route's floors certify again. + await send({ status: "endedId", id: 0n }); + await send({ status: "active", suffix: e, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 5n }); + expect(await announced.next()).toMatchObject({ prefix: e, kind: "start" }); + await send({ status: "endedId", id: 2n }); + await send({ status: "floor", id: 0n, revision: 6n }); + expect(await announced.next()).toMatchObject({ prefix: e, kind: "end" }); + + announced.close(); + subscriber.close(); +}); + // A claim at the session root has the empty path, which is a path all the same: its updates, // restarts, and end apply as any other's. test("a lite-07 claim at the root updates and ends", async () => { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index d74b920d459..ef9eb2de34d 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -981,6 +981,8 @@ export class Subscriber { claims: Map; // The Announce IDs of its live claims whose path this node refused. unnamed: Set; + // The Announce IDs of its live claims at a path whose content this node refuses. + withheld: Set; // How the claims differ from the view's, applied whole at the next floor. pending: Map; // The claims as of the last floor, and the revision it certified them at, if any. @@ -1176,7 +1178,8 @@ export class Subscriber { ids.add(wire.id); } serve(wire, path, restart ?? false); - refresh(path, false); + // A claim gone where no floor certifies it was let go, not withdrawn by its origin. + refresh(path, !now && !this.#floors.certified(wire.node, wire.view, path)); } }; const refreshRoute = (wire: Route, lost: boolean) => { @@ -1200,6 +1203,7 @@ export class Subscriber { held: false, claims: new Map(), unnamed: new Set(), + withheld: new Set(), pending: new Map(), view, shown: new Map(), @@ -1247,6 +1251,7 @@ export class Subscriber { claims.set(id, { route: routeId, path: named, claim }); wire.claims.set(named, id); if (claim) change(routeId, named, claim); + else wire.withheld.add(id); }; // The route now reaches its node by `advert`, or undefined while this node refuses what the @@ -1296,7 +1301,9 @@ export class Subscriber { `refusing a forwarded claim: route=${live.route} claim=${message.id} reason=${reason}`, ); live.claim = undefined; - if (live.path !== undefined) change(live.route, live.path, undefined); + if (live.path === undefined) return; + route(live.route).withheld.add(message.id); + change(live.route, live.path, undefined); return; } } @@ -1355,16 +1362,14 @@ export class Subscriber { break; } case "floor": { - // The batch applies whole, at the revision its floor certifies. + // The batch applies whole, at the revision its floor certifies: none while this + // node refuses one of the route's claims, which its origin never withdrew. const wire = route(msg.id); + const revision = wire.unnamed.size === 0 && wire.withheld.size === 0 ? msg.revision : undefined; // A floor that certifies less keeps what the route's last one certified. const certified = wire.view.revision; const joined = wire.advert !== undefined; - if ( - joined && - certified !== undefined && - (msg.revision === undefined || msg.revision < certified) - ) + if (joined && certified !== undefined && (revision === undefined || revision < certified)) this.#floors.regress(wire.node, wire.carrier); const touched = new Map(); for (const [path, claim] of wire.pending) { @@ -1389,7 +1394,7 @@ export class Subscriber { } } wire.pending.clear(); - wire.view.revision = msg.revision; + wire.view.revision = revision; // Every other route to the node is shown anew first; with one, this route's // unchanged claims may change state too. const others = joined && this.#floors.floor(wire.node, wire.carrier); @@ -1421,6 +1426,7 @@ export class Subscriber { // A claim without a path stays refused, whatever the peer sends for it. if (live.path === undefined) break; live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; + route(live.route).withheld.delete(msg.id); change(live.route, live.path, live.claim); break; } @@ -1437,6 +1443,7 @@ export class Subscriber { break; } wire.claims.delete(live.path); + wire.withheld.delete(msg.id); change(live.route, live.path, undefined); break; } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index abf6ff7b9dd..08f50f51727 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10443,7 +10443,8 @@ mod tests { ]); poll(&mut prefix); cursor.assert_next_active("c"); - cursor.assert_next_ended("a"); + // Refused, not withdrawn by its origin: it goes offline. + cursor.assert_next_offline("a"); cursor.assert_next_wait(); assert_eq!(refusals(®istry), 3); @@ -10469,10 +10470,11 @@ mod tests { // A restart to an empty instance is refused like the update was. push(&[&raw(ANNOUNCE_RESTART, vec![0, 0, 0, 0, 0]), &floor(0, Some(4))]); poll(&mut prefix); - cursor.assert_next_ended("a"); + cursor.assert_next_offline("a"); assert_eq!(refusals(®istry), 4); - // The refused claims end like any other, and `c`'s end still finds it. + // The refused claims end like any other, and `c`'s end still finds it, though only + // offline: the route still holds the refused `a`, so its floor certifies nothing. push(&[ &raw(ANNOUNCE_END, vec![2]), &raw(ANNOUNCE_END, vec![3]), @@ -10480,7 +10482,16 @@ mod tests { &floor(0, Some(5)), ]); poll(&mut prefix); - cursor.assert_next_ended("c"); + cursor.assert_next_offline("c"); + cursor.assert_next_wait(); + + // Once `a` ends too, the route's floors certify again: `e` (id 5) ends. + push(&[&raw(ANNOUNCE_END, vec![0]), &claim_on(0, "e"), &floor(0, Some(6))]); + poll(&mut prefix); + cursor.assert_next_active("e"); + push(&[&raw(ANNOUNCE_END, vec![5]), &floor(0, Some(7))]); + poll(&mut prefix); + cursor.assert_next_ended("e"); cursor.assert_next_wait(); } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 8c47884604c..4448f492b50 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -29,6 +29,8 @@ struct WireRoute { claims: BTreeMap, /// The Announce IDs of its live claims whose path this node refused. unnamed: BTreeSet, + /// The Announce IDs of its live claims at a path whose content this node refuses. + withheld: BTreeSet, /// How its claims differ from `committed`, by path, so what the stream keeps /// between floors is bounded by its claims rather than the messages it read. pending: BTreeMap>, @@ -38,6 +40,12 @@ struct WireRoute { } impl WireRoute { + /// Whether its floors may certify its claims: not while this node refuses one, since + /// the origin never withdrew it. + fn certifies(&self) -> bool { + self.unnamed.is_empty() && self.withheld.is_empty() + } + /// The claim at `path` is now `claim`, pending the next floor. fn change(&mut self, path: Path, claim: Option) { let committed = self.committed.as_ref().and_then(|(claims, _)| claims.get(&path)); @@ -206,6 +214,7 @@ impl Routed { } Msg::Floor { id, revision } => { let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; + let revision = revision.filter(|_| route.certifies()); let changes = std::mem::take(&mut route.pending); let (committed, floor) = route.committed.get_or_insert_default(); for (path, claim) in &changes { @@ -261,6 +270,9 @@ impl Routed { let fresh = self::claim(instance, epoch, cost, hops)?; let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; claim.claim = Some(fresh); + if let Some(route) = self.routes.get_mut(&claim.route) { + route.withheld.remove(&id); + } let change = claim.claim.clone(); self.pending(id, change)?; } @@ -310,6 +322,10 @@ impl Routed { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; tracing::warn!(route = claim.route, claim = id, %reason, "refusing a forwarded claim"); claim.claim = None; + if claim.path.is_some() { + let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; + route.withheld.insert(id); + } self.pending(id, None)?; } // Warm routes are ignored here anyway, so refusing one changes nothing. @@ -341,6 +357,7 @@ impl Routed { reach: reach.clone(), claims: BTreeMap::new(), unnamed: BTreeSet::new(), + withheld: BTreeSet::new(), pending: BTreeMap::new(), committed: None, }, @@ -405,6 +422,9 @@ impl Routed { Some(path) => { wire.claims.insert(path.clone(), id); wire.change(path.clone(), claim.clone()); + if claim.is_none() { + wire.withheld.insert(id); + } } None => { wire.unnamed.insert(id); @@ -425,6 +445,7 @@ impl Routed { match claim.path { Some(path) => { route.claims.remove(&path); + route.withheld.remove(&id); route.change(path, None); } None => { @@ -957,6 +978,91 @@ mod tests { assert_eq!(served(&narrow), ["live/a/public/x"]); } + /// A claim this node refuses was not withdrawn by its origin: while its route holds + /// one, the route's floors certify nothing, so a relay forwards no withdrawal for it, + /// and a usable restart certifies the route again. + #[kio::test] + async fn a_refused_claim_forwards_no_withdrawal() { + use crate::lite::{Refusable, Refusal}; + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let edge = routed + .node + .answer(crate::route::Link::default(), Path::empty()) + .unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = |revision| Msg::Floor { + id: 0, + revision: Some(revision), + }; + let refused = |id| Msg::Refused { + message: Refusable::Change { id }, + reason: Refusal::Instance, + }; + let restart = |id| Msg::Restart { + id, + instance: Some(Instance::random()), + epoch: 1, + cost: Cost::new(0), + hops: Default::default(), + }; + // Each batch the edge would send for node 7: the paths it changes, whether each is + // present, and its floor. + let batches = |routed: &mut super::Routed| { + routed.outputs(&subscriber); + let mut batches = Vec::new(); + while let Some(output) = edge.try_output() { + if let crate::route::node::Output::Batch { id: 7, changes, floor } = output { + let changes: Vec<_> = (changes.into_iter()) + .map(|(path, claim)| (path.to_string(), claim.is_some())) + .collect(); + batches.push((changes, floor)); + } + } + batches + }; + let route = Msg::RouteStart { + node: 7, + advert: advert(), + }; + for msg in [route, start("live/a"), start("live/b"), floor(0)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!( + batches(&mut routed), + [(vec![("live/a".into(), true), ("live/b".into(), true)], Some(0))] + ); + + // Refused, live/a goes without a withdrawal certified for it. + for msg in [refused(0), floor(1)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/a".into(), false)], None)]); + // Nor does live/b's change certify anything, while the route holds the refused claim. + let update = Msg::Update { + id: 1, + hops: Default::default(), + cost: Cost::new(5), + }; + for msg in [update, floor(2)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/b".into(), true)], None)]); + + // A usable restart brings live/a back, certified again. + for msg in [restart(0), floor(3)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/a".into(), true)], Some(3))]); + + // A refused restart withholds it as the update did. + for msg in [refused(0), floor(4)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/a".into(), false)], None)]); + } + /// A route the peer ended keeps its announce slot until its ROUTE_END echo is /// flushed, so a peer leaving its replies unread cannot churn routes past the cap. #[kio::test] diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9db3044703e..008021e654c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -8692,20 +8692,26 @@ impl Routes { /// The next update must be a retraction at `expected`. pub fn assert_next_ended(&mut self, expected: impl TryInto) { + self.assert_next_gone(RouteKind::End, expected); + } + + /// The next update must be `expected` let go of here, not retracted by its origin. + pub fn assert_next_offline(&mut self, expected: impl TryInto) { + self.assert_next_gone(RouteKind::Offline, expected); + } + + fn assert_next_gone(&mut self, kind: RouteKind, expected: impl TryInto) { match kio::wait(|waiter| self.poll_next(waiter)) .now_or_never() .expect("next blocked") .expect("no next") { - RouteEvent { - kind: RouteKind::End, - announce, - } => assert_eq!( + RouteEvent { kind: got, announce } if got == kind => assert_eq!( announce.prefix, expected.try_into().expect("expected path"), "wrong prefix" ), - other => panic!("should be a retraction: got {other:?}"), + other => panic!("should be {kind:?}: got {other:?}"), } } From 3f73435a8537cad44ad4cafb953e07fbca382088 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 22:10:17 -0800 Subject: [PATCH 13/16] fix(js/net): a claim ends only where the newest certifying floor withdrew it Another stream certifying a refused claim still present proves no withdrawal, so the refusing stream's reader sees it go offline rather than end. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/floors.ts | 19 ++++++++++++------- js/net/src/lite/subscriber.test.ts | 27 +++++++++++++++++++++++++++ js/net/src/lite/subscriber.ts | 5 +++-- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/js/net/src/lite/floors.ts b/js/net/src/lite/floors.ts index 36b59e0229a..b9a6420b9bb 100644 --- a/js/net/src/lite/floors.ts +++ b/js/net/src/lite/floors.ts @@ -167,14 +167,19 @@ export class Floors { } /** - * Whether `view` or a live view of `node` certifies whether a claim at `path` exists, so a - * claim gone there was withdrawn by its origin rather than let go, as rs/moq-net's router - * tells an end from going offline. + * Whether the newest of `view` and the live views of `node` certifying `path` presents no + * claim there, so a claim gone from `view` was withdrawn by its origin rather than let go, as + * rs/moq-net's router tells an end from going offline. */ - certified(node: bigint, view: View, path: Path.Valid): boolean { - if (certifies(view, path)) return true; - for (const carrier of this.#nodes.get(node)?.carriers ?? []) if (certifies(carrier.view, path)) return true; - return false; + withdrawn(node: bigint, view: View, path: Path.Valid): boolean { + // The view itself wins a tie, as in shows(). + let best: View | undefined = certifies(view, path) ? view : undefined; + for (const carrier of this.#nodes.get(node)?.carriers ?? []) { + const other = carrier.view; + // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. + if (certifies(other, path) && (best === undefined || other.revision! > best.revision!)) best = other; + } + return best !== undefined && presented(best, path, path === view.prefix) === undefined; } /** diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 3d552c3052d..28b37e7dfbf 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4116,6 +4116,33 @@ test("a lite-07 refused claim goes offline, its route certifying nothing meanwhi subscriber.close(); }); +// Another stream certifying the refused claim still present proves no withdrawal either: the +// refusing stream's reader sees it go offline. +test("a lite-07 refused claim another stream still certifies goes offline", async () => { + const { subscriber, announced, send, raw, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance }); + await live.send({ status: "floor", id: 0n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start" }); + + // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. + raw([3, 5, 0, 0, 0, 0, 0]); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + // A claim at the session root has the empty path, which is a path all the same: its updates, // restarts, and end apply as any other's. test("a lite-07 claim at the root updates and ends", async () => { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index ef9eb2de34d..bb9182851c6 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1178,8 +1178,9 @@ export class Subscriber { ids.add(wire.id); } serve(wire, path, restart ?? false); - // A claim gone where no floor certifies it was let go, not withdrawn by its origin. - refresh(path, !now && !this.#floors.certified(wire.node, wire.view, path)); + // A claim gone unless the newest floor certifying it says so was let go, not withdrawn by + // its origin. + refresh(path, !now && !this.#floors.withdrawn(wire.node, wire.view, path)); } }; const refreshRoute = (wire: Route, lost: boolean) => { From cf49d654052c190fbbc467b32ce2aafcc50441ce Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 22:35:34 -0800 Subject: [PATCH 14/16] fix(js/net): an uncertified withdrawal still ends unless a refusal is involved A JS reader asks for a prefix wider than its scope's region, so a relay certifies nothing to it, and a broadcast's clean finish would read offline. Report offline only where the newest certifying floor still presents the claim, or the route holds a refused claim and no floor certifies it gone. Co-Authored-By: Claude Opus 5.5 --- js/net/src/auth.test.ts | 3 +-- js/net/src/lite/floors.ts | 11 ++++++----- js/net/src/lite/subscriber.test.ts | 3 +-- js/net/src/lite/subscriber.ts | 10 +++++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/js/net/src/auth.test.ts b/js/net/src/auth.test.ts index 3e06542a2a2..54117af06ad 100644 --- a/js/net/src/auth.test.ts +++ b/js/net/src/auth.test.ts @@ -410,8 +410,7 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { issued[0]?.revoke(SessionCode.Unauthorized, "expired"); const second = await nextRoute(announced); expect(second?.prefix).toBe(Path.from("a/x")); - // Withdrawn under a floor that certifies nothing, so the reader sees it let go, not ended. - expect(second?.kind).toBe("offline"); + expect(second?.kind).toBe("end"); // The union is empty but still a grant, and a new token restores it. const empty = await waitFor(client.auth.grant, (g) => g !== undefined && g.publish.size === 0); diff --git a/js/net/src/lite/floors.ts b/js/net/src/lite/floors.ts index b9a6420b9bb..d44ec2c7db5 100644 --- a/js/net/src/lite/floors.ts +++ b/js/net/src/lite/floors.ts @@ -167,11 +167,11 @@ export class Floors { } /** - * Whether the newest of `view` and the live views of `node` certifying `path` presents no - * claim there, so a claim gone from `view` was withdrawn by its origin rather than let go, as - * rs/moq-net's router tells an end from going offline. + * What the newest of `view` and the live views of `node` certifying `path` says of a claim + * there: "absent" when it presents none, "present" when it does, or undefined when none + * certifies the path. */ - withdrawn(node: bigint, view: View, path: Path.Valid): boolean { + certified(node: bigint, view: View, path: Path.Valid): "absent" | "present" | undefined { // The view itself wins a tie, as in shows(). let best: View | undefined = certifies(view, path) ? view : undefined; for (const carrier of this.#nodes.get(node)?.carriers ?? []) { @@ -179,7 +179,8 @@ export class Floors { // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. if (certifies(other, path) && (best === undefined || other.revision! > best.revision!)) best = other; } - return best !== undefined && presented(best, path, path === view.prefix) === undefined; + if (best === undefined) return undefined; + return presented(best, path, path === view.prefix) === undefined ? "absent" : "present"; } /** diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 28b37e7dfbf..12e453442a2 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -421,8 +421,7 @@ test("a lite-07 restart replaces the instance, and the next consume subscribes f await send(async (w) => { await encodeAnnounceBroadcast(w, { status: "endedId", id: 0n }, v); - // Certified, the claim's origin withdrew it: it ends rather than going offline. - await encodeAnnounceBroadcast(w, { status: "floor", id: 0n, revision: 0n }, v); + await encodeAnnounceBroadcast(w, { status: "floor", id: 0n }, v); }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "end" }); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index bb9182851c6..4ba9726b29b 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1178,9 +1178,13 @@ export class Subscriber { ids.add(wire.id); } serve(wire, path, restart ?? false); - // A claim gone unless the newest floor certifying it says so was let go, not withdrawn by - // its origin. - refresh(path, !now && !this.#floors.withdrawn(wire.node, wire.view, path)); + // A claim gone was let go, not withdrawn by its origin, where the newest floor certifying it + // still presents it, or where this node refuses one of the route's claims and no floor + // certifies it gone. An uncertified withdrawal otherwise still ends: a relay certifies + // nothing to a request wider than its scope's region, as this one may be. + const said = now ? undefined : this.#floors.certified(wire.node, wire.view, path); + const refusing = wire.unnamed.size > 0 || wire.withheld.size > 0; + refresh(path, !now && (said === "present" || (refusing && said === undefined))); } }; const refreshRoute = (wire: Route, lost: boolean) => { From b7aa6a4c50e746486c3077b7f1b8898e70fceab0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 23:04:16 -0800 Subject: [PATCH 15/16] fix(js/net): tell an end from offline with the views shows() weighs Floors.certified() picked the newest certifying view from the live carriers only, so an older live floor without a refused claim could beat the refusing route's retained newer one and report a false end. Both now share one helper. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/floors.ts | 40 +++++++++++++++--------------- js/net/src/lite/subscriber.test.ts | 32 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/js/net/src/lite/floors.ts b/js/net/src/lite/floors.ts index d44ec2c7db5..86bffb5bee1 100644 --- a/js/net/src/lite/floors.ts +++ b/js/net/src/lite/floors.ts @@ -148,18 +148,7 @@ export class Floors { shows(node: bigint, view: View, path: Path.Valid): Claim | undefined { const claim = view.claims.get(path); if (!claim) return undefined; - const entry = this.#nodes.get(node); - // The view itself wins a tie: one revision presents the same claims on every stream. - let best: View | undefined = certifies(view, path) ? view : undefined; - const consider = (other: View) => { - if (!certifies(other, path)) return; - // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. - if (best === undefined || other.revision! > best.revision!) best = other; - }; - for (const other of entry?.carriers ?? []) consider(other.view); - // What an ended or regressed floor said counts only where a live one still certifies, as - // Rust's router prunes a merged region no certifier certifies any more. - if (best !== undefined) for (const other of entry?.retained ?? []) consider(other); + const best = this.#newest(node, view, path); if (best === undefined || best === view) return claim; const theirs = presented(best, path, path === view.prefix); if (theirs?.instance !== claim.instance || theirs.epoch !== claim.epoch) return undefined; @@ -167,22 +156,33 @@ export class Floors { } /** - * What the newest of `view` and the live views of `node` certifying `path` says of a claim + * What the newest view certifying `path`, as `shows` picks it for `view`, says of a claim * there: "absent" when it presents none, "present" when it does, or undefined when none * certifies the path. */ certified(node: bigint, view: View, path: Path.Valid): "absent" | "present" | undefined { - // The view itself wins a tie, as in shows(). - let best: View | undefined = certifies(view, path) ? view : undefined; - for (const carrier of this.#nodes.get(node)?.carriers ?? []) { - const other = carrier.view; - // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. - if (certifies(other, path) && (best === undefined || other.revision! > best.revision!)) best = other; - } + const best = this.#newest(node, view, path); if (best === undefined) return undefined; return presented(best, path, path === view.prefix) === undefined ? "absent" : "present"; } + // The newest view certifying `path` for `view`, under a route to `node`, or undefined when none does. + #newest(node: bigint, view: View, path: Path.Valid): View | undefined { + const entry = this.#nodes.get(node); + // The view itself wins a tie: one revision presents the same claims on every stream. + let best: View | undefined = certifies(view, path) ? view : undefined; + const consider = (other: View) => { + if (!certifies(other, path)) return; + // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. + if (best === undefined || other.revision! > best.revision!) best = other; + }; + for (const other of entry?.carriers ?? []) consider(other.view); + // What an ended or regressed floor said counts only where a live one still certifies, as + // Rust's router prunes a merged region no certifier certifies any more. + if (best !== undefined) for (const other of entry?.retained ?? []) consider(other); + return best; + } + /** * Whether a claim of `node` at `at` may serve a consume of `path` below it. The newest view * certifying `path`, the most specific of those alike, certifies what covers it there: a claim diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 12e453442a2..2db1b12eacf 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4142,6 +4142,38 @@ test("a lite-07 refused claim another stream still certifies goes offline", asyn subscriber.close(); }); +// What the refusing route's last floor certified still counts against an older floor of another +// stream: a lagging stream that never had the claim proves no withdrawal of it. +test("a lite-07 refused claim a lagging stream never had goes offline", async () => { + const { subscriber, announced, send, raw, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 7n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + // The second stream lags at revision 5, before live/x existed. + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance }); + await live.send({ status: "floor", id: 0n, revision: 5n }); + // A claim of another node on that stream shows once it read all of the above. + await live.send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); + await live.send({ status: "floor", id: 1n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. + raw([3, 5, 0, 0, 0, 0, 0]); + await send({ status: "floor", id: 0n, revision: 8n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + // A claim at the session root has the empty path, which is a path all the same: its updates, // restarts, and end apply as any other's. test("a lite-07 claim at the root updates and ends", async () => { From 1bc219c35cccc0a91aa40c78de36ef995e1dd143 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Fri, 13 Nov 2026 23:23:47 -0800 Subject: [PATCH 16/16] fix(net): disambiguate announce refusal type after base merge --- js/net/src/lite/subscriber.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 8535083554d..b5b42f6a777 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -67,12 +67,12 @@ import { AnnounceHistory, AnnounceInit, AnnounceOk, + type Refusal as AnnounceRefusal, AnnounceRequest, type Base, decodeAnnounceBroadcastMaybe, encodeAnnounceReply, type Refusable, - type Refusal, ROUTE_MAX_HOPS, } from "./announce.ts"; import { Datagram as DatagramMessage, Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; @@ -1302,7 +1302,7 @@ export class Subscriber { // Apply a message whose content, which the peer only forwarded, this node refuses: withdraw // the route or claim it names, and keep the stream. Mirrors rs/moq-net. - const refused = (message: Refusable, reason: Refusal) => { + const refused = (message: Refusable, reason: AnnounceRefusal) => { switch (message.kind) { case "routeStart": console.warn(`refusing a forwarded route: node=${message.node} reason=${reason}`);