From ed2486983fba8b477bf2211c9d86235fd0f56808 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:45:08 -0800 Subject: [PATCH 01/25] chore(quest): claim quest/m1/socket/rust.md Quest-Claim: 5cea315f8d3e0663 From f1f15251bd474fffb799534e96311db516e75a24 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:05:13 -0800 Subject: [PATCH 02/25] feat(net): dial and accept sockets over an origin moq_net::socket adds Listener, dial, and Socket: a dialer publishes L/.dial/ and the listener answers on L/.accept/ with the dial's instance. Streams are tracks named .//, discovered through the peer's track listing. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/broadcast.rs | 5 + rs/moq-net/src/model/mod.rs | 1 + rs/moq-net/src/model/socket.rs | 701 +++++++++++++++++++++++++ rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/socket.rs | 290 ++++++++++ 5 files changed, 998 insertions(+) create mode 100644 rs/moq-net/src/model/socket.rs create mode 100644 rs/moq-net/tests/integration/socket.rs diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index c0a836556e1..62678423047 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -480,6 +480,11 @@ impl Producer { self.alive.close(); } + /// `Ready` once the broadcast ended, by [`Self::close`] or the last producer dropping. + pub(super) fn poll_closed(&self, waiter: &kio::Waiter) -> Poll<()> { + self.alive.token.poll_closed(waiter) + } + /// Whether any [`Consumer`] handle exists, which is not itself demand for a track. pub(super) fn is_held(&self) -> bool { self.alive.token.wanted() diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index e0a67cc3cb5..b85b3dec06b 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -3,6 +3,7 @@ pub mod broadcast; pub mod cache; pub mod frame; pub mod group; +pub mod socket; pub mod track; // The origin + announce subsystem shares one implementation (a route table). diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs new file mode 100644 index 00000000000..67b1e6afdc2 --- /dev/null +++ b/rs/moq-net/src/model/socket.rs @@ -0,0 +1,701 @@ +//! Point-to-point sockets over an origin: a [`Listener`] accepts the [`Socket`]s that +//! [`dial`]ers open to its path. +//! +//! A socket is a convention over announcements and broadcasts, with no wire of its own. +//! A listener at `L` keeps `L` for whatever it shares with everyone. A dialer publishes +//! its half at `L/.dial/`, and the listener answers with its half at `L/.accept/` +//! once the application accepts. The answer carries the dial's [`Instance`], so a +//! dialer that dials again with the same id gets a new socket rather than an old answer. +//! +//! Inside a socket either side opens tracks the way QUIC opens streams. A uni stream is +//! one track in the opener's broadcast, and a bi stream is the same track name in both. +//! The peer learns of new streams from the broadcast's track listing, which needs +//! moq-lite-07 on every hop. See `doc/concept/socket.md` for the convention. + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Weak}; +use std::task::{Poll, ready}; + +use crate::{Error, Instance, Path, Pattern, Patterns}; + +use super::broadcast::{self, TracksEvent}; +use super::origin::{self, Route}; +use super::{announce, track}; + +/// The segment beneath a listener's path that dialers publish under. +const DIAL: &str = ".dial"; +/// The segment beneath a listener's path that the listener answers under. +const ACCEPT: &str = ".accept"; + +/// Which end of a socket a side is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Role { + Dialer, + Listener, +} + +impl Role { + /// The tag naming the streams this role opens. + fn tag(self) -> &'static str { + match self { + Self::Dialer => "dial", + Self::Listener => "accept", + } + } + + fn peer(self) -> Self { + match self { + Self::Dialer => Self::Listener, + Self::Listener => Self::Dialer, + } + } +} + +/// Whether a stream carries data one way or both. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Uni, + Bi, +} + +impl Kind { + fn tag(self) -> &'static str { + match self { + Self::Uni => "uni", + Self::Bi => "bi", + } + } +} + +/// The track name of `name`, opened by `opener` as a `kind` stream: `.//`. +fn track_name(opener: Role, kind: Kind, name: &str) -> String { + format!(".{}/{}/{name}", opener.tag(), kind.tag()) +} + +/// Split a track name into its opener, kind, and stream name, or `None` for a track +/// outside the scheme, such as one the application names itself. +fn parse(track: &str) -> Option<(Role, Kind, &str)> { + let (opener, rest) = track.strip_prefix('.')?.split_once('/')?; + let (kind, name) = rest.split_once('/')?; + let opener = match opener { + "dial" => Role::Dialer, + "accept" => Role::Listener, + _ => return None, + }; + let kind = match kind { + "uni" => Kind::Uni, + "bi" => Kind::Bi, + _ => return None, + }; + Some((opener, kind, name)) +} + +/// `L//`, refusing an id that is not exactly one path segment. +fn side(path: &Path, side: &'static str, id: &str) -> Result { + let segment = Path::new(id)?; + if segment.parts().count() != 1 { + return Err(crate::InvalidPattern::InvalidSegment(id.to_string()).into()); + } + Ok(path.join(&Path::from_static(side))?.join(&segment)?) +} + +/// Why a [`Socket`] ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Closed { + /// The peer's broadcast ended: it closed the socket, or a session on the way dropped. + #[error("the peer ended the socket")] + Remote, + /// This side's broadcast ended, through [`Socket::close`] or otherwise. + #[error("the socket was closed locally")] + Local, + /// The dialer dialed again with the same id, which replaces this socket. + #[error("a newer dial replaced the socket")] + Replaced, +} + +/// A uni stream the peer opened. +pub struct Uni { + /// The name the peer opened it with. + pub name: Arc, + /// The peer's track. + pub consumer: track::Consumer, +} + +/// A bi stream: this side's track and the peer's, under one name. +pub struct Bi { + /// The name the opener gave it. + pub name: Arc, + /// This side's track. + pub producer: track::Producer, + /// The peer's track. + pub consumer: track::Consumer, +} + +/// What the peer's track listing has reported and nobody has taken yet. +#[derive(Default)] +struct Listing { + /// The peer's listing, opened by the first accept or bi open. + tracks: Option, + /// Uni streams the peer opened, not yet accepted. + uni: VecDeque>, + /// Bi streams the peer opened, with the opener's track info, not yet accepted. + bi: VecDeque<(Arc, track::Info)>, + /// This side's bi opens waiting for an answer, and whether the peer listed it. + opening: HashMap, bool>, + /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. + end: Option>, +} + +impl Listing { + /// Sort every event on hand into the queues, registering `waiter` for the next. + /// + /// Every poller drains the one listing, so each sorts what it reads for the others: + /// they all park on the listing, which wakes each of them on a change. + fn drain(&mut self, waiter: &kio::Waiter, role: Role, remote: &broadcast::Consumer) { + if self.end.is_some() { + return; + } + let tracks = match &mut self.tracks { + Some(tracks) => tracks, + None => match remote.tracks() { + Ok(tracks) => self.tracks.insert(tracks), + // The peer's broadcast already ended. + Err(Error::Unroutable) => { + self.end = Some(Ok(())); + return; + } + Err(err) => { + self.end = Some(Err(err)); + return; + } + }, + }; + loop { + let event = match tracks.poll_next(waiter) { + Poll::Pending => return, + Poll::Ready(Ok(Some(event))) => event, + Poll::Ready(Ok(None)) => { + self.end = Some(Ok(())); + return; + } + Poll::Ready(Err(err)) => { + self.end = Some(Err(err)); + return; + } + }; + // Only additions open streams; a stream ends through its own track. + let TracksEvent::Added { name, info } = event else { + continue; + }; + // A track outside the scheme is the application's own. + let Some((opener, kind, stream)) = parse(&name) else { + continue; + }; + match (opener == role, kind) { + (false, Kind::Uni) => self.uni.push_back(stream.into()), + (false, Kind::Bi) => self.bi.push_back((stream.into(), info)), + (true, Kind::Bi) => { + if let Some(answered) = self.opening.get_mut(stream) { + *answered = true; + } + } + // This side's uni streams live only in its own broadcast. + (true, Kind::Uni) => {} + } + } + } + + /// The listing's end as what an accept returns: `None` once the peer is gone. + fn ended(&self) -> Option, Error>> { + match self.end.as_ref()? { + Ok(()) => Some(Ok(None)), + Err(err) => Some(Err(err.clone())), + } + } +} + +/// What a socket shares with its pending bi opens and its listener. +struct Inner { + role: Role, + id: Arc, + /// This side's broadcast. + local: origin::Publication, + /// The peer's broadcast. + remote: broadcast::Consumer, + listing: kio::Lock, + /// Why the listener ended the socket, set before it closes [`Self::local`]. + closed: kio::Lock>, +} + +impl Inner { + /// End the socket for `reason`, retracting this side's broadcast. + fn close(&self, reason: Closed) { + self.closed.lock().get_or_insert(reason); + self.local.close(); + } +} + +/// One end of a connection between a dialer and a listener. +/// +/// Holds this side's broadcast and the peer's. Drop it once it ends: dropping retracts +/// this side's broadcast, which ends the socket for the peer too. +pub struct Socket { + inner: Arc, +} + +impl Socket { + fn new(role: Role, id: Arc, local: origin::Publication, remote: broadcast::Consumer) -> Self { + Self { + inner: Arc::new(Inner { + role, + id, + local, + remote, + listing: kio::Lock::new(Listing::default()), + closed: kio::Lock::new(None), + }), + } + } + + /// The id the dialer chose. + pub fn id(&self) -> &str { + &self.inner.id + } + + /// This side's broadcast, for tracks an application names itself. + pub fn local(&self) -> &broadcast::Producer { + &self.inner.local + } + + /// The peer's broadcast, for tracks an application names itself. + pub fn remote(&self) -> &broadcast::Consumer { + &self.inner.remote + } + + /// Open a uni stream named `name`: a track only this side writes. + /// + /// Fails with [`Error::Duplicate`] when this side already opened a uni stream of + /// that name. + pub fn open_uni(&self, name: &str, info: impl Into>) -> Result { + self.inner + .local + .create_track(track_name(self.inner.role, Kind::Uni, name), info) + } + + /// Open a bi stream named `name`, resolving once the peer accepts it. + /// + /// `info` describes this side's track, and the peer's answer mirrors it. Fails with + /// [`Error::Duplicate`] when this side already opened a bi stream of that name. + pub fn open_bi(&self, name: &str, info: impl Into>) -> Result, Error> { + let name: Arc = name.into(); + let producer = self + .inner + .local + .create_track(track_name(self.inner.role, Kind::Bi, &name), info)?; + self.inner.listing.lock().opening.insert(name.clone(), false); + Ok(kio::Pending::new(OpenBi { + inner: self.inner.clone(), + name, + producer: Some(producer), + })) + } + + /// Poll for the next uni stream the peer opened; `None` once the peer's broadcast ended. + pub fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll, Error>> { + let inner = &self.inner; + let mut listing = inner.listing.lock(); + listing.drain(waiter, inner.role, &inner.remote); + if let Some(name) = listing.uni.pop_front() { + drop(listing); + return Poll::Ready( + match inner.remote.track(&track_name(inner.role.peer(), Kind::Uni, &name)) { + Ok(consumer) => Ok(Some(Uni { name, consumer })), + // The peer's broadcast ended. + Err(Error::Unroutable) => Ok(None), + Err(err) => Err(err), + }, + ); + } + match listing.ended() { + Some(end) => Poll::Ready(end), + None => Poll::Pending, + } + } + + /// Wait for the next uni stream the peer opened; see [`Self::poll_accept_uni`]. + #[cfg(feature = "async")] + pub async fn accept_uni(&self) -> Result, Error> { + kio::wait(|waiter| self.poll_accept_uni(waiter)).await + } + + /// Poll for the next bi stream the peer opened, answering it with a track that mirrors + /// the opener's info; `None` once the peer's broadcast ended. + pub fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll, Error>> { + let inner = &self.inner; + loop { + let mut listing = inner.listing.lock(); + listing.drain(waiter, inner.role, &inner.remote); + let Some((name, info)) = listing.bi.pop_front() else { + return match listing.ended() { + Some(end) => Poll::Ready(end), + None => Poll::Pending, + }; + }; + drop(listing); + let track = track_name(inner.role.peer(), Kind::Bi, &name); + let consumer = match inner.remote.track(&track) { + Ok(consumer) => consumer, + // The peer's broadcast ended. + Err(Error::Unroutable) => return Poll::Ready(Ok(None)), + Err(err) => return Poll::Ready(Err(err)), + }; + // The opener's track info, minus how its copy reached this hop. + let info = track::Info { + properties: None, + unchecked: false, + ..info + }; + let producer = match inner.local.create_track(track, info) { + Ok(producer) => producer, + // The peer listed a stream again that this side already answered. + Err(Error::Duplicate) => continue, + Err(err) => return Poll::Ready(Err(err)), + }; + return Poll::Ready(Ok(Some(Bi { + name, + producer, + consumer, + }))); + } + } + + /// Wait for the next bi stream the peer opened; see [`Self::poll_accept_bi`]. + #[cfg(feature = "async")] + pub async fn accept_bi(&self) -> Result, Error> { + kio::wait(|waiter| self.poll_accept_bi(waiter)).await + } + + /// Poll for the socket to end, and why. + pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll { + let inner = &self.inner; + if let Some(reason) = *inner.closed.lock() { + return Poll::Ready(reason); + } + if inner.local.poll_closed(waiter).is_ready() { + // The listener may have ended it between the check above and the close. + return Poll::Ready(inner.closed.lock().unwrap_or(Closed::Local)); + } + ready!(inner.remote.poll_closed(waiter)); + Poll::Ready(Closed::Remote) + } + + /// Wait for the socket to end, and why. + #[cfg(feature = "async")] + pub async fn closed(&self) -> Closed { + kio::wait(|waiter| self.poll_closed(waiter)).await + } + + /// End the socket, retracting this side's broadcast. + pub fn close(&self) { + self.inner.close(Closed::Local); + } +} + +/// A bi stream being opened, from [`Socket::open_bi`]: resolves once the peer answers. +pub struct OpenBi { + inner: Arc, + name: Arc, + /// This side's track, handed over once the peer answers. + producer: Option, +} + +impl OpenBi { + /// Poll for the peer's answer. Fails with [`Error::Unroutable`] once the peer's + /// broadcast ends without one. + pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { + let inner = &self.inner; + let mut listing = inner.listing.lock(); + listing.drain(waiter, inner.role, &inner.remote); + if listing.opening.get(&self.name) == Some(&true) { + listing.opening.remove(&self.name); + drop(listing); + let consumer = inner.remote.track(&track_name(inner.role, Kind::Bi, &self.name))?; + let producer = self.producer.take().ok_or(Error::Closed)?; + return Poll::Ready(Ok(Bi { + name: self.name.clone(), + producer, + consumer, + })); + } + match &listing.end { + Some(Ok(())) => Poll::Ready(Err(Error::Unroutable)), + Some(Err(err)) => Poll::Ready(Err(err.clone())), + None => Poll::Pending, + } + } +} + +impl kio::Task for OpenBi { + type Output = Result; + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll { + self.poll_ok(waiter) + } +} + +impl Drop for OpenBi { + fn drop(&mut self) { + if self.producer.is_some() { + self.inner.listing.lock().opening.remove(&self.name); + } + } +} + +/// Dial the listener at `path` as `id`, from [`dial`]: resolves once the listener accepts. +pub struct Dialing { + id: Arc, + /// This side's broadcast, announced for the listener to find. + local: Option, + /// The listener's answer, pinned to this dial's instance. + answer: origin::Following, +} + +impl Dialing { + /// Poll for the listener's answer. Waits for as long as it takes: drop this to give up. + pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { + let remote = ready!(self.answer.poll_available(waiter))?; + let local = self.local.take().ok_or(Error::Closed)?; + Poll::Ready(Ok(Socket::new(Role::Dialer, self.id.clone(), local, remote))) + } +} + +impl kio::Task for Dialing { + type Output = Result; + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll { + self.poll_ok(waiter) + } +} + +/// Dial the listener at `path` as `id`, resolving once it accepts. +/// +/// Publishes this side's broadcast at `path/.dial/` with a fresh [`Instance`], and +/// waits for the listener's answer at `path/.accept/` carrying that instance. There +/// is no timeout: drop the returned future to give up, which retracts the dial. +/// +/// `id` must be one path segment; an id with a `/` fails with [`Error::InvalidPath`]. +/// Fails like [`origin::Producer::create_broadcast`] when `origin` may not publish the +/// dial, and like [`origin::Consumer::follow`] when it may not read the answer. +pub fn dial( + origin: &origin::Producer, + path: impl TryInto>, + id: &str, +) -> Result, Error> { + let path = crate::path::convert(path)?; + let instance = Instance::random(); + let local = origin.create_broadcast(side(&path, DIAL, id)?)?; + let answer = origin.consume().follow(side(&path, ACCEPT, id)?, instance.clone())?; + local.announce(Route::default().with_instance(instance, 0))?; + Ok(kio::Pending::new(Dialing { + id: id.into(), + local: Some(local), + answer, + })) +} + +/// A socket the listener handed out, kept so a newer dial or the dialer leaving ends it. +struct Live { + instance: Instance, + socket: Weak, +} + +/// A dial being accepted: its broadcast requested, pinned to its instance. +struct Accepting { + id: Path, + instance: Instance, + request: origin::Requesting, +} + +/// Accepts the sockets dialed to a path. +/// +/// Dialers queue in the announcements until [`Self::accept`] takes them, without limit, +/// and a dialer's broadcast is requested only then: a listener that stops accepting +/// holds nothing for the dialers waiting on it. While it accepts, a dialer dialing again +/// with the same id ends the socket it replaces with [`Closed::Replaced`]. +pub struct Listener { + origin: origin::Producer, + path: Path, + /// The dialers' broadcasts, rooted at `path/.dial`. + dials: origin::Consumer, + announced: announce::Consumer, + live: HashMap, + accepting: Option, +} + +impl Listener { + /// Listen at `path` on `origin`, which publishes the answers and reads the dials. + /// + /// Fails with [`Error::Unauthorized`] when `origin` may not read `path/.dial`. + pub fn new(origin: &origin::Producer, path: impl TryInto>) -> Result { + let path = crate::path::convert(path)?; + let ids = Patterns::from("*".parse::()?); + // An id is any one segment, a dotted one included. + let dials = origin + .consume() + .scope(path.join(&Path::from_static(DIAL))?, &ids)? + .with_hidden(true); + let announced = dials.announced(); + Ok(Self { + origin: origin.clone(), + path, + dials, + announced, + live: HashMap::new(), + accepting: None, + }) + } + + /// The path this listener accepts dials at. + pub fn path(&self) -> &Path { + &self.path + } + + /// Poll for the next dialer, requesting its broadcast and answering it. + /// + /// A dialer that leaves before its broadcast resolves is skipped. Fails with + /// [`Error::Closed`] once the origin closes. + pub fn poll_accept(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + if let Some(accepting) = &mut self.accepting { + let remote = ready!(accepting.request.poll_ok(waiter)); + let accepting = self.accepting.take().expect("accepting"); + match remote { + Ok(remote) => match self.answer(accepting, remote) { + Ok(Some(socket)) => return Poll::Ready(Ok(socket)), + Ok(None) => {} + Err(err) => return Poll::Ready(Err(err)), + }, + Err(err) => tracing::debug!(id = %accepting.id, %err, "dialer left before it was accepted"), + } + continue; + } + + let Some(event) = ready!(self.announced.poll_next(waiter)) else { + return Poll::Ready(Err(Error::Closed)); + }; + let (announce, takeover) = match event { + announce::Event::Start(announce) | announce::Event::Restart(announce) => (announce, false), + announce::Event::Takeover { announce, .. } => (announce, true), + // The same instance, delivered in place. + announce::Event::Update(_) => continue, + announce::Event::End(announce) => { + // The dialer's routes are gone, and so is its broadcast; retract the + // answer too, so a return of the same instance never meets it. + if let Some(live) = self.live.remove(&announce.prefix) + && let Some(socket) = live.socket.upgrade() + { + socket.close(Closed::Remote); + } + continue; + } + }; + let Some(instance) = announce.route.instance else { + continue; + }; + if let Some(live) = self.live.get_mut(&announce.prefix) { + match live.socket.upgrade() { + Some(_) if live.instance == instance => continue, + // A takeover continues the stream the live socket already reads. + Some(_) if takeover => { + live.instance = instance; + continue; + } + Some(socket) => socket.close(Closed::Replaced), + None => {} + } + self.live.remove(&announce.prefix); + } + let request = self.dials.request(&announce.prefix, instance.clone()).into_inner(); + self.accepting = Some(Accepting { + id: announce.prefix, + instance, + request, + }); + } + } + + /// Wait for the next dialer; see [`Self::poll_accept`]. + #[cfg(feature = "async")] + pub async fn accept(&mut self) -> Result { + kio::wait(|waiter| self.poll_accept(waiter)).await + } + + /// Publish the answer to a resolved dial, or `None` when this dialer's answer can't be + /// published, such as an id this origin may not answer. + fn answer(&mut self, accepting: Accepting, remote: broadcast::Consumer) -> Result, Error> { + let Accepting { id, instance, .. } = accepting; + let local = side(&self.path, ACCEPT, id.as_str()) + .and_then(|path| self.origin.create_broadcast(path)) + .and_then(|local| { + local.announce(Route::default().with_instance(instance.clone(), 0))?; + Ok(local) + }); + let local = match local { + Ok(local) => local, + Err(Error::Closed) => return Err(Error::Closed), + Err(err) => { + tracing::warn!(%id, %err, "cannot answer the dialer"); + return Ok(None); + } + }; + let socket = Socket::new(Role::Listener, id.as_str().into(), local, remote); + self.live.insert( + id, + Live { + instance, + socket: Arc::downgrade(&socket.inner), + }, + ); + Ok(Some(socket)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn track_names_round_trip() { + for opener in [Role::Dialer, Role::Listener] { + for kind in [Kind::Uni, Kind::Bi] { + let name = track_name(opener, kind, "chat/room"); + assert_eq!(parse(&name), Some((opener, kind, "chat/room"))); + } + } + assert_eq!(track_name(Role::Dialer, Kind::Bi, "x"), ".dial/bi/x"); + assert_eq!(track_name(Role::Listener, Kind::Uni, "x"), ".accept/uni/x"); + } + + #[test] + fn foreign_track_names_are_skipped() { + for name in [ + "command", + ".dial", + ".dial/uni", + ".dial/other/x", + ".other/uni/x", + "dial/uni/x", + ] { + assert_eq!(parse(name), None, "{name}"); + } + } + + #[test] + fn an_id_is_one_segment() { + let path = Path::from_static("room"); + assert_eq!(side(&path, DIAL, "alice").unwrap().as_str(), "room/.dial/alice"); + assert!(side(&path, DIAL, "a/b").is_err()); + assert!(side(&path, DIAL, "").is_err()); + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 43d70c43bf5..ce58865c132 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -62,6 +62,7 @@ mod session_panic; mod settled_end_survives_abort; mod setup_negotiation; mod slow_joiner; +mod socket; mod source_errors; mod stream_credit; mod subscribe_priority; diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs new file mode 100644 index 00000000000..42a403e938d --- /dev/null +++ b/rs/moq-net/tests/integration/socket.rs @@ -0,0 +1,290 @@ +#![cfg(feature = "async")] + +//! Sockets through a relay: a dialer and a listener on their own origins, each with a +//! session to a relay, over moq-lite-07. + +use std::time::Duration; + +use moq_net::socket::{self, Closed, Listener, Socket}; +use moq_net::{Hop, Instance, Version, origin, track}; + +use crate::support::harness::{MockConnectOptions, MockPair, connect_mock, spawn}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Long enough for every hop to settle, in simulated time. +const SETTLE: Duration = Duration::from_secs(1); + +fn origin(hop: u64) -> origin::Producer { + let (origin, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + spawn(driver); + origin +} + +/// A client publishing and subscribing through one origin, connected to `relay`. +async fn client(origin: &origin::Producer, relay: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); + options.client_publish = Some(origin.consume()); + options.client_subscribe = Some(origin.clone()); + options.server_publish = Some(relay.consume()); + options.server_subscribe = Some(relay.clone()); + connect_mock(options).await +} + +/// A relay with a listener client and a dialer client. +struct Fixture { + relay: origin::Producer, + dialer: origin::Producer, + listener: origin::Producer, + sessions: Vec, +} + +impl Fixture { + async fn new() -> Self { + let relay = origin(1); + let dialer = origin(2); + let listener = origin(3); + let sessions = vec![client(&dialer, &relay).await, client(&listener, &relay).await]; + Self { + relay, + dialer, + listener, + sessions, + } + } + + /// Another dialer client of the same relay. + async fn dialer(&mut self, hop: u64) -> origin::Producer { + let dialer = origin(hop); + self.sessions.push(client(&dialer, &self.relay).await); + dialer + } +} + +/// Dial `id` from `origin` while `listener` accepts, returning both ends. +async fn connect(origin: &origin::Producer, listener: &mut Listener, id: &str) -> (Socket, Socket) { + let dial = socket::dial(origin, "room", id).unwrap(); + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, listener.accept()) }) + .await + .expect("connect timeout"); + (dialed.expect("dial"), accepted.expect("accept")) +} + +/// Write `payload` as one group of `track`. +fn send(track: &track::Producer, payload: &'static [u8]) { + let mut group = track.append_group().unwrap(); + group.write_frame(None, payload).unwrap(); + group.finish().unwrap(); +} + +/// Subscribe to `track` and read its first frame. +async fn recv(track: &track::Consumer) -> Vec { + kio::clock::timeout(TIMEOUT, async { + let mut subscriber = track.subscribe(None).await.expect("subscribe"); + let mut group = subscriber.recv_group().await.expect("group").expect("a group"); + group + .read_frame() + .await + .expect("frame") + .expect("a frame") + .payload + .to_vec() + }) + .await + .expect("read timeout") +} + +/// Uni and bi streams each way, through the relay. +#[kio::test] +async fn streams_flow_both_ways() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + assert_eq!(dialer.id(), "alice"); + assert_eq!(accepted.id(), "alice"); + + // A uni stream each way. + let up = dialer.open_uni("up", None).unwrap(); + send(&up, b"from the dialer"); + let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(&*uni.name, "up"); + assert_eq!(recv(&uni.consumer).await, b"from the dialer"); + + let down = accepted.open_uni("down", None).unwrap(); + send(&down, b"from the listener"); + let uni = kio::clock::timeout(TIMEOUT, dialer.accept_uni()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(&*uni.name, "down"); + assert_eq!(recv(&uni.consumer).await, b"from the listener"); + + // A bi stream each way, under the same name: the opener's role keeps them apart. + for (opener, acceptor) in [(&dialer, &accepted), (&accepted, &dialer)] { + let info = track::Info::default().with_priority(3); + let open = opener.open_bi("chat", info).unwrap(); + let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(open, acceptor.accept_bi()) }) + .await + .expect("bi timeout"); + let opened = opened.expect("open"); + let answered = answered.expect("accept").expect("a stream"); + assert_eq!(&*answered.name, "chat"); + assert_eq!( + answered.producer.info().priority, + 3, + "the answer mirrors the opener's info" + ); + + send(&opened.producer, b"ping"); + assert_eq!(recv(&answered.consumer).await, b"ping"); + send(&answered.producer, b"pong"); + assert_eq!(recv(&opened.consumer).await, b"pong"); + } + + // Neither side's own streams come back as the peer's. + assert!( + kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err(), + "the dialer accepted a stream it opened" + ); + assert!( + kio::clock::timeout(SETTLE, dialer.accept_uni()).await.is_err(), + "the dialer accepted a stream twice" + ); +} + +/// A dialer that dials again with the same id, as a restarted one does, ends the socket +/// it replaces, and the listener accepts the new one. +#[kio::test] +async fn a_new_dial_replaces_the_old_socket() { + let mut fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (old_dialer, old) = connect(&fixture.dialer, &mut listener, "alice").await; + + let restarted = fixture.dialer(4).await; + let (new_dialer, new) = connect(&restarted, &mut listener, "alice").await; + + assert_eq!( + kio::clock::timeout(TIMEOUT, old.closed()) + .await + .expect("old socket open"), + Closed::Replaced + ); + assert_eq!( + kio::clock::timeout(TIMEOUT, old_dialer.closed()) + .await + .expect("old dialer open"), + Closed::Remote + ); + + // The new socket is the new dialer's. + let up = new_dialer.open_uni("up", None).unwrap(); + send(&up, b"again"); + let uni = kio::clock::timeout(TIMEOUT, new.accept_uni()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(recv(&uni.consumer).await, b"again"); + assert!( + kio::clock::timeout(SETTLE, new.closed()).await.is_err(), + "the new socket ended" + ); +} + +/// A dialer the listener has not accepted is not read: its tracks see no demand until +/// the application accepts it. +#[kio::test] +async fn an_unaccepted_dialer_holds_no_subscription() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + + // A dialer by hand, so the test can watch its broadcast. + let dial = fixture.dialer.create_broadcast("room/.dial/bob").unwrap(); + let probe = dial.create_track("probe", None).unwrap(); + dial.announce(origin::Route::default().with_instance(Instance::random(), 0)) + .unwrap(); + + let mut demand = probe.demand(); + assert!(demand.wanted_changed().await.unwrap().is_none()); + assert!( + kio::clock::timeout(SETTLE, demand.wanted_changed()).await.is_err(), + "an unaccepted dialer was read" + ); + + let accepted = kio::clock::timeout(TIMEOUT, listener.accept()) + .await + .expect("accept timeout") + .unwrap(); + assert_eq!(accepted.id(), "bob"); + let _subscriber = kio::clock::timeout(TIMEOUT, accepted.remote().track("probe").unwrap().subscribe(None)) + .await + .expect("subscribe timeout") + .unwrap(); + assert!(demand.wanted().is_some(), "the accepted dialer is read"); +} + +/// A listener going away ends the sockets it accepted. +#[kio::test] +async fn a_listener_restart_ends_its_sockets() { + let mut fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, _accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + // The listener's session drops, as when its process restarts. + drop(fixture.sessions.remove(1)); + assert_eq!( + kio::clock::timeout(TIMEOUT, dialer.closed()) + .await + .expect("socket open"), + Closed::Remote + ); +} + +/// A dial resolves only on the answer to it: neither an answer carrying another instance +/// at its path nor another id's answer is taken for it. +#[kio::test] +async fn a_dial_takes_only_its_own_answer() { + let mut fixture = Fixture::new().await; + + // An answer at alice's path for some other dial of alice, such as a stale one. + let stale = fixture + .listener + .publish( + "room/.accept/alice", + origin::Route::default().with_instance(Instance::random(), 0), + ) + .unwrap(); + let dial = socket::dial(&fixture.dialer, "room", "alice").unwrap(); + let mut dial = std::pin::pin!(dial); + assert!( + kio::clock::timeout(SETTLE, dial.as_mut()).await.is_err(), + "the dial took another dial's answer" + ); + drop(stale); + + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let alice = kio::clock::timeout(TIMEOUT, listener.accept()) + .await + .expect("accept timeout") + .unwrap(); + assert_eq!(alice.id(), "alice"); + let dialer = kio::clock::timeout(TIMEOUT, dial).await.expect("dial timeout").unwrap(); + assert_eq!(dialer.id(), "alice"); + assert_eq!( + dialer.remote().info().instance, + alice.remote().info().instance, + "the answer carries the dial's instance" + ); + + // Another id is its own socket, never alice's. + let bob = fixture.dialer(4).await; + let (bob_dialer, bob_accepted) = connect(&bob, &mut listener, "bob").await; + assert_eq!(bob_dialer.id(), "bob"); + assert_eq!(bob_accepted.id(), "bob"); + assert_ne!(bob_accepted.remote().info().instance, alice.remote().info().instance); +} From 26db1cedf925f79fb268319b6c5ab437f6cc4231 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:06:46 -0800 Subject: [PATCH 03/25] doc: specify sockets on a concept page and finish the Rust quest Co-Authored-By: Claude Opus 5.5 --- doc/.vitepress/config.ts | 1 + doc/concept/index.md | 1 + doc/concept/socket.md | 103 ++++++++++++++++++++++++++++++++++++++ doc/lib/rs/moq-net.md | 1 + quest/m1/socket/README.md | 29 +++++++---- quest/m1/socket/boy.md | 1 - quest/m1/socket/js.md | 6 +-- quest/m1/socket/py.md | 6 +-- quest/m1/socket/rust.md | 39 --------------- quest/m2/mesh/endpoint.md | 3 +- quest/m2/stream/README.md | 1 - quest/m2/stream/rust.md | 3 +- rs/moq-net/CHANGELOG.md | 1 + 13 files changed, 129 insertions(+), 66 deletions(-) create mode 100644 doc/concept/socket.md delete mode 100644 quest/m1/socket/rust.md diff --git a/doc/.vitepress/config.ts b/doc/.vitepress/config.ts index 0a11949980d..d5134cdd50c 100644 --- a/doc/.vitepress/config.ts +++ b/doc/.vitepress/config.ts @@ -118,6 +118,7 @@ export default defineConfig({ { text: "Transport", link: "/concept/transport" }, { text: "moq-lite", link: "/concept/moq-lite" }, { text: "Identity and recovery", link: "/concept/identity" }, + { text: "Sockets", link: "/concept/socket" }, { text: "hang", link: "/concept/hang" }, { text: "Archives", link: "/concept/archives" }, { text: "Audio jitter", link: "/concept/audio-jitter" }, diff --git a/doc/concept/index.md b/doc/concept/index.md index 6f129d15cd8..b51f5d7a8dd 100644 --- a/doc/concept/index.md +++ b/doc/concept/index.md @@ -36,6 +36,7 @@ watched at 100 ms by one viewer and 10 s by another. - [Transport](/concept/transport): why QUIC, and the WebTransport, WebSocket, raw QUIC, and iroh paths onto it. - [moq-lite](/concept/moq-lite): the pub/sub protocol, discovery, path patterns, subscriptions, and congestion behavior. - [Identity and recovery](/concept/identity): choosing stream instances, following restarts, and answering work on demand without mixing streams. +- [Sockets](/concept/socket): dial and accept point-to-point connections through any relay, with tracks as streams. - [hang](/concept/hang): the media catalog, containers, encryption, and how to extend both. - [Archives](/concept/archives): record listed tracks into object storage and serve them back from a host. - [Audio jitter](/concept/audio-jitter): how a receiver sizes its audio playout target from arrival timing. diff --git a/doc/concept/socket.md b/doc/concept/socket.md new file mode 100644 index 00000000000..58699effec1 --- /dev/null +++ b/doc/concept/socket.md @@ -0,0 +1,103 @@ +--- +title: Sockets +description: Dial and accept point-to-point connections over moq-lite, with tracks as streams +--- + +# Sockets + +A socket connects one dialer to one listener through any relay, the way a TCP +connection does. A listener at a path accepts the sockets that dialers open to +it, and inside a socket either side opens and accepts tracks the way QUIC opens +and accepts streams. Use it for per-client state that only the two ends need, +such as a viewer's commands and the replies to them, while whatever every +client needs stays in an ordinary broadcast. + +A socket is a convention over moq-lite's announcements, broadcasts, and track +listings, not a wire extension, so relays carry it unchanged. This page is the +convention, enough to implement it in any language. +[`moq_net::socket`](https://docs.rs/moq-net/latest/moq_net/socket/) implements +it in Rust. + +## Paths + +A listener at `L` keeps `L` itself for what it shares with everyone. Each +socket is two broadcasts beneath it: + +| Path | Published by | Holds | +| --- | --- | --- | +| `L/.dial/` | The dialer | The dialer's tracks | +| `L/.accept/` | The listener | The listener's tracks | + +The id is one path segment that the dialer chooses, such as a random id or an +endpoint ID. Both segments start with `.`, so sockets are +[hidden](/concept/moq-lite#hidden-broadcasts) from anyone listing `L`. The +listener discovers dialers with a one-segment watch of `L/.dial/*`, which a +relay serves without scanning everything beneath `L`. + +Authorization uses ordinary path grants. A dialer needs publish on +`L/.dial/` and subscribe on `L/.accept/`, and the listener the reverse +for every id: subscribe on `L/.dial/*` and publish on `L/.accept/*`. A grant +naming the id is what stops one dialer from taking another's socket. + +## Instances + +Each dial is one socket, named by its +[instance](/concept/moq-lite#publisher-instances): + +1. The dialer announces `L/.dial/` under a fresh random instance and + follows `L/.accept/` pinned to that instance. +2. When the application accepts, the listener requests `L/.dial/` pinned + to the instance it saw announced, then announces `L/.accept/` under the + same instance at epoch 0. +3. The dialer's follow resolves on that answer, and the socket is open. + +An answer to another dial of the same id carries another instance, so a +dialer never takes it. Dialing again with the same id, as a restarted dialer +does, announces a new instance: the listener ends the socket it replaces and +answers the new one on its next accept. + +A dial waits for as long as it takes, so give up by dropping it. Nothing reads +a dialer before the application accepts it: dialers wait in the listener's +announcements without limit, and its broadcast is requested only on accept. + +## Streams + +A stream is a track, named by who opened it and which way it carries data: + +```text +.// +``` + +`` is `dial` or `accept`, `` is `uni` or `bi`, and `` is +the application's name for the stream, which may hold `/`. + +- **Uni.** The opener publishes the track in its own broadcast. The peer + accepts it and subscribes. +- **Bi.** The opener publishes the track in its own broadcast. The peer accepts + it by publishing the same name in its broadcast, with the opener's track + info, and each side subscribes to the other's. The opener's stream resolves + once the answer is listed. + +The opener prefix keeps the two sides' names apart, the way the low bit of a +QUIC stream ID does: the dialer's bi `chat` and the listener's bi `chat` are +two streams. A track's info, priority included, is the opener's to choose. + +The peer learns of new streams from the broadcast's track listing (the Tracks +stream, `TRACKS_ADD`). A listing reports every track the broadcast holds, so +accept skips any name outside the scheme. That leaves applications free to use +fixed track names in either broadcast beside the streams. + +## Ending + +A socket ends when either side's broadcast ends, which includes either side +closing it or a session on the way dropping, or when a newer dial of its id +replaces it. Each side then drops its socket, which retracts its broadcast and +ends the socket for the peer. + +## Versions + +Sockets need moq-lite 07 (`moq-lite-07-wip`, opt-in) on every hop: older +moq-lite versions and moq-transport carry neither instances nor track +listings. Over an older hop, accepting a stream fails with `Unsupported`, and +a dial never resolves, because the answer no longer carries the dial's +instance. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 0ed50e2dfe7..1f6e06d3fb3 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,6 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. +- **Dial and accept sockets.** `socket::Listener::new(&origin, path)` accepts the sockets that `socket::dial(&origin, path, id)` opens, and either side of a `Socket` opens and accepts uni and bi streams as tracks or reads the peer's broadcast by fixed track names. A dial waits until accepted; drop it to give up. Needs moq-lite 07 on every hop; see [sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index 06829fa33d7..80b7098835e 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -45,21 +45,29 @@ replacement arc): `moq-pattern` supports. The dot keeps sessions out of ordinary discovery ([hidden broadcasts](/doc/concept/moq-lite.md#hidden-broadcasts)), so boy's game list never sees viewers. -- Identity is a dialer-chosen id plus a pinned epoch. The id is the dialer's - own: an endpoint ID in the mesh, a random id in the demos. Each announce - epoch is one socket: a new epoch ends the old socket and starts a new one, - and the listener's request is pinned to the epoch (`origin::Consumer::request` - with an owned `Epoch`, as CARLA already does), so a restarted dialer never - inherits state. A socket ends when either side's broadcast ends or its epoch - is replaced. +- Identity is a dialer-chosen id plus a pinned instance (the epic first + called it an epoch; the code's term is the route's `Instance`). The id is the + dialer's own: an endpoint ID in the mesh, a random id in the demos. Each dial + instance is one socket: a new instance ends the old socket and starts a new + one, and the listener's request is pinned to the instance + (`origin::Consumer::request` with an owned `Instance`, as CARLA already + does), so a restarted dialer never inherits state. A socket ends when either + side's broadcast ends or its instance is replaced. Settled by the Rust + implementation: the listener's answer carries the dial's instance, and the + dialer follows its answer pinned to it, so it never takes an answer to an + older dial of its id. A dial waits indefinitely; callers bound it by dropping + it. - Streams are tracks. A uni stream is a track the opener publishes in its own socket broadcast. A bi stream is the same track name in both. The peer learns of new tracks from the Lite07 Tracks stream (`TRACKS_ADD`, `REMOVE`, `FINISH`), which pushes them like QUIC's accept. `requested_track` was declined because it is pull-shaped: only the side being subscribed to can - accept. Track names opened by the two sides must never collide (for example - a role prefix, like QUIC's stream-id low bit); the Rust quest settles the - scheme. + accept. Settled by the Rust implementation: tracks are named + `.//` with `` `dial` or `accept`, so the two + sides' names never collide; accept skips names outside the scheme, so fixed + track names coexist; the opener chooses the track info (priority included) + and a bi answer mirrors it. [`doc/concept/socket.md`](/doc/concept/socket.md) + is the convention JS and Python follow. - Accept is unbounded and lazy. Announced dialers queue without limit, and a dialer's broadcast is requested only when the application accepts it, so an application that stops accepting holds no subscriptions. Backpressure on the @@ -81,7 +89,6 @@ announce-watch-request pairing remains in `rs/moq-boy`, `js/moq-boy`, or ## Required -- [Rust sockets](/quest/m1/socket/rust.md) - `moq_net::socket` with `Listener`, `dial`, and `Socket`, plus the concept page - [JS sockets](/quest/m1/socket/js.md) - the same surface in `@moq/net`, interoperating with Rust - [Python sockets](/quest/m1/socket/py.md) - moq-ffi and `py/moq-rs` bindings, interoperating with Rust - [moq-boy on sockets](/quest/m1/socket/boy.md) - the emulator accepts viewers through a listener and echoes latency per socket diff --git a/quest/m1/socket/boy.md b/quest/m1/socket/boy.md index c00ea1d2909..7c63c25b43d 100644 --- a/quest/m1/socket/boy.md +++ b/quest/m1/socket/boy.md @@ -29,5 +29,4 @@ See the [epic](/quest/m1/socket/README.md) for the layout and the reply split. ## Required -- [Rust sockets](/quest/m1/socket/rust.md) - the emulator's listener - [JS sockets](/quest/m1/socket/js.md) - the viewer's dial diff --git a/quest/m1/socket/js.md b/quest/m1/socket/js.md index 5d7e930e55c..74c03bfbd8c 100644 --- a/quest/m1/socket/js.md +++ b/quest/m1/socket/js.md @@ -9,10 +9,6 @@ and accept uni and bi streams through a relay. ## Plan Follow `doc/concept/socket.md` and mirror the -[Rust surface](/quest/m1/socket/rust.md), including its track-naming scheme. +Rust surface (`rs/moq-net/src/model/socket.rs`), including its track-naming scheme. Add the Rust/JS pair to `just test interop`, and document the export in `doc/lib/js/net.md`. - -## Required - -- [Rust sockets](/quest/m1/socket/rust.md) - the reference surface and concept page diff --git a/quest/m1/socket/py.md b/quest/m1/socket/py.md index 7bd6904159d..e471d6c1dac 100644 --- a/quest/m1/socket/py.md +++ b/quest/m1/socket/py.md @@ -8,11 +8,7 @@ and the reverse, with uni and bi streams each way. ## Plan -Bind the [Rust surface](/quest/m1/socket/rust.md) through moq-ffi the way +Bind the Rust surface (`moq_net::socket`) through moq-ffi the way origin's request and announce APIs are bound today (`rs/moq-ffi/src/origin.rs`, `py/moq-rs/moq/origin.py`). Add a Python/Rust interop test, and document the API wherever `py/moq-rs` documents origin. - -## Required - -- [Rust sockets](/quest/m1/socket/rust.md) - the surface this binds diff --git a/quest/m1/socket/rust.md b/quest/m1/socket/rust.md deleted file mode 100644 index dc382147ec4..00000000000 --- a/quest/m1/socket/rust.md +++ /dev/null @@ -1,39 +0,0 @@ -# [M] Rust sockets - -## Goal - -`moq_net::socket` lets a Rust application bind a `Listener` at a path and -`accept()` the sockets dialed to it, or `dial(origin, path, id)` one itself. -Either side of a `Socket` can `open_uni`, `open_bi`, `accept_uni`, and -`accept_bi` tracks, and learns when the socket ends. `doc/concept/socket.md` -specifies the convention, so other languages can implement it from the page. - -## Plan - -The [epic](/quest/m1/socket/README.md) records the decisions: the -`L/.dial/` and `L/.accept/` layout, a dialer-chosen id with a pinned -epoch, tracks as streams discovered through the Tracks stream, and lazy, -unbounded accept. - -- `Listener` watches `L/.dial/*` and queues each announced `(id, epoch)`. - `accept()` requests the dialer's broadcast pinned to its epoch and publishes - `L/.accept/`. A new epoch for a queued or live id replaces it. -- `dial` publishes `L/.dial/` and requests `L/.accept/`; it resolves - once the listener answers. Settle and record here whether a dial times out - or waits indefinitely for a listener. -- `Socket` holds the local broadcast producer and the remote consumer, and - exposes both for applications that use fixed track names (boy, CARLA) - rather than open/accept. -- A bi stream returns a track producer and consumer pair; a uni stream - returns one end. Settle and record the naming scheme that keeps the two - sides' track names from colliding. Each opened track takes a priority from - the caller. -- The socket ends with a typed error when either broadcast ends, its epoch - is replaced, or the session drops. -- Write `doc/concept/socket.md` (paths, epochs, tracks as streams, ending) - and link it from the concept index and `doc/lib/rs`. -- Tests on controlled time with no sleeps: dial and accept through a relay; - a bi and a uni stream each way; a dialer re-announcing with a new epoch - ends the old socket; an unaccepted dialer holds no subscription; a listener - restart ends its sockets; a dialer for another id cannot be accepted as - this one. diff --git a/quest/m2/mesh/endpoint.md b/quest/m2/mesh/endpoint.md index 82f6ebc79a4..4584065e865 100644 --- a/quest/m2/mesh/endpoint.md +++ b/quest/m2/mesh/endpoint.md @@ -16,7 +16,7 @@ Decided 2026-10-25 (maintainer): iroh-shaped, not drop-in compatible; tickets, - A service is the iroh ALPN analog: a name under `mesh//`. - Decided 2026-11-01 (maintainer, socket planning interview): an `Endpoint` - wraps [sockets](/quest/m1/socket/rust.md). A service is a `Listener` at + wraps [sockets](/doc/concept/socket.md). A service is a `Listener` at `mesh//`, `connect` dials it with the caller's own endpoint ID, and `accept` reads the remote endpoint ID from the `.dial/` segment, which the grant pins. Extend `moq auth`'s mesh grant with publish on @@ -34,5 +34,4 @@ Decided 2026-10-25 (maintainer): iroh-shaped, not drop-in compatible; tickets, ## Required -- [Rust sockets](/quest/m1/socket/rust.md) - the dial and accept this wraps - [Rust byte streams](/quest/m2/stream/rust.md) - the stream this API returns diff --git a/quest/m2/stream/README.md b/quest/m2/stream/README.md index 87cd7854dcb..a2659f78d3d 100644 --- a/quest/m2/stream/README.md +++ b/quest/m2/stream/README.md @@ -54,7 +54,6 @@ the window. ## Required -- [Rust sockets](/quest/m1/socket/rust.md) - the dial/accept primitive and tracks as streams - [Fastest-reader pacing](/quest/m1/unbounded-groups/fastest-reader.md) - relays push back instead of evicting a reader that keeps up - [Rust byte streams](/quest/m2/stream/rust.md) - a duplex `AsyncRead`/`AsyncWrite` stream, a listener and a dialer, and `moq` CLI forwarding - [JS byte streams](/quest/m2/stream/js.md) - the same surface in `@moq/net` diff --git a/quest/m2/stream/rust.md b/quest/m2/stream/rust.md index 1a7b7404a4a..a053e883b36 100644 --- a/quest/m2/stream/rust.md +++ b/quest/m2/stream/rust.md @@ -3,7 +3,7 @@ ## Goal A Rust application opens a duplex byte stream on a -[socket](/quest/m1/socket/rust.md) and gets an `AsyncRead + AsyncWrite` that +[socket](/doc/concept/socket.md) and gets an `AsyncRead + AsyncWrite` that carries TCP semantics over one bi track: ordered, lossless, FIN as half-close, reset with a code. A socket's listener accepts the streams its dialers open. `moq` gains TCP forwarding on both ends so an @@ -32,5 +32,4 @@ per TCP read, no fill handler. ## Required -- [Rust sockets](/quest/m1/socket/rust.md) - the socket each stream is a bi track on - [Fastest-reader pacing](/quest/m1/unbounded-groups/fastest-reader.md) - lossless delivery to a reader that keeps up diff --git a/rs/moq-net/CHANGELOG.md b/rs/moq-net/CHANGELOG.md index 456e1513be5..ad70164d532 100644 --- a/rs/moq-net/CHANGELOG.md +++ b/rs/moq-net/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `socket`: `Listener`, `dial`, and `Socket`, point-to-point connections over an origin whose streams are tracks; see `doc/concept/socket.md`. - `transport::SETUP_RECEIVE_WINDOW`, the connection credit a server offers before it admits a session: enough for the SETUP and one in-band AUTH. ## [0.3.10](https://github.com/moq-dev/moq/compare/moq-net-v0.3.9...moq-net-v0.3.10) - 2026-10-05 From 90147b47d9c6f8125e60ee158ad301f563a9c080 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:09:14 -0800 Subject: [PATCH 04/25] test(net): let socket discovery follow its own watches Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/socket.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 42a403e938d..3cdf7b59563 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -28,6 +28,8 @@ async fn client(origin: &origin::Producer, relay: &origin::Producer) -> MockPair options.client_subscribe = Some(origin.clone()); options.server_publish = Some(relay.consume()); options.server_subscribe = Some(relay.clone()); + // Discovery follows the socket's own watches, as it does for an application. + options.client_discovers_all = false; connect_mock(options).await } From 9b72bb3ce7b4bbb428310b1781163549a7019808 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:15:50 -0800 Subject: [PATCH 05/25] fix(net): bound socket stream queues, register bi opens first, close on drop Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/socket.rs | 70 +++++++++++++++++++------- rs/moq-net/tests/integration/socket.rs | 58 +++++++++++++++++++++ 2 files changed, 111 insertions(+), 17 deletions(-) diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 67b1e6afdc2..3b1fcf04dfa 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -12,10 +12,12 @@ //! The peer learns of new streams from the broadcast's track listing, which needs //! moq-lite-07 on every hop. See `doc/concept/socket.md` for the convention. -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::sync::{Arc, Weak}; use std::task::{Poll, ready}; +use hashlink::LinkedHashMap; + use crate::{Error, Instance, Path, Pattern, Patterns}; use super::broadcast::{self, TracksEvent}; @@ -137,10 +139,11 @@ pub struct Bi { struct Listing { /// The peer's listing, opened by the first accept or bi open. tracks: Option, - /// Uni streams the peer opened, not yet accepted. - uni: VecDeque>, + /// Uni streams the peer opened, not yet accepted, in order. A stream the peer withdraws + /// unfinished leaves, so these hold no more than the peer lists plus what it finished. + uni: LinkedHashMap, ()>, /// Bi streams the peer opened, with the opener's track info, not yet accepted. - bi: VecDeque<(Arc, track::Info)>, + bi: LinkedHashMap, track::Info>, /// This side's bi opens waiting for an answer, and whether the peer listed it. opening: HashMap, bool>, /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. @@ -184,24 +187,36 @@ impl Listing { return; } }; - // Only additions open streams; a stream ends through its own track. - let TracksEvent::Added { name, info } = event else { - continue; + // A finished stream stays readable, so only a withdrawal takes one back. + let (name, info) = match event { + TracksEvent::Added { name, info } => (name, Some(info)), + TracksEvent::Removed { name } => (name, None), + TracksEvent::Synced | TracksEvent::Finished { .. } => continue, }; // A track outside the scheme is the application's own. let Some((opener, kind, stream)) = parse(&name) else { continue; }; - match (opener == role, kind) { - (false, Kind::Uni) => self.uni.push_back(stream.into()), - (false, Kind::Bi) => self.bi.push_back((stream.into(), info)), - (true, Kind::Bi) => { + match (opener == role, kind, info) { + (false, Kind::Uni, Some(_)) => { + self.uni.replace(stream.into(), ()); + } + (false, Kind::Uni, None) => { + self.uni.remove(stream); + } + (false, Kind::Bi, Some(info)) => { + self.bi.replace(stream.into(), info); + } + (false, Kind::Bi, None) => { + self.bi.remove(stream); + } + (true, Kind::Bi, info) => { if let Some(answered) = self.opening.get_mut(stream) { - *answered = true; + *answered = info.is_some(); } } // This side's uni streams live only in its own broadcast. - (true, Kind::Uni) => {} + (true, Kind::Uni, _) => {} } } } @@ -289,11 +304,25 @@ impl Socket { /// [`Error::Duplicate`] when this side already opened a bi stream of that name. pub fn open_bi(&self, name: &str, info: impl Into>) -> Result, Error> { let name: Arc = name.into(); - let producer = self + // Registered first, so a poll that reads the peer's answer before this returns keeps it. + { + let mut listing = self.inner.listing.lock(); + if listing.opening.contains_key(&name) { + return Err(Error::Duplicate); + } + listing.opening.insert(name.clone(), false); + } + let producer = match self .inner .local - .create_track(track_name(self.inner.role, Kind::Bi, &name), info)?; - self.inner.listing.lock().opening.insert(name.clone(), false); + .create_track(track_name(self.inner.role, Kind::Bi, &name), info) + { + Ok(producer) => producer, + Err(err) => { + self.inner.listing.lock().opening.remove(&name); + return Err(err); + } + }; Ok(kio::Pending::new(OpenBi { inner: self.inner.clone(), name, @@ -306,7 +335,7 @@ impl Socket { let inner = &self.inner; let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - if let Some(name) = listing.uni.pop_front() { + if let Some((name, ())) = listing.uni.pop_front() { drop(listing); return Poll::Ready( match inner.remote.track(&track_name(inner.role.peer(), Kind::Uni, &name)) { @@ -402,6 +431,13 @@ impl Socket { } } +impl Drop for Socket { + fn drop(&mut self) { + // A pending bi open still holds the shared half, but not the socket. + self.inner.local.close(); + } +} + /// A bi stream being opened, from [`Socket::open_bi`]: resolves once the peer answers. pub struct OpenBi { inner: Arc, diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 3cdf7b59563..0e2b836a2f2 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -290,3 +290,61 @@ async fn a_dial_takes_only_its_own_answer() { assert_eq!(bob_accepted.id(), "bob"); assert_ne!(bob_accepted.remote().info().instance, alice.remote().info().instance); } + +/// A stream the peer withdraws unfinished before it is accepted is never handed out, +/// while one it finished still is. +#[kio::test] +async fn a_withdrawn_stream_is_not_accepted() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + let gone = dialer.open_uni("gone", None).unwrap(); + let done = dialer.open_uni("done", None).unwrap(); + send(&done, b"finished"); + done.finish().unwrap(); + // Let the listener's listing hear of both before either ends. + assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); + drop(gone); + drop(done); + // And of the withdrawal. + assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); + + let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(&*uni.name, "done"); + assert!( + kio::clock::timeout(SETTLE, accepted.accept_uni()).await.is_err(), + "a withdrawn stream was accepted" + ); +} + +/// Opening a bi stream of a name still being opened fails without disturbing the first, +/// and dropping the socket ends it for the peer even while an open is pending. +#[kio::test] +async fn a_pending_open_neither_doubles_nor_outlives_its_socket() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + let first = dialer.open_bi("chat", None).unwrap(); + assert!(matches!(dialer.open_bi("chat", None), Err(moq_net::Error::Duplicate))); + let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) + .await + .expect("bi timeout"); + opened.expect("open"); + answered.expect("accept").expect("a stream"); + + let pending = dialer.open_bi("unanswered", None).unwrap(); + drop(dialer); + assert_eq!( + kio::clock::timeout(TIMEOUT, accepted.closed()) + .await + .expect("socket open"), + Closed::Remote + ); + drop(pending); +} From f57d27070e9abee47346933564ec6c6b5d5a18bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:19:40 -0800 Subject: [PATCH 06/25] fix(net): hand out only the socket streams the peer still lists Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 5 +++++ rs/moq-net/src/model/socket.rs | 19 ++++++++++++------- rs/moq-net/tests/integration/socket.rs | 19 ++++++++++--------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 58699effec1..07b6030d133 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -87,6 +87,11 @@ stream, `TRACKS_ADD`). A listing reports every track the broadcast holds, so accept skips any name outside the scheme. That leaves applications free to use fixed track names in either broadcast beside the streams. +A stream can be accepted only while its opener lists it, which holds until the +opener drops the track, finished or not. One dropped before the peer accepts it +is never handed out, so a backlog of unaccepted streams costs the peer no more +than the tracks the opener keeps. + ## Ending A socket ends when either side's broadcast ends, which includes either side diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 3b1fcf04dfa..0d8706f6db4 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -139,8 +139,9 @@ pub struct Bi { struct Listing { /// The peer's listing, opened by the first accept or bi open. tracks: Option, - /// Uni streams the peer opened, not yet accepted, in order. A stream the peer withdraws - /// unfinished leaves, so these hold no more than the peer lists plus what it finished. + /// Uni streams the peer opened, not yet accepted, in order. A stream leaves once the + /// peer stops listing it, as the listing drops an add its reader never reached, so + /// these hold no more than the peer lists. uni: LinkedHashMap, ()>, /// Bi streams the peer opened, with the opener's track info, not yet accepted. bi: LinkedHashMap, track::Info>, @@ -187,11 +188,10 @@ impl Listing { return; } }; - // A finished stream stays readable, so only a withdrawal takes one back. let (name, info) = match event { TracksEvent::Added { name, info } => (name, Some(info)), - TracksEvent::Removed { name } => (name, None), - TracksEvent::Synced | TracksEvent::Finished { .. } => continue, + TracksEvent::Removed { name } | TracksEvent::Finished { name } => (name, None), + TracksEvent::Synced => continue, }; // A track outside the scheme is the application's own. let Some((opener, kind, stream)) = parse(&name) else { @@ -210,11 +210,13 @@ impl Listing { (false, Kind::Bi, None) => { self.bi.remove(stream); } - (true, Kind::Bi, info) => { + // An answer seen once stays seen: the opener reads whatever it wrote. + (true, Kind::Bi, Some(_)) => { if let Some(answered) = self.opening.get_mut(stream) { - *answered = info.is_some(); + *answered = true; } } + (true, Kind::Bi, None) => {} // This side's uni streams live only in its own broadcast. (true, Kind::Uni, _) => {} } @@ -331,6 +333,9 @@ impl Socket { } /// Poll for the next uni stream the peer opened; `None` once the peer's broadcast ended. + /// + /// A stream is handed out only while the peer still lists it: one whose track the peer + /// dropped first, finished or not, is skipped. pub fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll, Error>> { let inner = &self.inner; let mut listing = inner.listing.lock(); diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 0e2b836a2f2..edf495aba57 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -291,23 +291,23 @@ async fn a_dial_takes_only_its_own_answer() { assert_ne!(bob_accepted.remote().info().instance, alice.remote().info().instance); } -/// A stream the peer withdraws unfinished before it is accepted is never handed out, -/// while one it finished still is. +/// A stream is accepted only while its opener lists it: one whose producer the opener +/// dropped first, finished or not, is never handed out. #[kio::test] -async fn a_withdrawn_stream_is_not_accepted() { +async fn a_stream_is_accepted_only_while_listed() { let fixture = Fixture::new().await; let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; let gone = dialer.open_uni("gone", None).unwrap(); let done = dialer.open_uni("done", None).unwrap(); - send(&done, b"finished"); - done.finish().unwrap(); - // Let the listener's listing hear of both before either ends. + let live = dialer.open_uni("live", None).unwrap(); + // Let the listener's listing hear of all three before any ends. assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); drop(gone); + done.finish().unwrap(); drop(done); - // And of the withdrawal. + // And of the ends. assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) @@ -315,11 +315,12 @@ async fn a_withdrawn_stream_is_not_accepted() { .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(&*uni.name, "done"); + assert_eq!(&*uni.name, "live"); assert!( kio::clock::timeout(SETTLE, accepted.accept_uni()).await.is_err(), - "a withdrawn stream was accepted" + "a stream that left the listing was accepted" ); + drop(live); } /// Opening a bi stream of a name still being opened fails without disturbing the first, From dec0c2346484dddd0df7b6a23d48846667999a5a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:54:18 -0800 Subject: [PATCH 07/25] fix(net): end socket opens on local close, guard bi reopens - A pending `OpenBi` now fails `Closed` once this side closes, so a peer that keeps its broadcast up without answering cannot hold it forever, and a late answer no longer hands back a track in an ended broadcast. - `open_uni`, `open_bi`, and the accepts refuse with `Closed` once this side closed, checked after the create so a racing close is seen. - Accepts skip a stream whose track the peer dropped between the listing and the lookup (`NotFound`), as their docs already promised. - `open_bi` refuses a name while the peer still lists its answer to an earlier open: the peer cannot answer it again until it releases that answer, so the open would wait forever. - Docs: opt-in moq-lite 07 on the concept index, the socket page's opening, and the moq-net page; one listener per path; why dials are accepted in order. Co-Authored-By: Claude Opus 5.5 --- doc/concept/index.md | 2 +- doc/concept/socket.md | 15 ++-- doc/lib/rs/moq-net.md | 2 +- rs/moq-net/src/model/socket.rs | 116 +++++++++++++++++-------- rs/moq-net/tests/integration/socket.rs | 88 +++++++++++++++++++ 5 files changed, 182 insertions(+), 41 deletions(-) diff --git a/doc/concept/index.md b/doc/concept/index.md index b51f5d7a8dd..6655e0e0317 100644 --- a/doc/concept/index.md +++ b/doc/concept/index.md @@ -36,7 +36,7 @@ watched at 100 ms by one viewer and 10 s by another. - [Transport](/concept/transport): why QUIC, and the WebTransport, WebSocket, raw QUIC, and iroh paths onto it. - [moq-lite](/concept/moq-lite): the pub/sub protocol, discovery, path patterns, subscriptions, and congestion behavior. - [Identity and recovery](/concept/identity): choosing stream instances, following restarts, and answering work on demand without mixing streams. -- [Sockets](/concept/socket): dial and accept point-to-point connections through any relay, with tracks as streams. +- [Sockets](/concept/socket): dial and accept point-to-point connections through relays, with tracks as streams. Needs opt-in moq-lite 07 on every hop. - [hang](/concept/hang): the media catalog, containers, encryption, and how to extend both. - [Archives](/concept/archives): record listed tracks into object storage and serve them back from a host. - [Audio jitter](/concept/audio-jitter): how a receiver sizes its audio playout target from arrival timing. diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 07b6030d133..2b6e87210ad 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -5,8 +5,9 @@ description: Dial and accept point-to-point connections over moq-lite, with trac # Sockets -A socket connects one dialer to one listener through any relay, the way a TCP -connection does. A listener at a path accepts the sockets that dialers open to +A socket connects one dialer to one listener through relays, the way a TCP +connection does, provided every hop runs moq-lite 07, which is opt-in (see +[Versions](#versions)). A listener at a path accepts the sockets that dialers open to it, and inside a socket either side opens and accepts tracks the way QUIC opens and accepts streams. Use it for per-client state that only the two ends need, such as a viewer's commands and the replies to them, while whatever every @@ -29,7 +30,8 @@ socket is two broadcasts beneath it: | `L/.accept/` | The listener | The listener's tracks | The id is one path segment that the dialer chooses, such as a random id or an -endpoint ID. Both segments start with `.`, so sockets are +endpoint ID. Run one listener per path: two would both answer a dial, and the +dialer would follow only one of them. Both segments start with `.`, so sockets are [hidden](/concept/moq-lite#hidden-broadcasts) from anyone listing `L`. The listener discovers dialers with a one-segment watch of `L/.dial/*`, which a relay serves without scanning everything beneath `L`. @@ -76,7 +78,9 @@ the application's name for the stream, which may hold `/`. - **Bi.** The opener publishes the track in its own broadcast. The peer accepts it by publishing the same name in its broadcast, with the opener's track info, and each side subscribes to the other's. The opener's stream resolves - once the answer is listed. + once the answer is listed. A peer still holding its answer to an earlier open + of a name can't answer the name again, so the opener refuses to reopen a name + while the peer lists its answer. The opener prefix keeps the two sides' names apart, the way the low bit of a QUIC stream ID does: the dialer's bi `chat` and the listener's bi `chat` are @@ -97,7 +101,8 @@ than the tracks the opener keeps. A socket ends when either side's broadcast ends, which includes either side closing it or a session on the way dropping, or when a newer dial of its id replaces it. Each side then drops its socket, which retracts its broadcast and -ends the socket for the peer. +ends the socket for the peer. A side that closed opens and accepts nothing +more, and its pending bi opens fail, even when the peer answers them later. ## Versions diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 1f6e06d3fb3..cd775400bf8 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Listener::new(&origin, path)` accepts the sockets that `socket::dial(&origin, path, id)` opens, and either side of a `Socket` opens and accepts uni and bi streams as tracks or reads the peer's broadcast by fixed track names. A dial waits until accepted; drop it to give up. Needs moq-lite 07 on every hop; see [sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Listener::new(&origin, path)` accepts the sockets that `socket::dial(&origin, path, id)` opens, and either side of a `Socket` opens and accepts uni and bi streams as tracks or reads the peer's broadcast by fixed track names. A dial waits until accepted; drop it to give up. Needs moq-lite 07 (`moq-lite-07-wip`, opt-in) on every hop; see [sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 0d8706f6db4..f2c09f47b5d 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -12,7 +12,7 @@ //! The peer learns of new streams from the broadcast's track listing, which needs //! moq-lite-07 on every hop. See `doc/concept/socket.md` for the convention. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Weak}; use std::task::{Poll, ready}; @@ -147,6 +147,9 @@ struct Listing { bi: LinkedHashMap, track::Info>, /// This side's bi opens waiting for an answer, and whether the peer listed it. opening: HashMap, bool>, + /// The peer's answers to this side's bi opens that it still lists, so a name is not + /// opened again while the peer holds the answer to an earlier open of it. + answers: HashSet>, /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. end: Option>, } @@ -215,8 +218,11 @@ impl Listing { if let Some(answered) = self.opening.get_mut(stream) { *answered = true; } + self.answers.insert(stream.into()); + } + (true, Kind::Bi, None) => { + self.answers.remove(stream); } - (true, Kind::Bi, None) => {} // This side's uni streams live only in its own broadcast. (true, Kind::Uni, _) => {} } @@ -251,6 +257,22 @@ impl Inner { self.closed.lock().get_or_insert(reason); self.local.close(); } + + /// Whether this side's broadcast ended, registering `waiter` for when it does. + fn poll_local_closed(&self, waiter: &kio::Waiter) -> bool { + self.local.poll_closed(waiter).is_ready() + } + + /// Create a track in this side's broadcast, refusing with [`Error::Closed`] once it + /// ended: a track in an ended broadcast reaches nobody. Checked after the create, so a + /// close racing it is either seen here or comes after the stream opened. + fn create_track(&self, name: String, info: impl Into>) -> Result { + let producer = self.local.create_track(name, info)?; + if self.poll_local_closed(&kio::Waiter::noop()) { + return Err(Error::Closed); + } + Ok(producer) + } } /// One end of a connection between a dialer and a listener. @@ -293,32 +315,33 @@ impl Socket { /// Open a uni stream named `name`: a track only this side writes. /// /// Fails with [`Error::Duplicate`] when this side already opened a uni stream of - /// that name. + /// that name, and with [`Error::Closed`] once this side closed. pub fn open_uni(&self, name: &str, info: impl Into>) -> Result { self.inner - .local .create_track(track_name(self.inner.role, Kind::Uni, name), info) } /// Open a bi stream named `name`, resolving once the peer accepts it. /// /// `info` describes this side's track, and the peer's answer mirrors it. Fails with - /// [`Error::Duplicate`] when this side already opened a bi stream of that name. + /// [`Error::Duplicate`] when this side already opened a bi stream of that name, or + /// the peer still lists its answer to an earlier one, and with [`Error::Closed`] + /// once this side closed. pub fn open_bi(&self, name: &str, info: impl Into>) -> Result, Error> { + let inner = &self.inner; let name: Arc = name.into(); // Registered first, so a poll that reads the peer's answer before this returns keeps it. { - let mut listing = self.inner.listing.lock(); - if listing.opening.contains_key(&name) { + let mut listing = inner.listing.lock(); + // Caught up first: the peer would skip a name it still answers, leaving this + // open unanswered. Parks nowhere, so no poller loses its wakeup. + listing.drain(&kio::Waiter::noop(), inner.role, &inner.remote); + if listing.opening.contains_key(&name) || listing.answers.contains(&name) { return Err(Error::Duplicate); } listing.opening.insert(name.clone(), false); } - let producer = match self - .inner - .local - .create_track(track_name(self.inner.role, Kind::Bi, &name), info) - { + let producer = match inner.create_track(track_name(inner.role, Kind::Bi, &name), info) { Ok(producer) => producer, Err(err) => { self.inner.listing.lock().opening.remove(&name); @@ -332,28 +355,34 @@ impl Socket { })) } - /// Poll for the next uni stream the peer opened; `None` once the peer's broadcast ended. + /// Poll for the next uni stream the peer opened; `None` once the peer's broadcast ended, + /// and [`Error::Closed`] once this side closed. /// /// A stream is handed out only while the peer still lists it: one whose track the peer /// dropped first, finished or not, is skipped. pub fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll, Error>> { let inner = &self.inner; - let mut listing = inner.listing.lock(); - listing.drain(waiter, inner.role, &inner.remote); - if let Some((name, ())) = listing.uni.pop_front() { + loop { + if inner.poll_local_closed(waiter) { + return Poll::Ready(Err(Error::Closed)); + } + let mut listing = inner.listing.lock(); + listing.drain(waiter, inner.role, &inner.remote); + let Some((name, ())) = listing.uni.pop_front() else { + return match listing.ended() { + Some(end) => Poll::Ready(end), + None => Poll::Pending, + }; + }; drop(listing); - return Poll::Ready( - match inner.remote.track(&track_name(inner.role.peer(), Kind::Uni, &name)) { - Ok(consumer) => Ok(Some(Uni { name, consumer })), - // The peer's broadcast ended. - Err(Error::Unroutable) => Ok(None), - Err(err) => Err(err), - }, - ); - } - match listing.ended() { - Some(end) => Poll::Ready(end), - None => Poll::Pending, + return match inner.remote.track(&track_name(inner.role.peer(), Kind::Uni, &name)) { + Ok(consumer) => Poll::Ready(Ok(Some(Uni { name, consumer }))), + // The peer's broadcast ended. + Err(Error::Unroutable) => Poll::Ready(Ok(None)), + // The peer dropped the track since the listing read it. + Err(Error::NotFound) => continue, + Err(err) => Poll::Ready(Err(err)), + }; } } @@ -364,10 +393,14 @@ impl Socket { } /// Poll for the next bi stream the peer opened, answering it with a track that mirrors - /// the opener's info; `None` once the peer's broadcast ended. + /// the opener's info; `None` once the peer's broadcast ended, and [`Error::Closed`] + /// once this side closed. Skips streams as [`Self::poll_accept_uni`] does. pub fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll, Error>> { let inner = &self.inner; loop { + if inner.poll_local_closed(waiter) { + return Poll::Ready(Err(Error::Closed)); + } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); let Some((name, info)) = listing.bi.pop_front() else { @@ -382,6 +415,8 @@ impl Socket { Ok(consumer) => consumer, // The peer's broadcast ended. Err(Error::Unroutable) => return Poll::Ready(Ok(None)), + // The peer dropped the track since the listing read it. + Err(Error::NotFound) => continue, Err(err) => return Poll::Ready(Err(err)), }; // The opener's track info, minus how its copy reached this hop. @@ -390,9 +425,10 @@ impl Socket { unchecked: false, ..info }; - let producer = match inner.local.create_track(track, info) { + let producer = match inner.create_track(track, info) { Ok(producer) => producer, - // The peer listed a stream again that this side already answered. + // The peer listed a stream again while this side still holds its answer to + // the earlier one; the opener refuses that open itself once it sees the answer. Err(Error::Duplicate) => continue, Err(err) => return Poll::Ready(Err(err)), }; @@ -453,9 +489,15 @@ pub struct OpenBi { impl OpenBi { /// Poll for the peer's answer. Fails with [`Error::Unroutable`] once the peer's - /// broadcast ends without one. + /// broadcast ends without one, and with [`Error::Closed`] once this side closed, even + /// if the answer came. pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let inner = &self.inner; + // This side's track went with its broadcast, and a peer that keeps its own up + // without answering would otherwise hold this open forever. + if inner.poll_local_closed(waiter) { + return Poll::Ready(Err(Error::Closed)); + } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); if listing.opening.get(&self.name) == Some(&true) { @@ -564,6 +606,9 @@ struct Accepting { /// and a dialer's broadcast is requested only then: a listener that stops accepting /// holds nothing for the dialers waiting on it. While it accepts, a dialer dialing again /// with the same id ends the socket it replaces with [`Closed::Replaced`]. +/// +/// Run one listener per path. Two would both answer a dial under its instance, and the +/// dialer would follow only one of them, leaving the other's socket unread. pub struct Listener { origin: origin::Producer, path: Path, @@ -604,8 +649,11 @@ impl Listener { /// Poll for the next dialer, requesting its broadcast and answering it. /// - /// A dialer that leaves before its broadcast resolves is skipped. Fails with - /// [`Error::Closed`] once the origin closes. + /// Dialers are taken one at a time, in order. A session resolves a request for a + /// broadcast its peer announced itself, without asking the peer, so a remote dialer + /// cannot hold up the ones behind it; only an [`origin::Requests`] handler on this + /// origin that holds a dial's request can. A dialer that leaves before its broadcast + /// resolves is skipped. Fails with [`Error::Closed`] once the origin closes. pub fn poll_accept(&mut self, waiter: &kio::Waiter) -> Poll> { loop { if let Some(accepting) = &mut self.accepting { diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index edf495aba57..73f65460bd1 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -141,6 +141,11 @@ async fn streams_flow_both_ways() { 3, "the answer mirrors the opener's info" ); + let info = kio::clock::timeout(TIMEOUT, opened.consumer.info()) + .await + .expect("info timeout") + .expect("the answer's info"); + assert_eq!(info.priority, 3, "the opener reads the mirrored info"); send(&opened.producer, b"ping"); assert_eq!(recv(&answered.consumer).await, b"ping"); @@ -349,3 +354,86 @@ async fn a_pending_open_neither_doubles_nor_outlives_its_socket() { ); drop(pending); } + +/// Closing a socket ends its pending bi opens, both one the peer never answers and one +/// answered too late, and refuses new streams. +#[kio::test] +async fn closing_a_socket_ends_its_opens() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + let late = dialer.open_bi("late", None).unwrap(); + let stalled = dialer.open_bi("stalled", None).unwrap(); + // The peer answers the first and never the second, and keeps its broadcast up: the + // listener is not polled again, so nothing ends its socket. + let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(&*answered.name, "late"); + // Let the dialer's listing hear the answer before the close. + assert!(kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err()); + + dialer.close(); + assert!( + matches!( + kio::clock::timeout(TIMEOUT, late).await.expect("late open"), + Err(moq_net::Error::Closed) + ), + "an answer after the close was taken" + ); + assert!( + matches!( + kio::clock::timeout(TIMEOUT, stalled).await.expect("stalled open"), + Err(moq_net::Error::Closed) + ), + "a stalled open outlived the close" + ); + + assert!(matches!(dialer.open_uni("after", None), Err(moq_net::Error::Closed))); + assert!(matches!(dialer.open_bi("after", None), Err(moq_net::Error::Closed))); + assert!(matches!( + kio::clock::timeout(TIMEOUT, dialer.accept_uni()).await.expect("accept"), + Err(moq_net::Error::Closed) + )); + drop(answered); +} + +/// A bi stream's name opens again only once the peer released its answer to the earlier +/// open, which the peer could not answer before then. +#[kio::test] +async fn a_bi_name_reopens_once_its_answer_is_released() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + let first = dialer.open_bi("chat", None).unwrap(); + let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) + .await + .expect("bi timeout"); + let answered = answered.expect("accept").expect("a stream"); + drop(opened.expect("open")); + + // The peer still holds its answer. + assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); + assert!( + matches!(dialer.open_bi("chat", None), Err(moq_net::Error::Duplicate)), + "reopened a name the peer still answers" + ); + + drop(answered); + // Let the dialer's listing hear the answer go. + assert!(kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err()); + let again = dialer.open_bi("chat", None).unwrap(); + let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(again, accepted.accept_bi()) }) + .await + .expect("reopen timeout"); + let opened = opened.expect("open"); + let answered = answered.expect("accept").expect("a stream"); + send(&opened.producer, b"ping"); + assert_eq!(recv(&answered.consumer).await, b"ping"); + send(&answered.producer, b"pong"); + assert_eq!(recv(&opened.consumer).await, b"pong"); +} From 775653cda8235a57a76e3233ea37e4dc1bf4f298 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:29:33 -0800 Subject: [PATCH 08/25] fix(net): number socket streams so a track carries one stream Stream tracks are now `.///`, where the number counts the opener's streams and is never reused within a socket, like a QUIC stream ID. Before, reopening a name reused its track, so a lazy handle the peer kept to the earlier stream resolved to the new one, breaking the rule that a track name means one content. Each open is now its own stream, so opening a name twice is no longer `Duplicate`, and a bi reopen is answered while the peer still holds its earlier answer. That replaces the answer-listing guard from the previous commit. Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 26 +++-- quest/m1/socket/README.md | 6 +- rs/moq-net/src/model/socket.rs | 142 +++++++++++++------------ rs/moq-net/tests/integration/socket.rs | 84 ++++++++++----- 4 files changed, 149 insertions(+), 109 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 2b6e87210ad..13bff661ad3 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -64,27 +64,31 @@ announcements without limit, and its broadcast is requested only on accept. ## Streams -A stream is a track, named by who opened it and which way it carries data: +A stream is a track, named by who opened it, which way it carries data, and +its number: ```text -.// +./// ``` -`` is `dial` or `accept`, `` is `uni` or `bi`, and `` is -the application's name for the stream, which may hold `/`. +`` is `dial` or `accept`, and `` is `uni` or `bi`. `` +counts the opener's streams in decimal from 0 with no leading zeros, uni and bi +alike, and is never reused within a socket. `` is the application's name +for the stream, which may hold `/` and may repeat: each open is a new stream. - **Uni.** The opener publishes the track in its own broadcast. The peer accepts it and subscribes. - **Bi.** The opener publishes the track in its own broadcast. The peer accepts it by publishing the same name in its broadcast, with the opener's track info, and each side subscribes to the other's. The opener's stream resolves - once the answer is listed. A peer still holding its answer to an earlier open - of a name can't answer the name again, so the opener refuses to reopen a name - while the peer lists its answer. - -The opener prefix keeps the two sides' names apart, the way the low bit of a -QUIC stream ID does: the dialer's bi `chat` and the listener's bi `chat` are -two streams. A track's info, priority included, is the opener's to choose. + once the answer is listed. + +The opener prefix and the number together work like a QUIC stream ID. The +prefix keeps the two sides' names apart, the way the ID's low bit does, so the +dialer's bi `chat` and the listener's bi `chat` are two streams. The number +keeps one side's opens apart, so a track name always means one stream: a +handle the peer kept to an earlier `chat` never reads a later one. A track's +info, priority included, is the opener's to choose. The peer learns of new streams from the broadcast's track listing (the Tracks stream, `TRACKS_ADD`). A listing reports every track the broadcast holds, so diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index 80b7098835e..e260f2e9a6c 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -63,8 +63,10 @@ replacement arc): `FINISH`), which pushes them like QUIC's accept. `requested_track` was declined because it is pull-shaped: only the side being subscribed to can accept. Settled by the Rust implementation: tracks are named - `.//` with `` `dial` or `accept`, so the two - sides' names never collide; accept skips names outside the scheme, so fixed + `.///` with `` `dial` or `accept`, so + the two sides' names never collide, and `` counting the opener's + streams, so a track never carries two streams and a name may be reopened; + accept skips names outside the scheme, so fixed track names coexist; the opener chooses the track info (priority included) and a bi answer mirrors it. [`doc/concept/socket.md`](/doc/concept/socket.md) is the convention JS and Python follow. diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index f2c09f47b5d..ff69a102846 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -9,10 +9,12 @@ //! //! Inside a socket either side opens tracks the way QUIC opens streams. A uni stream is //! one track in the opener's broadcast, and a bi stream is the same track name in both. +//! Each open takes the next number of its opener, so a track never carries two streams. //! The peer learns of new streams from the broadcast's track listing, which needs //! moq-lite-07 on every hop. See `doc/concept/socket.md` for the convention. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::task::{Poll, ready}; @@ -44,13 +46,6 @@ impl Role { Self::Listener => "accept", } } - - fn peer(self) -> Self { - match self { - Self::Dialer => Self::Listener, - Self::Listener => Self::Dialer, - } - } } /// Whether a stream carries data one way or both. @@ -69,16 +64,22 @@ impl Kind { } } -/// The track name of `name`, opened by `opener` as a `kind` stream: `.//`. -fn track_name(opener: Role, kind: Kind, name: &str) -> String { - format!(".{}/{}/{name}", opener.tag(), kind.tag()) +/// The track name of stream `name`, the opener's `number`th, opened by `opener` as a +/// `kind` stream: `.///`. +fn track_name(opener: Role, kind: Kind, number: u64, name: &str) -> String { + format!(".{}/{}/{number}/{name}", opener.tag(), kind.tag()) } /// Split a track name into its opener, kind, and stream name, or `None` for a track /// outside the scheme, such as one the application names itself. fn parse(track: &str) -> Option<(Role, Kind, &str)> { let (opener, rest) = track.strip_prefix('.')?.split_once('/')?; - let (kind, name) = rest.split_once('/')?; + let (kind, rest) = rest.split_once('/')?; + let (number, name) = rest.split_once('/')?; + // Decimal, as the opener writes it, so one stream has one name. + if number.parse::().ok()?.to_string() != number { + return None; + } let opener = match opener { "dial" => Role::Dialer, "accept" => Role::Listener, @@ -139,17 +140,15 @@ pub struct Bi { struct Listing { /// The peer's listing, opened by the first accept or bi open. tracks: Option, - /// Uni streams the peer opened, not yet accepted, in order. A stream leaves once the - /// peer stops listing it, as the listing drops an add its reader never reached, so - /// these hold no more than the peer lists. - uni: LinkedHashMap, ()>, + /// Uni streams the peer opened, not yet accepted, in order, by track name with their + /// stream name. A stream leaves once the peer stops listing it, as the listing drops + /// an add its reader never reached, so these hold no more than the peer lists. + uni: LinkedHashMap, Arc>, /// Bi streams the peer opened, with the opener's track info, not yet accepted. - bi: LinkedHashMap, track::Info>, - /// This side's bi opens waiting for an answer, and whether the peer listed it. + bi: LinkedHashMap, (Arc, track::Info)>, + /// This side's bi opens waiting for an answer, by track name, and whether the peer + /// listed it. opening: HashMap, bool>, - /// The peer's answers to this side's bi opens that it still lists, so a name is not - /// opened again while the peer holds the answer to an earlier open of it. - answers: HashSet>, /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. end: Option>, } @@ -200,29 +199,27 @@ impl Listing { let Some((opener, kind, stream)) = parse(&name) else { continue; }; + let stream: Arc = stream.into(); match (opener == role, kind, info) { (false, Kind::Uni, Some(_)) => { - self.uni.replace(stream.into(), ()); + self.uni.replace(name, stream); } (false, Kind::Uni, None) => { - self.uni.remove(stream); + self.uni.remove(&name); } (false, Kind::Bi, Some(info)) => { - self.bi.replace(stream.into(), info); + self.bi.replace(name, (stream, info)); } (false, Kind::Bi, None) => { - self.bi.remove(stream); + self.bi.remove(&name); } // An answer seen once stays seen: the opener reads whatever it wrote. (true, Kind::Bi, Some(_)) => { - if let Some(answered) = self.opening.get_mut(stream) { + if let Some(answered) = self.opening.get_mut(&name) { *answered = true; } - self.answers.insert(stream.into()); - } - (true, Kind::Bi, None) => { - self.answers.remove(stream); } + (true, Kind::Bi, None) => {} // This side's uni streams live only in its own broadcast. (true, Kind::Uni, _) => {} } @@ -249,6 +246,8 @@ struct Inner { listing: kio::Lock, /// Why the listener ended the socket, set before it closes [`Self::local`]. closed: kio::Lock>, + /// The number of this side's next stream. + next: AtomicU64, } impl Inner { @@ -263,6 +262,11 @@ impl Inner { self.local.poll_closed(waiter).is_ready() } + /// The track name of this side's next `kind` stream, named `name`. + fn open(&self, kind: Kind, name: &str) -> String { + track_name(self.role, kind, self.next.fetch_add(1, Ordering::Relaxed), name) + } + /// Create a track in this side's broadcast, refusing with [`Error::Closed`] once it /// ended: a track in an ended broadcast reaches nobody. Checked after the create, so a /// close racing it is either seen here or comes after the stream opened. @@ -293,6 +297,7 @@ impl Socket { remote, listing: kio::Lock::new(Listing::default()), closed: kio::Lock::new(None), + next: AtomicU64::new(0), }), } } @@ -314,43 +319,33 @@ impl Socket { /// Open a uni stream named `name`: a track only this side writes. /// - /// Fails with [`Error::Duplicate`] when this side already opened a uni stream of - /// that name, and with [`Error::Closed`] once this side closed. + /// Each open is a new stream, even under a name opened before. Fails with + /// [`Error::Closed`] once this side closed. pub fn open_uni(&self, name: &str, info: impl Into>) -> Result { - self.inner - .create_track(track_name(self.inner.role, Kind::Uni, name), info) + self.inner.create_track(self.inner.open(Kind::Uni, name), info) } /// Open a bi stream named `name`, resolving once the peer accepts it. /// - /// `info` describes this side's track, and the peer's answer mirrors it. Fails with - /// [`Error::Duplicate`] when this side already opened a bi stream of that name, or - /// the peer still lists its answer to an earlier one, and with [`Error::Closed`] - /// once this side closed. + /// `info` describes this side's track, and the peer's answer mirrors it. Each open is + /// a new stream, even under a name opened before. Fails with [`Error::Closed`] once + /// this side closed. pub fn open_bi(&self, name: &str, info: impl Into>) -> Result, Error> { let inner = &self.inner; - let name: Arc = name.into(); + let track: Arc = inner.open(Kind::Bi, name).into(); // Registered first, so a poll that reads the peer's answer before this returns keeps it. - { - let mut listing = inner.listing.lock(); - // Caught up first: the peer would skip a name it still answers, leaving this - // open unanswered. Parks nowhere, so no poller loses its wakeup. - listing.drain(&kio::Waiter::noop(), inner.role, &inner.remote); - if listing.opening.contains_key(&name) || listing.answers.contains(&name) { - return Err(Error::Duplicate); - } - listing.opening.insert(name.clone(), false); - } - let producer = match inner.create_track(track_name(inner.role, Kind::Bi, &name), info) { + inner.listing.lock().opening.insert(track.clone(), false); + let producer = match inner.create_track(track.to_string(), info) { Ok(producer) => producer, Err(err) => { - self.inner.listing.lock().opening.remove(&name); + inner.listing.lock().opening.remove(&track); return Err(err); } }; Ok(kio::Pending::new(OpenBi { - inner: self.inner.clone(), - name, + inner: inner.clone(), + name: name.into(), + track, producer: Some(producer), })) } @@ -368,14 +363,14 @@ impl Socket { } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - let Some((name, ())) = listing.uni.pop_front() else { + let Some((track, name)) = listing.uni.pop_front() else { return match listing.ended() { Some(end) => Poll::Ready(end), None => Poll::Pending, }; }; drop(listing); - return match inner.remote.track(&track_name(inner.role.peer(), Kind::Uni, &name)) { + return match inner.remote.track(&track) { Ok(consumer) => Poll::Ready(Ok(Some(Uni { name, consumer }))), // The peer's broadcast ended. Err(Error::Unroutable) => Poll::Ready(Ok(None)), @@ -403,14 +398,13 @@ impl Socket { } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - let Some((name, info)) = listing.bi.pop_front() else { + let Some((track, (name, info))) = listing.bi.pop_front() else { return match listing.ended() { Some(end) => Poll::Ready(end), None => Poll::Pending, }; }; drop(listing); - let track = track_name(inner.role.peer(), Kind::Bi, &name); let consumer = match inner.remote.track(&track) { Ok(consumer) => consumer, // The peer's broadcast ended. @@ -425,10 +419,11 @@ impl Socket { unchecked: false, ..info }; - let producer = match inner.create_track(track, info) { + // Answered under the opener's track name. + let producer = match inner.create_track(track.to_string(), info) { Ok(producer) => producer, - // The peer listed a stream again while this side still holds its answer to - // the earlier one; the opener refuses that open itself once it sees the answer. + // The peer listed a stream again that this side already answered, which it + // does only by opening a number twice. Err(Error::Duplicate) => continue, Err(err) => return Poll::Ready(Err(err)), }; @@ -483,6 +478,8 @@ impl Drop for Socket { pub struct OpenBi { inner: Arc, name: Arc, + /// The stream's track name, which the answer shares. + track: Arc, /// This side's track, handed over once the peer answers. producer: Option, } @@ -500,10 +497,10 @@ impl OpenBi { } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - if listing.opening.get(&self.name) == Some(&true) { - listing.opening.remove(&self.name); + if listing.opening.get(&self.track) == Some(&true) { + listing.opening.remove(&self.track); drop(listing); - let consumer = inner.remote.track(&track_name(inner.role, Kind::Bi, &self.name))?; + let consumer = inner.remote.track(&self.track)?; let producer = self.producer.take().ok_or(Error::Closed)?; return Poll::Ready(Ok(Bi { name: self.name.clone(), @@ -530,7 +527,7 @@ impl kio::Task for OpenBi { impl Drop for OpenBi { fn drop(&mut self) { if self.producer.is_some() { - self.inner.listing.lock().opening.remove(&self.name); + self.inner.listing.lock().opening.remove(&self.track); } } } @@ -758,12 +755,12 @@ mod tests { fn track_names_round_trip() { for opener in [Role::Dialer, Role::Listener] { for kind in [Kind::Uni, Kind::Bi] { - let name = track_name(opener, kind, "chat/room"); + let name = track_name(opener, kind, 7, "chat/room"); assert_eq!(parse(&name), Some((opener, kind, "chat/room"))); } } - assert_eq!(track_name(Role::Dialer, Kind::Bi, "x"), ".dial/bi/x"); - assert_eq!(track_name(Role::Listener, Kind::Uni, "x"), ".accept/uni/x"); + assert_eq!(track_name(Role::Dialer, Kind::Bi, 0, "x"), ".dial/bi/0/x"); + assert_eq!(track_name(Role::Listener, Kind::Uni, 12, "x"), ".accept/uni/12/x"); } #[test] @@ -772,9 +769,14 @@ mod tests { "command", ".dial", ".dial/uni", - ".dial/other/x", - ".other/uni/x", - "dial/uni/x", + ".dial/other/0/x", + ".other/uni/0/x", + "dial/uni/0/x", + ".dial/uni/x", + ".dial/uni/0", + ".dial/uni/07/x", + ".dial/uni/-1/x", + ".dial/uni/+1/x", ] { assert_eq!(parse(name), None, "{name}"); } diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 73f65460bd1..eb1ef7d67d2 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -328,21 +328,35 @@ async fn a_stream_is_accepted_only_while_listed() { drop(live); } -/// Opening a bi stream of a name still being opened fails without disturbing the first, -/// and dropping the socket ends it for the peer even while an open is pending. +/// Two bi opens of one name are two streams, each answered on its own, and dropping the +/// socket ends it for the peer even while an open is pending. #[kio::test] -async fn a_pending_open_neither_doubles_nor_outlives_its_socket() { +async fn a_pending_open_neither_merges_nor_outlives_its_socket() { let fixture = Fixture::new().await; let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; let first = dialer.open_bi("chat", None).unwrap(); - assert!(matches!(dialer.open_bi("chat", None), Err(moq_net::Error::Duplicate))); - let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) - .await - .expect("bi timeout"); - opened.expect("open"); - answered.expect("accept").expect("a stream"); + let second = dialer.open_bi("chat", None).unwrap(); + let (first, second, answers) = kio::clock::timeout(TIMEOUT, async { + futures::join!(first, second, async { + (accepted.accept_bi().await, accepted.accept_bi().await) + }) + }) + .await + .expect("bi timeout"); + let (first, second) = (first.expect("first open"), second.expect("second open")); + let one = answers.0.expect("accept").expect("a stream"); + let two = answers.1.expect("accept").expect("a stream"); + assert_eq!((&*one.name, &*two.name), ("chat", "chat")); + send(&first.producer, b"one"); + send(&second.producer, b"two"); + assert_eq!(recv(&one.consumer).await, b"one"); + assert_eq!(recv(&two.consumer).await, b"two"); + send(&one.producer, b"one back"); + send(&two.producer, b"two back"); + assert_eq!(recv(&first.consumer).await, b"one back"); + assert_eq!(recv(&second.consumer).await, b"two back"); let pending = dialer.open_bi("unanswered", None).unwrap(); drop(dialer); @@ -401,32 +415,50 @@ async fn closing_a_socket_ends_its_opens() { drop(answered); } -/// A bi stream's name opens again only once the peer released its answer to the earlier -/// open, which the peer could not answer before then. +/// A name opened again is a new stream: a handle to the earlier one never reads it, and +/// a bi reopen is answered while the peer still holds its earlier answer. #[kio::test] -async fn a_bi_name_reopens_once_its_answer_is_released() { +async fn a_reopened_name_is_a_new_stream() { let fixture = Fixture::new().await; let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - let first = dialer.open_bi("chat", None).unwrap(); - let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) + // Accepted and held without subscribing, then the opener moves on to a new "chat". + let old = dialer.open_uni("chat", None).unwrap(); + let held = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) .await - .expect("bi timeout"); - let answered = answered.expect("accept").expect("a stream"); - drop(opened.expect("open")); - - // The peer still holds its answer. - assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); + .expect("accept timeout") + .unwrap() + .expect("a stream"); + drop(old); + let new = dialer.open_uni("chat", None).unwrap(); + send(&new, b"new"); + let fresh = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(&*fresh.name, "chat"); + assert_eq!(recv(&fresh.consumer).await, b"new"); + let stale = kio::clock::timeout(SETTLE, async { + let mut subscriber = held.consumer.subscribe(None).await.ok()?; + let mut group = subscriber.recv_group().await.ok()??; + Some(group.read_frame().await.ok()??.payload.to_vec()) + }) + .await; assert!( - matches!(dialer.open_bi("chat", None), Err(moq_net::Error::Duplicate)), - "reopened a name the peer still answers" + !matches!(stale, Ok(Some(payload)) if payload == b"new"), + "the earlier stream's handle read the new one" ); - drop(answered); - // Let the dialer's listing hear the answer go. - assert!(kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err()); - let again = dialer.open_bi("chat", None).unwrap(); + // A bi answer the peer keeps does not stop the name opening again. + let first = dialer.open_bi("call", None).unwrap(); + let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) + .await + .expect("bi timeout"); + let _kept = answered.expect("accept").expect("a stream"); + drop(opened.expect("open")); + let again = dialer.open_bi("call", None).unwrap(); let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(again, accepted.accept_bi()) }) .await .expect("reopen timeout"); From 0e0e62fec2bccc3bbec606f739ba60514822abd0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:53:37 -0800 Subject: [PATCH 09/25] feat(net)!: unnamed socket streams, with a token-auth example and relay test Socket streams copy QUIC's API: no application name. Tracks are `.//`, and `open_uni`, `open_bi`, `accept_uni`, and `accept_bi` take only the track info. `accept_uni` returns the peer's track consumer, and `Bi` drops its name. Well-known channels are fixed-name tracks on the socket's broadcasts. doc/concept/socket.md gains an Authorization section with prefix-only tokens rooted at the listener's path, and moq-relay gains a test that admits a dial and accept with real moq-auth tokens and refuses another id's dial and answer. Decided 2026-11-03 (maintainer, /quest-complete). Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 59 ++++-- doc/lib/rs/moq-net.md | 2 +- quest/m1/socket/README.md | 48 +++-- quest/m1/socket/boy.md | 9 +- quest/m1/socket/carla.md | 6 +- quest/m1/socket/js.md | 5 +- quest/m1/socket/py.md | 3 +- rs/moq-net/CHANGELOG.md | 2 +- rs/moq-net/src/model/socket.rs | 143 +++++++------- rs/moq-net/tests/integration/socket.rs | 106 +++++++---- rs/moq-relay/tests/integration/main.rs | 2 + rs/moq-relay/tests/integration/socket_auth.rs | 178 ++++++++++++++++++ 12 files changed, 409 insertions(+), 154 deletions(-) create mode 100644 rs/moq-relay/tests/integration/socket_auth.rs diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 13bff661ad3..9de7c064415 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -36,10 +36,36 @@ dialer would follow only one of them. Both segments start with `.`, so sockets a listener discovers dialers with a one-segment watch of `L/.dial/*`, which a relay serves without scanning everything beneath `L`. -Authorization uses ordinary path grants. A dialer needs publish on -`L/.dial/` and subscribe on `L/.accept/`, and the listener the reverse -for every id: subscribe on `L/.dial/*` and publish on `L/.accept/*`. A grant -naming the id is what stops one dialer from taking another's socket. +## Authorization + +Sockets need nothing beyond ordinary +[token grants](/bin/relay/auth#path-matching). With the token's root at the +listener's path `L`, each grant is a prefix: + +```jsonc +// The listener: reads every dial and answers every id. +{ "root": "L", "subscribe": [".dial/**"], "publish": [".accept/**"] } + +// The dialer with id : its own dial and its own answer. +{ "root": "L", "publish": [".dial/"], "subscribe": [".accept/"] } +``` + +The dialer's grant names its id, which is what stops one dialer from dialing as +another, and so replacing its socket, or reading another's answer. A pattern +matches `.`-prefixed segments like any other, so `**` would cover sockets too; +hiding narrows only discovery. No mid-path wildcard is needed, because the +root carries `L`. It's the same shape as a request/response pair, such as +`request/` and `response/`. + +`moq auth sign` writes these as, for example: + +```bash +moq auth sign --key key.jwk --root L --subscribe '.dial/**' --publish '.accept/**' > listener.jwt +moq auth sign --key key.jwk --root L --publish ".dial/$ID" --subscribe ".accept/$ID" > dialer.jwt +``` + +Each client then connects to the relay at `L` with its token, and its socket's +path is the session root. ## Instances @@ -64,17 +90,16 @@ announcements without limit, and its broadcast is requested only on accept. ## Streams -A stream is a track, named by who opened it, which way it carries data, and -its number: +A stream is a track. Like a QUIC stream it has no name of its own: it is +named by who opened it, which way it carries data, and its number. ```text -./// +.// ``` `` is `dial` or `accept`, and `` is `uni` or `bi`. `` counts the opener's streams in decimal from 0 with no leading zeros, uni and bi -alike, and is never reused within a socket. `` is the application's name -for the stream, which may hold `/` and may repeat: each open is a new stream. +alike, and is never reused within a socket, so each open is a new track. - **Uni.** The opener publishes the track in its own broadcast. The peer accepts it and subscribes. @@ -84,16 +109,22 @@ for the stream, which may hold `/` and may repeat: each open is a new stream. once the answer is listed. The opener prefix and the number together work like a QUIC stream ID. The -prefix keeps the two sides' names apart, the way the ID's low bit does, so the -dialer's bi `chat` and the listener's bi `chat` are two streams. The number +prefix keeps the two sides' streams apart, the way the ID's low bit does, so +the dialer's first bi stream and the listener's are two streams. The number keeps one side's opens apart, so a track name always means one stream: a -handle the peer kept to an earlier `chat` never reads a later one. A track's +handle the peer kept to an earlier stream never reads a later one. A track's info, priority included, is the opener's to choose. The peer learns of new streams from the broadcast's track listing (the Tracks stream, `TRACKS_ADD`). A listing reports every track the broadcast holds, so -accept skips any name outside the scheme. That leaves applications free to use -fixed track names in either broadcast beside the streams. +accept skips any name outside the scheme. + +A channel both sides know in advance, such as a viewer's `command` track or the +listener's acks back to it, isn't a stream. It's an ordinary track with a fixed +name in the socket's broadcast: the writer creates it in its own half, and the +reader subscribes to it by name in the peer's half, with no accept. Streams are +for what an application opens at run time, such as one per request or per +byte-stream connection, with any framing of its own. A stream can be accepted only while its opener lists it, which holds until the opener drops the track, finished or not. One dropped before the peer accepts it diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index cd775400bf8..1a1fc08693e 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Listener::new(&origin, path)` accepts the sockets that `socket::dial(&origin, path, id)` opens, and either side of a `Socket` opens and accepts uni and bi streams as tracks or reads the peer's broadcast by fixed track names. A dial waits until accepted; drop it to give up. Needs moq-lite 07 (`moq-lite-07-wip`, opt-in) on every hop; see [sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Listener::new(&origin, path)` accepts the sockets that `socket::dial(&origin, path, id)` opens, and either side of a `Socket` opens and accepts unnamed uni and bi streams as tracks, the way QUIC does (`open_uni`, `open_bi`, `accept_uni`, `accept_bi`), or reads well-known channels from the peer's broadcast by fixed track name. A dial waits until accepted; drop it to give up. Needs moq-lite 07 (`moq-lite-07-wip`, opt-in) on every hop; see [sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index e260f2e9a6c..75c2802c972 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -4,7 +4,7 @@ `moq-net` gains a generic dial/accept primitive. A `Listener` at a path accepts the `Socket`s that dialers announce. Inside a socket either side opens and -accepts named tracks the way QUIC opens and accepts streams. moq-boy, CARLA, +accepts unnamed streams, as tracks, the way QUIC does. moq-boy, CARLA, [byte streams](/quest/m2/stream/README.md), and the mesh [Endpoint](/quest/m2/mesh/endpoint.md) all build on it instead of each hand-rolling the announce, watch, and request loop. Rust, JS, and Python @@ -18,6 +18,25 @@ one-driver lease and 1 s watchdog); announcement backpressure, which ## Plan +Decided 2026-11-03 (maintainer, /quest-complete), on +[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677): + +- Copy QUIC's API: socket streams have no application name. Tracks are + `.//`, with `` `dial` or `accept`, `` + `uni` or `bi`, and `` the opener's never-reused counter. The surface + is quinn's: `open_uni`, `open_bi`, `accept_uni`, `accept_bi`, each taking + the track info an open needs. Why: no consumer needs the name. Well-known + channels (boy's `command`; CARLA's `control`, ack, and latency) are ordinary + fixed-name tracks on the socket's broadcasts, and Kixel T3 runs byte + streams with its own framing. Not chosen: a hidden number plus an app name, + or making name uniqueness the app's job. +- Auth uses moq.pro-style tokens with prefix grants only, rooted at the + listener's path `L`: the listener subscribes `.dial/**` and publishes + `.accept/**`, and a dialer publishes `.dial/` and subscribes + `.accept/`. No mid-path wildcard is needed; it's the shape of moq.pro's + voice demo (`request/`, `response/`). A relay test with real + `moq-auth` tokens covers it, and `doc/concept/socket.md` shows the tokens. + Decided 2026-11-02 (maintainer, Kixel planning): the whole epic moves from m2 to m1, so the next cut publishes sockets in `@moq/net` and moq.pro's relays carry them. Why: Kixel's T3 Code prototype @@ -41,10 +60,10 @@ replacement arc): is unchanged. A dialer announces `L/.dial/` and the listener answers on `L/.accept/`. Discovery is a one-segment watch of `L/.dial/*`, which is cheap for relays, unlike a globstar over the whole origin. Auth enforces the - id with a mid-path wildcard grant (`mesh/*/*/.dial/`), which - `moq-pattern` supports. The dot keeps sessions out of ordinary discovery - ([hidden broadcasts](/doc/concept/moq-lite.md#hidden-broadcasts)), so boy's game list - never sees viewers. + id with prefix grants rooted at `L` (see the 2026-11-03 decision). The dot + keeps sessions out of ordinary discovery + ([hidden broadcasts](/doc/concept/moq-lite.md#hidden-broadcasts)), so boy's + game list never sees viewers. - Identity is a dialer-chosen id plus a pinned instance (the epic first called it an epoch; the code's term is the route's `Instance`). The id is the dialer's own: an endpoint ID in the mesh, a random id in the demos. Each dial @@ -62,12 +81,11 @@ replacement arc): learns of new tracks from the Lite07 Tracks stream (`TRACKS_ADD`, `REMOVE`, `FINISH`), which pushes them like QUIC's accept. `requested_track` was declined because it is pull-shaped: only the side being subscribed to can - accept. Settled by the Rust implementation: tracks are named - `.///` with `` `dial` or `accept`, so - the two sides' names never collide, and `` counting the opener's - streams, so a track never carries two streams and a name may be reopened; - accept skips names outside the scheme, so fixed - track names coexist; the opener chooses the track info (priority included) + accept. Settled by the Rust implementation and the 2026-11-03 decision: + tracks are named `.//` with `` `dial` or + `accept`, so the two sides' streams never collide, and `` counting + the opener's streams, so a track never carries two streams; accept skips + names outside the scheme, so fixed track names coexist; the opener chooses the track info (priority included) and a bi answer mirrors it. [`doc/concept/socket.md`](/doc/concept/socket.md) is the convention JS and Python follow. - Accept is unbounded and lazy. Announced dialers queue without limit, and a @@ -80,9 +98,11 @@ replacement arc): `doc/concept/socket.md`, written with the Rust implementation. It is not a section of the moq-lite draft or a new draft: it is an application convention over moq-lite. -- The demos move per-client echoes onto the socket: boy's per-viewer latency - and CARLA's driver ack and sequence each go on that client's `.accept` - track. State every viewer needs (the game, who is driving, who holds the +- The demos move per-client channels onto the socket as fixed-name tracks on + its broadcasts, not streams: boy's `command` and CARLA's `control` on the + dialer's `.dial` broadcast, and boy's per-viewer latency and CARLA's driver + ack and sequence on the listener's `.accept` broadcast. State every viewer + needs (the game, who is driving, who holds the e-stop) stays in the shared broadcast at `L`. This epic owns removing the hand-rolled loops: after the ports, no diff --git a/quest/m1/socket/boy.md b/quest/m1/socket/boy.md index 7c63c25b43d..c375e3beab6 100644 --- a/quest/m1/socket/boy.md +++ b/quest/m1/socket/boy.md @@ -13,10 +13,13 @@ when nobody watches, and the 60 s idle stop behave as before. See the [epic](/quest/m1/socket/README.md) for the layout and the reply split. - Viewers dial `boy/game/` with their random id. Their `command` track - moves from `{viewerPrefix}/{name}/{viewerId}` to the socket. Drop the + moves from `{viewerPrefix}/{name}/{viewerId}` to the socket, as a fixed + track named `command` on the viewer's `.dial` broadcast (`Socket::local`), + which the emulator reads by name from `Socket::remote`. It isn't a socket + stream: streams are unnamed and for what opens at run time. Drop the separate viewer-prefix configuration in `rs/moq-boy/src/main.rs`. -- Per-viewer latency moves from `status`'s `latency` map to a track on that - viewer's `.accept` broadcast. Game state stays in `status`. +- Per-viewer latency moves from `status`'s `latency` map to a fixed track on + that viewer's `.accept` broadcast. Game state stays in `status`. - Requests become pinned to the viewer's epoch (boy requests unpinned today), so a reload is a new socket. - The demo relay's anonymous publish grant (`demo/relay/prod.toml`, and the diff --git a/quest/m1/socket/carla.md b/quest/m1/socket/carla.md index 085acb3b869..486cbe9d7f0 100644 --- a/quest/m1/socket/carla.md +++ b/quest/m1/socket/carla.md @@ -15,8 +15,10 @@ See the [epic](/quest/m1/socket/README.md) for the layout and the reply split. - The worker (`publish.py`) uses the Python `Listener`; `paths.ts` and `truck.ts` dial `carla/` with the per-lease driver id. A new lease is a new epoch and so a new socket, which matches today's lease binding. -- The `control` track moves onto the socket. The driver's ack, sequence, and - round-trip echo move from the shared `status` to the driver's `.accept` +- The `control` track moves onto the socket as a fixed track named `control` + on the driver's `.dial` broadcast, not a socket stream (streams are unnamed + and for what opens at run time). The driver's ack, sequence, and round-trip + echo move from the shared `status` to fixed tracks on the driver's `.accept` broadcast. `driver` and `estop_by` stay in `status`, because every viewer shows them. - Decide in the PR whether the `safety-` e-stop broadcast becomes its own diff --git a/quest/m1/socket/js.md b/quest/m1/socket/js.md index 74c03bfbd8c..ca8bf40c225 100644 --- a/quest/m1/socket/js.md +++ b/quest/m1/socket/js.md @@ -9,6 +9,9 @@ and accept uni and bi streams through a relay. ## Plan Follow `doc/concept/socket.md` and mirror the -Rust surface (`rs/moq-net/src/model/socket.rs`), including its track-naming scheme. +Rust surface (`rs/moq-net/src/model/socket.rs`), including its track-naming +scheme: streams are unnamed, like QUIC's (`openUni`, `openBi`, `acceptUni`, +`acceptBi`, in JS's casing), and well-known channels are fixed-name tracks on +the socket's `local` and `remote` broadcasts. Add the Rust/JS pair to `just test interop`, and document the export in `doc/lib/js/net.md`. diff --git a/quest/m1/socket/py.md b/quest/m1/socket/py.md index e471d6c1dac..b8706900cb2 100644 --- a/quest/m1/socket/py.md +++ b/quest/m1/socket/py.md @@ -8,7 +8,8 @@ and the reverse, with uni and bi streams each way. ## Plan -Bind the Rust surface (`moq_net::socket`) through moq-ffi the way +Bind the Rust surface (`moq_net::socket`), whose streams are unnamed like +QUIC's (`open_uni`, `open_bi`, `accept_uni`, `accept_bi`), through moq-ffi the way origin's request and announce APIs are bound today (`rs/moq-ffi/src/origin.rs`, `py/moq-rs/moq/origin.py`). Add a Python/Rust interop test, and document the API wherever `py/moq-rs` documents origin. diff --git a/rs/moq-net/CHANGELOG.md b/rs/moq-net/CHANGELOG.md index ad70164d532..7e32e7ae11e 100644 --- a/rs/moq-net/CHANGELOG.md +++ b/rs/moq-net/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `socket`: `Listener`, `dial`, and `Socket`, point-to-point connections over an origin whose streams are tracks; see `doc/concept/socket.md`. +- `socket`: `Listener`, `dial`, and `Socket`, point-to-point connections over an origin whose streams are unnamed tracks, opened and accepted like QUIC's; see `doc/concept/socket.md`. - `transport::SETUP_RECEIVE_WINDOW`, the connection credit a server offers before it admits a session: enough for the SETUP and one in-band AUTH. ## [0.3.10](https://github.com/moq-dev/moq/compare/moq-net-v0.3.9...moq-net-v0.3.10) - 2026-10-05 diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index ff69a102846..18c8e8e9b72 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -7,18 +7,21 @@ //! once the application accepts. The answer carries the dial's [`Instance`], so a //! dialer that dials again with the same id gets a new socket rather than an old answer. //! -//! Inside a socket either side opens tracks the way QUIC opens streams. A uni stream is -//! one track in the opener's broadcast, and a bi stream is the same track name in both. -//! Each open takes the next number of its opener, so a track never carries two streams. -//! The peer learns of new streams from the broadcast's track listing, which needs -//! moq-lite-07 on every hop. See `doc/concept/socket.md` for the convention. +//! Inside a socket either side opens and accepts streams the way QUIC does, and like +//! QUIC's they carry no name: a stream is told apart by who opened it and its number. A +//! uni stream is one track in the opener's broadcast, and a bi stream is the same track +//! name in both. Each open takes the next number of its opener, so a track never carries +//! two streams. The peer learns of new streams from the broadcast's track listing, which +//! needs moq-lite-07 on every hop. Channels both sides know in advance are ordinary +//! fixed-name tracks on [`Socket::local`] and [`Socket::remote`]. See +//! `doc/concept/socket.md` for the convention. use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::task::{Poll, ready}; -use hashlink::LinkedHashMap; +use hashlink::{LinkedHashMap, LinkedHashSet}; use crate::{Error, Instance, Path, Pattern, Patterns}; @@ -64,18 +67,17 @@ impl Kind { } } -/// The track name of stream `name`, the opener's `number`th, opened by `opener` as a -/// `kind` stream: `.///`. -fn track_name(opener: Role, kind: Kind, number: u64, name: &str) -> String { - format!(".{}/{}/{number}/{name}", opener.tag(), kind.tag()) +/// The track name of the opener's `number`th stream, a `kind` one: +/// `.//`. +fn track_name(opener: Role, kind: Kind, number: u64) -> String { + format!(".{}/{}/{number}", opener.tag(), kind.tag()) } -/// Split a track name into its opener, kind, and stream name, or `None` for a track -/// outside the scheme, such as one the application names itself. -fn parse(track: &str) -> Option<(Role, Kind, &str)> { +/// Split a track name into its opener and kind, or `None` for a track outside the +/// scheme, such as one the application names itself. +fn parse(track: &str) -> Option<(Role, Kind)> { let (opener, rest) = track.strip_prefix('.')?.split_once('/')?; - let (kind, rest) = rest.split_once('/')?; - let (number, name) = rest.split_once('/')?; + let (kind, number) = rest.split_once('/')?; // Decimal, as the opener writes it, so one stream has one name. if number.parse::().ok()?.to_string() != number { return None; @@ -90,7 +92,7 @@ fn parse(track: &str) -> Option<(Role, Kind, &str)> { "bi" => Kind::Bi, _ => return None, }; - Some((opener, kind, name)) + Some((opener, kind)) } /// `L//`, refusing an id that is not exactly one path segment. @@ -117,18 +119,8 @@ pub enum Closed { Replaced, } -/// A uni stream the peer opened. -pub struct Uni { - /// The name the peer opened it with. - pub name: Arc, - /// The peer's track. - pub consumer: track::Consumer, -} - -/// A bi stream: this side's track and the peer's, under one name. +/// A bi stream: this side's track and the peer's, under one track name. pub struct Bi { - /// The name the opener gave it. - pub name: Arc, /// This side's track. pub producer: track::Producer, /// The peer's track. @@ -140,12 +132,12 @@ pub struct Bi { struct Listing { /// The peer's listing, opened by the first accept or bi open. tracks: Option, - /// Uni streams the peer opened, not yet accepted, in order, by track name with their - /// stream name. A stream leaves once the peer stops listing it, as the listing drops - /// an add its reader never reached, so these hold no more than the peer lists. - uni: LinkedHashMap, Arc>, + /// Uni streams the peer opened, not yet accepted, in order, by track name. A stream + /// leaves once the peer stops listing it, as the listing drops an add its reader + /// never reached, so these hold no more than the peer lists. + uni: LinkedHashSet>, /// Bi streams the peer opened, with the opener's track info, not yet accepted. - bi: LinkedHashMap, (Arc, track::Info)>, + bi: LinkedHashMap, track::Info>, /// This side's bi opens waiting for an answer, by track name, and whether the peer /// listed it. opening: HashMap, bool>, @@ -196,19 +188,18 @@ impl Listing { TracksEvent::Synced => continue, }; // A track outside the scheme is the application's own. - let Some((opener, kind, stream)) = parse(&name) else { + let Some((opener, kind)) = parse(&name) else { continue; }; - let stream: Arc = stream.into(); match (opener == role, kind, info) { (false, Kind::Uni, Some(_)) => { - self.uni.replace(name, stream); + self.uni.replace(name); } (false, Kind::Uni, None) => { self.uni.remove(&name); } (false, Kind::Bi, Some(info)) => { - self.bi.replace(name, (stream, info)); + self.bi.replace(name, info); } (false, Kind::Bi, None) => { self.bi.remove(&name); @@ -262,9 +253,9 @@ impl Inner { self.local.poll_closed(waiter).is_ready() } - /// The track name of this side's next `kind` stream, named `name`. - fn open(&self, kind: Kind, name: &str) -> String { - track_name(self.role, kind, self.next.fetch_add(1, Ordering::Relaxed), name) + /// The track name of this side's next `kind` stream. + fn open(&self, kind: Kind) -> String { + track_name(self.role, kind, self.next.fetch_add(1, Ordering::Relaxed)) } /// Create a track in this side's broadcast, refusing with [`Error::Closed`] once it @@ -307,32 +298,33 @@ impl Socket { &self.inner.id } - /// This side's broadcast, for tracks an application names itself. + /// This side's broadcast, for well-known channels: tracks with fixed names that both + /// sides agree on, beside the streams. pub fn local(&self) -> &broadcast::Producer { &self.inner.local } - /// The peer's broadcast, for tracks an application names itself. + /// The peer's broadcast, for well-known channels: tracks with fixed names that both + /// sides agree on, beside the streams. pub fn remote(&self) -> &broadcast::Consumer { &self.inner.remote } - /// Open a uni stream named `name`: a track only this side writes. + /// Open a uni stream: a new track only this side writes, which the peer takes with + /// [`Self::accept_uni`]. /// - /// Each open is a new stream, even under a name opened before. Fails with - /// [`Error::Closed`] once this side closed. - pub fn open_uni(&self, name: &str, info: impl Into>) -> Result { - self.inner.create_track(self.inner.open(Kind::Uni, name), info) + /// `info` describes the track. Fails with [`Error::Closed`] once this side closed. + pub fn open_uni(&self, info: impl Into>) -> Result { + self.inner.create_track(self.inner.open(Kind::Uni), info) } - /// Open a bi stream named `name`, resolving once the peer accepts it. + /// Open a bi stream, resolving once the peer accepts it with [`Self::accept_bi`]. /// - /// `info` describes this side's track, and the peer's answer mirrors it. Each open is - /// a new stream, even under a name opened before. Fails with [`Error::Closed`] once - /// this side closed. - pub fn open_bi(&self, name: &str, info: impl Into>) -> Result, Error> { + /// `info` describes this side's track, and the peer's answer mirrors it. Fails with + /// [`Error::Closed`] once this side closed. + pub fn open_bi(&self, info: impl Into>) -> Result, Error> { let inner = &self.inner; - let track: Arc = inner.open(Kind::Bi, name).into(); + let track: Arc = inner.open(Kind::Bi).into(); // Registered first, so a poll that reads the peer's answer before this returns keeps it. inner.listing.lock().opening.insert(track.clone(), false); let producer = match inner.create_track(track.to_string(), info) { @@ -344,7 +336,6 @@ impl Socket { }; Ok(kio::Pending::new(OpenBi { inner: inner.clone(), - name: name.into(), track, producer: Some(producer), })) @@ -355,7 +346,7 @@ impl Socket { /// /// A stream is handed out only while the peer still lists it: one whose track the peer /// dropped first, finished or not, is skipped. - pub fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll, Error>> { + pub fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll, Error>> { let inner = &self.inner; loop { if inner.poll_local_closed(waiter) { @@ -363,7 +354,7 @@ impl Socket { } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - let Some((track, name)) = listing.uni.pop_front() else { + let Some(track) = listing.uni.pop_front() else { return match listing.ended() { Some(end) => Poll::Ready(end), None => Poll::Pending, @@ -371,7 +362,7 @@ impl Socket { }; drop(listing); return match inner.remote.track(&track) { - Ok(consumer) => Poll::Ready(Ok(Some(Uni { name, consumer }))), + Ok(consumer) => Poll::Ready(Ok(Some(consumer))), // The peer's broadcast ended. Err(Error::Unroutable) => Poll::Ready(Ok(None)), // The peer dropped the track since the listing read it. @@ -383,7 +374,7 @@ impl Socket { /// Wait for the next uni stream the peer opened; see [`Self::poll_accept_uni`]. #[cfg(feature = "async")] - pub async fn accept_uni(&self) -> Result, Error> { + pub async fn accept_uni(&self) -> Result, Error> { kio::wait(|waiter| self.poll_accept_uni(waiter)).await } @@ -398,7 +389,7 @@ impl Socket { } let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - let Some((track, (name, info))) = listing.bi.pop_front() else { + let Some((track, info)) = listing.bi.pop_front() else { return match listing.ended() { Some(end) => Poll::Ready(end), None => Poll::Pending, @@ -427,11 +418,7 @@ impl Socket { Err(Error::Duplicate) => continue, Err(err) => return Poll::Ready(Err(err)), }; - return Poll::Ready(Ok(Some(Bi { - name, - producer, - consumer, - }))); + return Poll::Ready(Ok(Some(Bi { producer, consumer }))); } } @@ -477,7 +464,6 @@ impl Drop for Socket { /// A bi stream being opened, from [`Socket::open_bi`]: resolves once the peer answers. pub struct OpenBi { inner: Arc, - name: Arc, /// The stream's track name, which the answer shares. track: Arc, /// This side's track, handed over once the peer answers. @@ -502,11 +488,7 @@ impl OpenBi { drop(listing); let consumer = inner.remote.track(&self.track)?; let producer = self.producer.take().ok_or(Error::Closed)?; - return Poll::Ready(Ok(Bi { - name: self.name.clone(), - producer, - consumer, - })); + return Poll::Ready(Ok(Bi { producer, consumer })); } match &listing.end { Some(Ok(())) => Poll::Ready(Err(Error::Unroutable)), @@ -755,12 +737,12 @@ mod tests { fn track_names_round_trip() { for opener in [Role::Dialer, Role::Listener] { for kind in [Kind::Uni, Kind::Bi] { - let name = track_name(opener, kind, 7, "chat/room"); - assert_eq!(parse(&name), Some((opener, kind, "chat/room"))); + let name = track_name(opener, kind, 7); + assert_eq!(parse(&name), Some((opener, kind))); } } - assert_eq!(track_name(Role::Dialer, Kind::Bi, 0, "x"), ".dial/bi/0/x"); - assert_eq!(track_name(Role::Listener, Kind::Uni, 12, "x"), ".accept/uni/12/x"); + assert_eq!(track_name(Role::Dialer, Kind::Bi, 0), ".dial/bi/0"); + assert_eq!(track_name(Role::Listener, Kind::Uni, 12), ".accept/uni/12"); } #[test] @@ -769,14 +751,15 @@ mod tests { "command", ".dial", ".dial/uni", - ".dial/other/0/x", - ".other/uni/0/x", - "dial/uni/0/x", + ".dial/uni/", + ".dial/other/0", + ".other/uni/0", + "dial/uni/0", ".dial/uni/x", - ".dial/uni/0", - ".dial/uni/07/x", - ".dial/uni/-1/x", - ".dial/uni/+1/x", + ".dial/uni/0/x", + ".dial/uni/07", + ".dial/uni/-1", + ".dial/uni/+1", ] { assert_eq!(parse(name), None, "{name}"); } diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index eb1ef7d67d2..eb07f1fcb1b 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -106,36 +106,40 @@ async fn streams_flow_both_ways() { assert_eq!(accepted.id(), "alice"); // A uni stream each way. - let up = dialer.open_uni("up", None).unwrap(); + let up = dialer.open_uni(None).unwrap(); + assert_eq!(up.name(), ".dial/uni/0"); send(&up, b"from the dialer"); let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) .await .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(&*uni.name, "up"); - assert_eq!(recv(&uni.consumer).await, b"from the dialer"); + assert_eq!(uni.name(), up.name()); + assert_eq!(recv(&uni).await, b"from the dialer"); - let down = accepted.open_uni("down", None).unwrap(); + let down = accepted.open_uni(None).unwrap(); + assert_eq!(down.name(), ".accept/uni/0"); send(&down, b"from the listener"); let uni = kio::clock::timeout(TIMEOUT, dialer.accept_uni()) .await .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(&*uni.name, "down"); - assert_eq!(recv(&uni.consumer).await, b"from the listener"); + assert_eq!(uni.name(), down.name()); + assert_eq!(recv(&uni).await, b"from the listener"); - // A bi stream each way, under the same name: the opener's role keeps them apart. + // A bi stream each way, both the opener's second stream: the opener's role keeps + // them apart. for (opener, acceptor) in [(&dialer, &accepted), (&accepted, &dialer)] { let info = track::Info::default().with_priority(3); - let open = opener.open_bi("chat", info).unwrap(); + let open = opener.open_bi(info).unwrap(); let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(open, acceptor.accept_bi()) }) .await .expect("bi timeout"); let opened = opened.expect("open"); let answered = answered.expect("accept").expect("a stream"); - assert_eq!(&*answered.name, "chat"); + assert!(opened.producer.name().ends_with("/bi/1"), "{}", opened.producer.name()); + assert_eq!(answered.producer.name(), opened.producer.name()); assert_eq!( answered.producer.info().priority, 3, @@ -189,14 +193,14 @@ async fn a_new_dial_replaces_the_old_socket() { ); // The new socket is the new dialer's. - let up = new_dialer.open_uni("up", None).unwrap(); + let up = new_dialer.open_uni(None).unwrap(); send(&up, b"again"); let uni = kio::clock::timeout(TIMEOUT, new.accept_uni()) .await .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(recv(&uni.consumer).await, b"again"); + assert_eq!(recv(&uni).await, b"again"); assert!( kio::clock::timeout(SETTLE, new.closed()).await.is_err(), "the new socket ended" @@ -304,9 +308,9 @@ async fn a_stream_is_accepted_only_while_listed() { let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - let gone = dialer.open_uni("gone", None).unwrap(); - let done = dialer.open_uni("done", None).unwrap(); - let live = dialer.open_uni("live", None).unwrap(); + let gone = dialer.open_uni(None).unwrap(); + let done = dialer.open_uni(None).unwrap(); + let live = dialer.open_uni(None).unwrap(); // Let the listener's listing hear of all three before any ends. assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); drop(gone); @@ -320,7 +324,7 @@ async fn a_stream_is_accepted_only_while_listed() { .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(&*uni.name, "live"); + assert_eq!(uni.name(), live.name()); assert!( kio::clock::timeout(SETTLE, accepted.accept_uni()).await.is_err(), "a stream that left the listing was accepted" @@ -328,7 +332,7 @@ async fn a_stream_is_accepted_only_while_listed() { drop(live); } -/// Two bi opens of one name are two streams, each answered on its own, and dropping the +/// Two concurrent bi opens are two streams, each answered on its own, and dropping the /// socket ends it for the peer even while an open is pending. #[kio::test] async fn a_pending_open_neither_merges_nor_outlives_its_socket() { @@ -336,8 +340,8 @@ async fn a_pending_open_neither_merges_nor_outlives_its_socket() { let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - let first = dialer.open_bi("chat", None).unwrap(); - let second = dialer.open_bi("chat", None).unwrap(); + let first = dialer.open_bi(None).unwrap(); + let second = dialer.open_bi(None).unwrap(); let (first, second, answers) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, second, async { (accepted.accept_bi().await, accepted.accept_bi().await) @@ -348,7 +352,7 @@ async fn a_pending_open_neither_merges_nor_outlives_its_socket() { let (first, second) = (first.expect("first open"), second.expect("second open")); let one = answers.0.expect("accept").expect("a stream"); let two = answers.1.expect("accept").expect("a stream"); - assert_eq!((&*one.name, &*two.name), ("chat", "chat")); + assert_ne!(one.producer.name(), two.producer.name()); send(&first.producer, b"one"); send(&second.producer, b"two"); assert_eq!(recv(&one.consumer).await, b"one"); @@ -358,7 +362,7 @@ async fn a_pending_open_neither_merges_nor_outlives_its_socket() { assert_eq!(recv(&first.consumer).await, b"one back"); assert_eq!(recv(&second.consumer).await, b"two back"); - let pending = dialer.open_bi("unanswered", None).unwrap(); + let pending = dialer.open_bi(None).unwrap(); drop(dialer); assert_eq!( kio::clock::timeout(TIMEOUT, accepted.closed()) @@ -377,8 +381,8 @@ async fn closing_a_socket_ends_its_opens() { let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - let late = dialer.open_bi("late", None).unwrap(); - let stalled = dialer.open_bi("stalled", None).unwrap(); + let late = dialer.open_bi(None).unwrap(); + let stalled = dialer.open_bi(None).unwrap(); // The peer answers the first and never the second, and keeps its broadcast up: the // listener is not polled again, so nothing ends its socket. let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) @@ -386,7 +390,7 @@ async fn closing_a_socket_ends_its_opens() { .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(&*answered.name, "late"); + assert_eq!(answered.producer.name(), ".dial/bi/0", "the earlier open is answered"); // Let the dialer's listing hear the answer before the close. assert!(kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err()); @@ -406,8 +410,8 @@ async fn closing_a_socket_ends_its_opens() { "a stalled open outlived the close" ); - assert!(matches!(dialer.open_uni("after", None), Err(moq_net::Error::Closed))); - assert!(matches!(dialer.open_bi("after", None), Err(moq_net::Error::Closed))); + assert!(matches!(dialer.open_uni(None), Err(moq_net::Error::Closed))); + assert!(matches!(dialer.open_bi(None), Err(moq_net::Error::Closed))); assert!(matches!( kio::clock::timeout(TIMEOUT, dialer.accept_uni()).await.expect("accept"), Err(moq_net::Error::Closed) @@ -415,33 +419,34 @@ async fn closing_a_socket_ends_its_opens() { drop(answered); } -/// A name opened again is a new stream: a handle to the earlier one never reads it, and -/// a bi reopen is answered while the peer still holds its earlier answer. +/// Each open is a new track, never an earlier one's: a held handle to an earlier stream +/// never reads a later one, and a bi open is answered while the peer still holds its +/// answer to an earlier one. #[kio::test] -async fn a_reopened_name_is_a_new_stream() { +async fn each_open_is_a_new_track() { let fixture = Fixture::new().await; let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - // Accepted and held without subscribing, then the opener moves on to a new "chat". - let old = dialer.open_uni("chat", None).unwrap(); + // Accepted and held without subscribing, then the opener moves on to a new stream. + let old = dialer.open_uni(None).unwrap(); let held = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) .await .expect("accept timeout") .unwrap() .expect("a stream"); drop(old); - let new = dialer.open_uni("chat", None).unwrap(); + let new = dialer.open_uni(None).unwrap(); send(&new, b"new"); let fresh = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) .await .expect("accept timeout") .unwrap() .expect("a stream"); - assert_eq!(&*fresh.name, "chat"); - assert_eq!(recv(&fresh.consumer).await, b"new"); + assert_ne!(fresh.name(), held.name()); + assert_eq!(recv(&fresh).await, b"new"); let stale = kio::clock::timeout(SETTLE, async { - let mut subscriber = held.consumer.subscribe(None).await.ok()?; + let mut subscriber = held.subscribe(None).await.ok()?; let mut group = subscriber.recv_group().await.ok()??; Some(group.read_frame().await.ok()??.payload.to_vec()) }) @@ -451,14 +456,14 @@ async fn a_reopened_name_is_a_new_stream() { "the earlier stream's handle read the new one" ); - // A bi answer the peer keeps does not stop the name opening again. - let first = dialer.open_bi("call", None).unwrap(); + // A bi answer the peer keeps does not stop the next bi open. + let first = dialer.open_bi(None).unwrap(); let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) .await .expect("bi timeout"); let _kept = answered.expect("accept").expect("a stream"); drop(opened.expect("open")); - let again = dialer.open_bi("call", None).unwrap(); + let again = dialer.open_bi(None).unwrap(); let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(again, accepted.accept_bi()) }) .await .expect("reopen timeout"); @@ -469,3 +474,30 @@ async fn a_reopened_name_is_a_new_stream() { send(&answered.producer, b"pong"); assert_eq!(recv(&opened.consumer).await, b"pong"); } + +/// A fixed-name track sits beside the streams: accept skips it, and the peer reads it by +/// name from [`Socket::remote`]. +#[kio::test] +async fn fixed_tracks_sit_beside_streams() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + let command = dialer.local().create_track("command", None).unwrap(); + send(&command, b"press a"); + let stream = dialer.open_uni(None).unwrap(); + send(&stream, b"stream"); + + let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + assert_eq!(uni.name(), stream.name(), "accept skips the fixed track"); + assert_eq!(recv(&uni).await, b"stream"); + assert_eq!(recv(&accepted.remote().track("command").unwrap()).await, b"press a"); + assert!( + kio::clock::timeout(SETTLE, accepted.accept_uni()).await.is_err(), + "the fixed track was accepted as a stream" + ); +} diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index c5d6607ff1b..c1ee021130c 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -33,6 +33,8 @@ mod runtime_workers; mod session_revalidate; mod shutdown_signal; mod smoke; +#[cfg(feature = "test-support")] +mod socket_auth; mod stack; mod tenant_matrix; mod upstream_links; diff --git a/rs/moq-relay/tests/integration/socket_auth.rs b/rs/moq-relay/tests/integration/socket_auth.rs new file mode 100644 index 00000000000..29746e74395 --- /dev/null +++ b/rs/moq-relay/tests/integration/socket_auth.rs @@ -0,0 +1,178 @@ +//! Sockets through a relay that admits with `moq-auth` tokens, verified by the reference +//! auth server, granting only the prefixes `doc/concept/socket.md` lists. +//! +//! The listener's token, rooted at the listener's path `room`, subscribes to `.dial/**` +//! and publishes `.accept/**`. A dialer's token publishes `.dial/` and subscribes to +//! `.accept/`, so it can neither dial as another id nor read another id's answer. + +use std::time::Duration; + +use moq_net::socket::{self, Closed, Listener, Socket}; + +use crate::memory; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Long enough for a refused route to have arrived had it been allowed. The relay links +/// are in memory, so this is many round trips. +const SETTLE: Duration = Duration::from_millis(500); + +/// The listener's path, and every token's root. +const ROOT: &str = "room"; + +/// A key, the reference auth server verifying tokens it signs, and a relay asking it. +struct Fixture { + key: moq_auth::Key, + cluster: moq_relay::cluster::Cluster, + auth: moq_relay::auth::Auth, + _keys: tempfile::TempDir, + _server: tokio::task::JoinHandle<()>, +} + +impl Fixture { + async fn new() -> Self { + let keys = tempfile::tempdir().expect("tempdir"); + let key = moq_auth::Key::generate(moq_auth::Algorithm::HS256, None).expect("generate key"); + let key_path = keys.path().join("key.jwk"); + key.to_file(&key_path).expect("write key"); + + let mut policy = moq_auth::serve::Policy::default(); + policy.keys = Some(moq_auth::serve::Keys::File(key_path)); + let server = moq_auth::serve::Server::new(policy).expect("auth server"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind auth"); + let url = format!("http://{}/", listener.local_addr().unwrap()).parse().unwrap(); + let server = tokio::spawn(async move { + let _ = server.serve(listener).await; + }); + + let mut config = moq_relay::auth::Config::default(); + config.url = Some(url); + let auth = config + .init("test", &moq_tokio::tls::Connect::default(), false) + .expect("auth init"); + Self { + key, + cluster: memory::cluster(), + auth, + _keys: keys, + _server: server, + } + } + + /// Sign a token rooted at [`ROOT`] with these grants. + fn token(&self, publish: &str, subscribe: &str) -> String { + let claims = moq_auth::Claims::default() + .with_root(ROOT) + .with_publish([publish.parse().expect("publish pattern")]) + .with_subscribe([subscribe.parse().expect("subscribe pattern")]); + self.key.sign(&claims).expect("sign") + } + + /// A client at [`ROOT`] presenting `token`, publishing and subscribing through one + /// origin, over moq-lite-07. + async fn client(&self, token: &str) -> (moq_net::origin::Producer, moq_net::Session) { + let origin = moq_tokio::origin::spawn(); + let url = format!("moql://relay.example.com/{ROOT}?jwt={token}").parse().unwrap(); + let client = moq_net::Client::new() + .with_publisher(origin.consume()) + .with_subscriber(origin.clone()) + .with_url(url); + let version = "moq-lite-07-wip".parse().expect("lite-07"); + let session = memory::connect(client, version, &self.cluster, &self.auth) + .await + .expect("connect"); + (origin, session) + } + + async fn listener(&self) -> (moq_net::origin::Producer, moq_net::Session) { + self.client(&self.token(".accept/**", ".dial/**")).await + } + + async fn dialer(&self, id: &str) -> (moq_net::origin::Producer, moq_net::Session) { + let token = self.token(&format!(".dial/{id}"), &format!(".accept/{id}")); + self.client(&token).await + } +} + +/// Dial as `id` from `origin` while `listener` accepts, returning both ends. +async fn connect(origin: &moq_net::origin::Producer, listener: &mut Listener, id: &str) -> (Socket, Socket) { + // The session is rooted at the listener's path, so the socket's path is the root. + let dial = socket::dial(origin, "", id).expect("dial"); + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, listener.accept()) }) + .await + .expect("connect timeout"); + (dialed.expect("dial"), accepted.expect("accept")) +} + +/// Write `payload` as one group of `track`. +fn send(track: &moq_net::track::Producer, payload: &'static [u8]) { + let mut group = track.append_group().expect("group"); + group.write_frame(None, payload).expect("frame"); + group.finish().expect("finish"); +} + +/// Subscribe to `track` and read its first frame. +async fn recv(track: &moq_net::track::Consumer) -> Vec { + kio::clock::timeout(TIMEOUT, async { + let mut subscriber = track.subscribe(None).await.expect("subscribe"); + let mut group = subscriber.recv_group().await.expect("group").expect("a group"); + let frame = group.read_frame().await.expect("frame").expect("a frame"); + frame.payload.to_vec() + }) + .await + .expect("read timeout") +} + +/// The prefix grants admit a dial and its accept, and a bi stream carries data both +/// ways. A dialer holding another id's token can neither dial as that id, which would +/// replace its socket, nor read that id's answer. +#[tokio::test] +async fn prefix_grants_scope_sockets_to_their_id() { + let fixture = Fixture::new().await; + let (listening, _listener_session) = fixture.listener().await; + let mut listener = Listener::new(&listening, "").expect("listen"); + + let (alice_origin, _alice_session) = fixture.dialer("alice").await; + let (alice, accepted) = connect(&alice_origin, &mut listener, "alice").await; + assert_eq!(accepted.id(), "alice"); + + let open = alice.open_bi(None).expect("open"); + let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(open, accepted.accept_bi()) }) + .await + .expect("bi timeout"); + let opened = opened.expect("open"); + let answered = answered.expect("accept").expect("a stream"); + send(&opened.producer, b"ping"); + assert_eq!(recv(&answered.consumer).await, b"ping"); + send(&answered.producer, b"pong"); + assert_eq!(recv(&opened.consumer).await, b"pong"); + + // Mallory's token works for mallory's own socket. + let (mallory_origin, _mallory_session) = fixture.dialer("mallory").await; + let (_mallory, mallory_accepted) = connect(&mallory_origin, &mut listener, "mallory").await; + assert_eq!(mallory_accepted.id(), "mallory"); + + // It cannot read alice's answer. + let mut following = mallory_origin + .consume() + .follow(".accept/alice", None) + .expect("follow locally"); + let read = kio::clock::timeout(SETTLE, following.available()).await; + assert!(!matches!(read, Ok(Ok(_))), "mallory read alice's answer"); + + // Nor dial as alice. The dial is announced as it is made, so the listener would hear + // of it without the dial being polled, and would end alice's socket as replaced. + let dial = socket::dial(&mallory_origin, "", "alice").expect("dial locally"); + assert!( + kio::clock::timeout(SETTLE, listener.accept()).await.is_err(), + "the listener heard mallory dial as alice" + ); + assert!( + kio::clock::timeout(SETTLE, dial).await.is_err(), + "mallory's dial as alice resolved" + ); + for socket in [&alice, &accepted] { + let closed: Option = kio::clock::timeout(SETTLE, socket.closed()).await.ok(); + assert_eq!(closed, None, "alice's socket ended"); + } +} From 2fd0dc2c0f911d04c7ca61d4b829aeb89d77aad9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 15:25:30 -0800 Subject: [PATCH 10/25] fix(net): a bi open resolves on the peer's subscription too A listing drops an add its reader never reached once the track leaves it, so a peer that accepts a bi stream, finishes its half at once, and only reads was never seen to answer, and the open hung. The open now also resolves once the peer subscribes to the opener's half. Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 4 ++- rs/moq-net/src/model/socket.rs | 18 ++++++++++--- rs/moq-net/tests/integration/socket.rs | 35 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 9de7c064415..5af71dfcf9e 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -106,7 +106,9 @@ alike, and is never reused within a socket, so each open is a new track. - **Bi.** The opener publishes the track in its own broadcast. The peer accepts it by publishing the same name in its broadcast, with the opener's track info, and each side subscribes to the other's. The opener's stream resolves - once the answer is listed. + once it sees the answer listed or the peer subscribing to its track. The + subscription counts because a listing drops an add its reader never reached, + so an answer finished at once may never be seen listed. The opener prefix and the number together work like a QUIC stream ID. The prefix keeps the two sides' streams apart, the way the ID's low bit does, so diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 18c8e8e9b72..5b84f71b65a 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -337,6 +337,7 @@ impl Socket { Ok(kio::Pending::new(OpenBi { inner: inner.clone(), track, + demand: producer.demand(), producer: Some(producer), })) } @@ -468,12 +469,18 @@ pub struct OpenBi { track: Arc, /// This side's track, handed over once the peer answers. producer: Option, + /// Whether anyone reads this side's track, which only the peer does once it accepted. + demand: track::Demand, } impl OpenBi { - /// Poll for the peer's answer. Fails with [`Error::Unroutable`] once the peer's - /// broadcast ends without one, and with [`Error::Closed`] once this side closed, even - /// if the answer came. + /// Poll for the peer's answer: its half listed, or its subscription to this side's + /// half. Fails with [`Error::Unroutable`] once the peer's broadcast ends without one, + /// and with [`Error::Closed`] once this side closed, even if the answer came. + /// + /// The subscription counts because a listing drops an add its reader never reached + /// once the track leaves it: a peer that finishes its half at once and only reads + /// would otherwise never be seen to answer. pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let inner = &self.inner; // This side's track went with its broadcast, and a peer that keeps its own up @@ -481,9 +488,12 @@ impl OpenBi { if inner.poll_local_closed(waiter) { return Poll::Ready(Err(Error::Closed)); } + // Polled until it parks, so a later change wakes this. + while let Poll::Ready(Ok(_)) = self.demand.poll_wanted_changed(waiter) {} + let read = self.demand.wanted().is_some(); let mut listing = inner.listing.lock(); listing.drain(waiter, inner.role, &inner.remote); - if listing.opening.get(&self.track) == Some(&true) { + if read || listing.opening.get(&self.track) == Some(&true) { listing.opening.remove(&self.track); drop(listing); let consumer = inner.remote.track(&self.track)?; diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index eb07f1fcb1b..0135454e6ca 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -475,6 +475,41 @@ async fn each_open_is_a_new_track() { assert_eq!(recv(&opened.consumer).await, b"pong"); } +/// A peer that accepts a bi stream, finishes its half at once, and only reads still +/// answers the open, though its half left the listing before the opener read it. +#[kio::test] +async fn a_finished_answer_still_answers_the_open() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + + let open = dialer.open_bi(None).unwrap(); + let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + answered.producer.finish().unwrap(); + drop(answered.producer); + // Long enough for the answer to come and go before the open is polled. + assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); + + let (opened, read) = kio::clock::timeout(TIMEOUT, async { + futures::join!( + async { + let opened = open.await.expect("open"); + send(&opened.producer, b"request"); + opened + }, + recv(&answered.consumer) + ) + }) + .await + .expect("open timeout"); + assert_eq!(read, b"request"); + drop(opened); +} + /// A fixed-name track sits beside the streams: accept skips it, and the peer reads it by /// name from [`Socket::remote`]. #[kio::test] From 1378093d6ba01257f7a12091b2f516aff45130d9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 16:04:37 -0800 Subject: [PATCH 11/25] fix(net): an accepted bi stream reads the opener's half as its answer The opener learned of an accept only from the answer's listing, which drops a track that comes and goes unread, so an answer dropped at once left the open pending. Demand helped only once the application subscribed. Accepting now subscribes to the opener's half, held until the opener drops it, and an open resolves once its half is read. The `opening` bookkeeping goes away. As with any stream, the answer reaches the opener only while the acceptor holds it, finished or not. Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 10 ++-- rs/moq-net/src/model/socket.rs | 68 ++++++++++-------------- rs/moq-net/tests/integration/socket.rs | 72 +++++++++++++++++++------- 3 files changed, 86 insertions(+), 64 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 5af71dfcf9e..3bb69358fec 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -105,10 +105,12 @@ alike, and is never reused within a socket, so each open is a new track. accepts it and subscribes. - **Bi.** The opener publishes the track in its own broadcast. The peer accepts it by publishing the same name in its broadcast, with the opener's track - info, and each side subscribes to the other's. The opener's stream resolves - once it sees the answer listed or the peer subscribing to its track. The - subscription counts because a listing drops an add its reader never reached, - so an answer finished at once may never be seen listed. + info, and subscribes to the opener's track until the opener drops it. That + subscription is the accept: the opener's stream resolves once its track is + read. A listing can't carry it, because it drops an add its reader never + reached, so an answer dropped at once might never be seen. As with any + stream, the answer reaches the opener only while the peer holds it, finished + or not. The opener prefix and the number together work like a QUIC stream ID. The prefix keeps the two sides' streams apart, the way the ID's low bit does, so diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 5b84f71b65a..e4112098767 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -138,9 +138,12 @@ struct Listing { uni: LinkedHashSet>, /// Bi streams the peer opened, with the opener's track info, not yet accepted. bi: LinkedHashMap, track::Info>, - /// This side's bi opens waiting for an answer, by track name, and whether the peer - /// listed it. - opening: HashMap, bool>, + /// This side reading the opener's half of each bi stream it accepted, by track name, + /// until the opener's half leaves the listing. The opener learns of the accept from + /// this demand, not from the answer's listing: a listing drops a track that comes and + /// goes before its reader reaches it, so an answer dropped at once would never be + /// seen, and the application may not read the opener's half before it hears back. + reading: HashMap, kio::Pending>, /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. end: Option>, } @@ -175,10 +178,12 @@ impl Listing { Poll::Ready(Ok(Some(event))) => event, Poll::Ready(Ok(None)) => { self.end = Some(Ok(())); + self.reading.clear(); return; } Poll::Ready(Err(err)) => { self.end = Some(Err(err)); + self.reading.clear(); return; } }; @@ -203,14 +208,11 @@ impl Listing { } (false, Kind::Bi, None) => { self.bi.remove(&name); + // The opener dropped its half, so it resolved or gave up. + self.reading.remove(&name); } - // An answer seen once stays seen: the opener reads whatever it wrote. - (true, Kind::Bi, Some(_)) => { - if let Some(answered) = self.opening.get_mut(&name) { - *answered = true; - } - } - (true, Kind::Bi, None) => {} + // An open learns of its answer from the peer reading it, not the listing. + (true, Kind::Bi, _) => {} // This side's uni streams live only in its own broadcast. (true, Kind::Uni, _) => {} } @@ -325,15 +327,7 @@ impl Socket { pub fn open_bi(&self, info: impl Into>) -> Result, Error> { let inner = &self.inner; let track: Arc = inner.open(Kind::Bi).into(); - // Registered first, so a poll that reads the peer's answer before this returns keeps it. - inner.listing.lock().opening.insert(track.clone(), false); - let producer = match inner.create_track(track.to_string(), info) { - Ok(producer) => producer, - Err(err) => { - inner.listing.lock().opening.remove(&track); - return Err(err); - } - }; + let producer = inner.create_track(track.to_string(), info)?; Ok(kio::Pending::new(OpenBi { inner: inner.clone(), track, @@ -382,6 +376,11 @@ impl Socket { /// Poll for the next bi stream the peer opened, answering it with a track that mirrors /// the opener's info; `None` once the peer's broadcast ended, and [`Error::Closed`] /// once this side closed. Skips streams as [`Self::poll_accept_uni`] does. + /// + /// The socket reads the opener's half until the opener drops it, which is how the + /// opener learns of the accept. As with any stream, the answer reaches the opener only + /// while this side holds it: finishing it at once is fine, dropping it before the + /// opener reads it is not. pub fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll, Error>> { let inner = &self.inner; loop { @@ -419,6 +418,8 @@ impl Socket { Err(Error::Duplicate) => continue, Err(err) => return Poll::Ready(Err(err)), }; + // How the opener learns of the accept. + inner.listing.lock().reading.insert(track, consumer.subscribe(None)); return Poll::Ready(Ok(Some(Bi { producer, consumer }))); } } @@ -469,18 +470,14 @@ pub struct OpenBi { track: Arc, /// This side's track, handed over once the peer answers. producer: Option, - /// Whether anyone reads this side's track, which only the peer does once it accepted. + /// Whether anyone reads this side's track, which the peer does from its accept on. demand: track::Demand, } impl OpenBi { - /// Poll for the peer's answer: its half listed, or its subscription to this side's - /// half. Fails with [`Error::Unroutable`] once the peer's broadcast ends without one, - /// and with [`Error::Closed`] once this side closed, even if the answer came. - /// - /// The subscription counts because a listing drops an add its reader never reached - /// once the track leaves it: a peer that finishes its half at once and only reads - /// would otherwise never be seen to answer. + /// Poll for the peer's answer, which it gives by reading this side's half. Fails with + /// [`Error::Unroutable`] once the peer's broadcast ends without one, and with + /// [`Error::Closed`] once this side closed, even if the answer came. pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let inner = &self.inner; // This side's track went with its broadcast, and a peer that keeps its own up @@ -490,16 +487,13 @@ impl OpenBi { } // Polled until it parks, so a later change wakes this. while let Poll::Ready(Ok(_)) = self.demand.poll_wanted_changed(waiter) {} - let read = self.demand.wanted().is_some(); - let mut listing = inner.listing.lock(); - listing.drain(waiter, inner.role, &inner.remote); - if read || listing.opening.get(&self.track) == Some(&true) { - listing.opening.remove(&self.track); - drop(listing); + if self.demand.wanted().is_some() { let consumer = inner.remote.track(&self.track)?; let producer = self.producer.take().ok_or(Error::Closed)?; return Poll::Ready(Ok(Bi { producer, consumer })); } + let mut listing = inner.listing.lock(); + listing.drain(waiter, inner.role, &inner.remote); match &listing.end { Some(Ok(())) => Poll::Ready(Err(Error::Unroutable)), Some(Err(err)) => Poll::Ready(Err(err.clone())), @@ -516,14 +510,6 @@ impl kio::Task for OpenBi { } } -impl Drop for OpenBi { - fn drop(&mut self) { - if self.producer.is_some() { - self.inner.listing.lock().opening.remove(&self.track); - } - } -} - /// Dial the listener at `path` as `id`, from [`dial`]: resolves once the listener accepts. pub struct Dialing { id: Arc, diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 0135454e6ca..741cda39bcc 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -475,10 +475,53 @@ async fn each_open_is_a_new_track() { assert_eq!(recv(&opened.consumer).await, b"pong"); } -/// A peer that accepts a bi stream, finishes its half at once, and only reads still -/// answers the open, though its half left the listing before the opener read it. +/// Open a bi stream from `dialer`, and have `accepted` accept it, write one group, and +/// finish its half before the open is polled, never reading the opener's half itself. +/// The opener still resolves and reads what the peer wrote. +async fn a_finished_answer_reaches_the_opener(dialer: &Socket, accepted: &Socket) { + let open = dialer.open_bi(None).unwrap(); + let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + send(&answered.producer, b"greeting"); + answered.producer.finish().unwrap(); + // Held, finished, as any stream is until the peer reads it. + let answer = answered.producer; + assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); + + let opened = kio::clock::timeout(TIMEOUT, open) + .await + .expect("open timeout") + .expect("open"); + assert_eq!(recv(&opened.consumer).await, b"greeting"); + drop(answer); +} + +/// A finished answer reaches the opener through a relay. +#[kio::test] +async fn a_finished_answer_reaches_the_opener_through_a_relay() { + let fixture = Fixture::new().await; + let mut listener = Listener::new(&fixture.listener, "room").unwrap(); + let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; + a_finished_answer_reaches_the_opener(&dialer, &accepted).await; +} + +/// A finished answer reaches the opener on one origin, with no session between them. #[kio::test] -async fn a_finished_answer_still_answers_the_open() { +async fn a_finished_answer_reaches_the_opener_on_one_origin() { + let shared = origin(1); + let mut listener = Listener::new(&shared, "room").unwrap(); + let (dialer, accepted) = connect(&shared, &mut listener, "alice").await; + a_finished_answer_reaches_the_opener(&dialer, &accepted).await; +} + +/// A peer that accepts a bi stream and drops its half at once, only reading, still +/// answers the open: its half left the listing before the opener could see it, and it +/// has not subscribed to the opener's half yet either. +#[kio::test] +async fn a_dropped_answer_still_answers_the_open() { let fixture = Fixture::new().await; let mut listener = Listener::new(&fixture.listener, "room").unwrap(); let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; @@ -489,25 +532,16 @@ async fn a_finished_answer_still_answers_the_open() { .expect("accept timeout") .unwrap() .expect("a stream"); - answered.producer.finish().unwrap(); drop(answered.producer); - // Long enough for the answer to come and go before the open is polled. + let request = answered.consumer; assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); - let (opened, read) = kio::clock::timeout(TIMEOUT, async { - futures::join!( - async { - let opened = open.await.expect("open"); - send(&opened.producer, b"request"); - opened - }, - recv(&answered.consumer) - ) - }) - .await - .expect("open timeout"); - assert_eq!(read, b"request"); - drop(opened); + let opened = kio::clock::timeout(TIMEOUT, open) + .await + .expect("open timeout") + .expect("open"); + send(&opened.producer, b"request"); + assert_eq!(recv(&request).await, b"request"); } /// A fixed-name track sits beside the streams: accept skips it, and the peer reads it by From c5e1854879ce691ff9f322faa01701c72144bf53 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 20:15:17 -0800 Subject: [PATCH 12/25] fix(net): release a socket's accept reads on close, ordered against removal A bi accept holds a read of the opener's half, which is how the opener learns of the accept. Closing or dropping the socket now releases every such read, and an accept registers its read under the listing lock behind a placeholder, so a close or the opener's removal drained by a concurrent poller either sees the read or is seen by the accept. The concept page notes that any reader of a bi track counts as its accept, so reads of `.dial` and `.accept` belong to the two ends. Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 14 ++- rs/moq-net/src/model/socket.rs | 127 ++++++++++++++++++------- rs/moq-net/tests/integration/socket.rs | 39 ++++++++ 3 files changed, 142 insertions(+), 38 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 3bb69358fec..f35ae230bd7 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -53,8 +53,12 @@ listener's path `L`, each grant is a prefix: The dialer's grant names its id, which is what stops one dialer from dialing as another, and so replacing its socket, or reading another's answer. A pattern matches `.`-prefixed segments like any other, so `**` would cover sockets too; -hiding narrows only discovery. No mid-path wildcard is needed, because the -root carries `L`. It's the same shape as a request/response pair, such as +hiding narrows only discovery. Keep reads of `.dial` and `.accept` to the two +ends: a bi open counts any read of its track as the accept (see +[Streams](#streams)), so a third party that may read them, such as an archiver +granted `**` or anyone on an origin without auth, can resolve an open before +the peer accepts it. No mid-path wildcard is needed, because the root carries +`L`. It's the same shape as a request/response pair, such as `request/` and `response/`. `moq auth sign` writes these as, for example: @@ -105,9 +109,9 @@ alike, and is never reused within a socket, so each open is a new track. accepts it and subscribes. - **Bi.** The opener publishes the track in its own broadcast. The peer accepts it by publishing the same name in its broadcast, with the opener's track - info, and subscribes to the opener's track until the opener drops it. That - subscription is the accept: the opener's stream resolves once its track is - read. A listing can't carry it, because it drops an add its reader never + info, and subscribes to the opener's track until the opener drops it or the + peer closes the socket. That subscription is the accept: the opener's stream + resolves once its track is read. A listing can't carry it, because it drops an add its reader never reached, so an answer dropped at once might never be seen. As with any stream, the answer reaches the opener only while the peer holds it, finished or not. diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index e4112098767..14a7080d753 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -139,11 +139,13 @@ struct Listing { /// Bi streams the peer opened, with the opener's track info, not yet accepted. bi: LinkedHashMap, track::Info>, /// This side reading the opener's half of each bi stream it accepted, by track name, - /// until the opener's half leaves the listing. The opener learns of the accept from - /// this demand, not from the answer's listing: a listing drops a track that comes and - /// goes before its reader reaches it, so an answer dropped at once would never be - /// seen, and the application may not read the opener's half before it hears back. - reading: HashMap, kio::Pending>, + /// until the opener's half leaves the listing or this side closes. The opener learns + /// of the accept from this demand, not from the answer's listing: a listing drops a + /// track that comes and goes before its reader reaches it, so an answer dropped at once + /// would never be seen, and the application may not read the opener's half before it + /// hears back. `None` while an accept is answering the stream, so a removal drained + /// meanwhile, by any poller, also drops the read the accept is about to register. + reading: HashMap, Option>>, /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. end: Option>, } @@ -247,7 +249,17 @@ impl Inner { /// End the socket for `reason`, retracting this side's broadcast. fn close(&self, reason: Closed) { self.closed.lock().get_or_insert(reason); + self.retract(); + } + + /// Retract this side's broadcast and stop reading the opener's half of every bi stream + /// it accepted. Closed first, so an accept registering a read after the release sees + /// the close and drops it. + fn retract(&self) { self.local.close(); + // Dropped outside the lock. + let reading = std::mem::take(&mut self.listing.lock().reading); + drop(reading); } /// Whether this side's broadcast ended, registering `waiter` for when it does. @@ -377,8 +389,8 @@ impl Socket { /// the opener's info; `None` once the peer's broadcast ended, and [`Error::Closed`] /// once this side closed. Skips streams as [`Self::poll_accept_uni`] does. /// - /// The socket reads the opener's half until the opener drops it, which is how the - /// opener learns of the accept. As with any stream, the answer reaches the opener only + /// The socket reads the opener's half until the opener drops it or this side closes, + /// which is how the opener learns of the accept. As with any stream, the answer reaches the opener only /// while this side holds it: finishing it at once is fine, dropping it before the /// opener reads it is not. pub fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll, Error>> { @@ -395,35 +407,82 @@ impl Socket { None => Poll::Pending, }; }; + // Already answered: the peer listed it again, which it does only by opening a + // number twice. + if listing.reading.contains_key(&track) { + continue; + } + listing.reading.insert(track.clone(), None); drop(listing); - let consumer = match inner.remote.track(&track) { - Ok(consumer) => consumer, - // The peer's broadcast ended. - Err(Error::Unroutable) => return Poll::Ready(Ok(None)), - // The peer dropped the track since the listing read it. - Err(Error::NotFound) => continue, - Err(err) => return Poll::Ready(Err(err)), - }; - // The opener's track info, minus how its copy reached this hop. - let info = track::Info { - properties: None, - unchecked: false, - ..info - }; - // Answered under the opener's track name. - let producer = match inner.create_track(track.to_string(), info) { - Ok(producer) => producer, - // The peer listed a stream again that this side already answered, which it - // does only by opening a number twice. - Err(Error::Duplicate) => continue, - Err(err) => return Poll::Ready(Err(err)), + + let answered = self.answer_bi(&track, info); + let mut listing = inner.listing.lock(); + let (bi, read) = match answered { + Ok(Some(answered)) => answered, + Ok(None) => { + listing.reading.remove(&track); + continue; + } + Err(err) => { + listing.reading.remove(&track); + return match err { + // The peer's broadcast ended. + Error::Unroutable => Poll::Ready(Ok(None)), + err => Poll::Ready(Err(err)), + }; + } }; - // How the opener learns of the accept. - inner.listing.lock().reading.insert(track, consumer.subscribe(None)); - return Poll::Ready(Ok(Some(Bi { producer, consumer }))); + // Under the lock, so a close or a removal of the stream either comes after this + // and drops the read, or came first and is seen here. + let closed = inner.poll_local_closed(&kio::Waiter::noop()); + if !closed && let Some(slot) = listing.reading.get_mut(&track) { + // How the opener learns of the accept. + *slot = Some(read); + return Poll::Ready(Ok(Some(bi))); + } + // Dropped outside the lock. + let stale = listing.reading.remove(&track); + drop(listing); + drop((stale, read, bi)); + if closed { + return Poll::Ready(Err(Error::Closed)); + } + // The opener dropped its half while this answered it. } } + /// Answer the peer's bi stream `track` with a track mirroring the opener's `info`, and + /// start reading the opener's half. `None` for a stream to skip. + fn answer_bi( + &self, + track: &str, + info: track::Info, + ) -> Result)>, Error> { + let inner = &self.inner; + let consumer = match inner.remote.track(track) { + Ok(consumer) => consumer, + // The peer dropped the track since the listing read it. + Err(Error::NotFound) => return Ok(None), + Err(err) => return Err(err), + }; + // The opener's track info, minus how its copy reached this hop. + let info = track::Info { + properties: None, + unchecked: false, + ..info + }; + // Answered under the opener's track name. + let producer = match inner.create_track(track.to_string(), info) { + Ok(producer) => producer, + // This side still holds an answer under the name, which the peer listed again + // only by opening a number twice. + Err(Error::Duplicate) => return Ok(None), + Err(err) => return Err(err), + }; + let read = consumer.subscribe(None); + Ok(Some((Bi { producer, consumer }, read))) + } + /// Wait for the next bi stream the peer opened; see [`Self::poll_accept_bi`]. #[cfg(feature = "async")] pub async fn accept_bi(&self) -> Result, Error> { @@ -459,7 +518,7 @@ impl Socket { impl Drop for Socket { fn drop(&mut self) { // A pending bi open still holds the shared half, but not the socket. - self.inner.local.close(); + self.inner.retract(); } } @@ -475,7 +534,9 @@ pub struct OpenBi { } impl OpenBi { - /// Poll for the peer's answer, which it gives by reading this side's half. Fails with + /// Poll for the peer's answer, which it gives by reading this side's half. Any reader + /// counts, so only the peer should be able to read this side's broadcast; see the + /// authorization section of `doc/concept/socket.md`. Fails with /// [`Error::Unroutable`] once the peer's broadcast ends without one, and with /// [`Error::Closed`] once this side closed, even if the answer came. pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 741cda39bcc..6c04585d354 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -544,6 +544,45 @@ async fn a_dropped_answer_still_answers_the_open() { assert_eq!(recv(&request).await, b"request"); } +/// Closing or dropping a socket stops it reading the opener's half of the bi streams it +/// accepted, even while the opener keeps its half: the read that told the opener of the +/// accept is the socket's own, not the application's. +#[kio::test] +async fn closing_a_socket_releases_its_accepted_streams() { + for drop_it in [false, true] { + let shared = origin(1); + let mut listener = Listener::new(&shared, "room").unwrap(); + let (dialer, accepted) = connect(&shared, &mut listener, "alice").await; + + let open = dialer.open_bi(None).unwrap(); + let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) + .await + .expect("accept timeout") + .unwrap() + .expect("a stream"); + // The application is done with the stream; only the socket's own read is left. + drop(answered); + let opened = kio::clock::timeout(TIMEOUT, open) + .await + .expect("open timeout") + .expect("open"); + let mut demand = opened.producer.demand(); + assert!(demand.wanted().is_some(), "the accept reads the opener's half"); + + match drop_it { + true => drop(accepted), + false => accepted.close(), + } + kio::clock::timeout(TIMEOUT, async { + while demand.wanted().is_some() { + demand.wanted_changed().await.unwrap(); + } + }) + .await + .expect("the closed socket still reads the opener's half"); + } +} + /// A fixed-name track sits beside the streams: accept skips it, and the peer reads it by /// name from [`Socket::remote`]. #[kio::test] From 0ec5fd9dcacf5a234688aa18e44003a41329685c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 22:16:23 -0800 Subject: [PATCH 13/25] feat(net)!: replace socket listeners with .tcp endpoints, one pipe per session Each endpoint announces one hidden broadcast, `/.tcp/`, once, and dials and accepts sessions over it. A session is two unlisted tracks named `/`, one per endpoint, carrying one message per group. The dialer answers only a session it dialed, so a dial forged in its name never connects, and no listing or announcement names a session. The handshake uses no instances or track listings, so it works over moq-transport hops. Removes the Listener, dial, Socket, and QUIC-style uni/bi streams. Co-Authored-By: Claude Opus 5.5 --- doc/concept/index.md | 2 +- doc/concept/socket.md | 248 ++-- doc/lib/rs/moq-net.md | 2 +- quest/m1/socket/README.md | 193 +-- quest/m1/socket/boy.md | 40 +- quest/m1/socket/carla.md | 48 +- quest/m1/socket/js.md | 20 +- quest/m1/socket/py.md | 13 +- quest/m2/announce-credit.md | 6 +- quest/m2/mesh/README.md | 6 +- quest/m2/mesh/endpoint.md | 19 +- quest/m2/stream/README.md | 23 +- quest/m2/stream/rust.md | 21 +- rs/moq-net/CHANGELOG.md | 2 +- rs/moq-net/src/model/broadcast.rs | 9 + rs/moq-net/src/model/socket.rs | 1186 +++++++---------- rs/moq-net/src/model/track.rs | 15 + rs/moq-net/tests/integration/socket.rs | 746 ++++------- rs/moq-relay/tests/integration/socket_auth.rs | 189 ++- 19 files changed, 1197 insertions(+), 1591 deletions(-) diff --git a/doc/concept/index.md b/doc/concept/index.md index f5778d5210e..3009b31f92c 100644 --- a/doc/concept/index.md +++ b/doc/concept/index.md @@ -38,7 +38,7 @@ watched at 100 ms by one viewer and 10 s by another. - [Transport](/concept/transport): why QUIC, and the WebTransport, WebSocket, and raw QUIC paths onto it. - [moq-lite](/concept/moq-lite): the pub/sub protocol, discovery, path patterns, subscriptions, and congestion behavior. - [Identity and recovery](/concept/identity): choosing stream instances, following restarts, and answering work on demand without mixing streams. -- [Sockets](/concept/socket): dial and accept point-to-point connections through relays, with tracks as streams. Needs opt-in moq-lite 07 on every hop. +- [Sockets](/concept/socket): dial and accept point-to-point sessions through relays, each a reliable, ordered pipe of messages. - [hang](/concept/hang): the media catalog, containers, encryption, and how to extend both. - [Archives](/concept/archives): record listed tracks into object storage and serve them back from a host. - [Audio jitter](/concept/audio-jitter): how a receiver sizes its audio playout target from arrival timing. diff --git a/doc/concept/socket.md b/doc/concept/socket.md index f35ae230bd7..3490274df4e 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -1,156 +1,150 @@ --- title: Sockets -description: Dial and accept point-to-point connections over moq-lite, with tracks as streams +description: Dial and accept point-to-point sessions over MoQ, each a reliable, ordered pipe of messages --- # Sockets -A socket connects one dialer to one listener through relays, the way a TCP -connection does, provided every hop runs moq-lite 07, which is opt-in (see -[Versions](#versions)). A listener at a path accepts the sockets that dialers open to -it, and inside a socket either side opens and accepts tracks the way QUIC opens -and accepts streams. Use it for per-client state that only the two ends need, -such as a viewer's commands and the replies to them, while whatever every -client needs stays in an ordinary broadcast. - -A socket is a convention over moq-lite's announcements, broadcasts, and track -listings, not a wire extension, so relays carry it unchanged. This page is the -convention, enough to implement it in any language. +A socket connects two endpoints through relays, the way a TCP connection +connects two hosts. Each endpoint announces one hidden broadcast, dials other +endpoints, and accepts their dials. Each connection is a session: a reliable, +ordered, duplex pipe of messages that only the two ends can read. Use it for +per-client state that only the two ends need, such as a viewer's commands and +the replies to them, while whatever every client needs stays in an ordinary +broadcast. + +A socket is a convention over announcements, broadcasts, and track +subscriptions, not a wire extension. It uses no instances and no track +listings, so it works over every moq-lite version and moq-transport, and relays +carry it unchanged. This page is the convention, enough to implement it in any +language. [`moq_net::socket`](https://docs.rs/moq-net/latest/moq_net/socket/) implements it in Rust. -## Paths +## Endpoints -A listener at `L` keeps `L` itself for what it shares with everyone. Each -socket is two broadcasts beneath it: +An endpoint with id `` announces one broadcast, once: -| Path | Published by | Holds | -| --- | --- | --- | -| `L/.dial/` | The dialer | The dialer's tracks | -| `L/.accept/` | The listener | The listener's tracks | +```text +/.tcp/ +``` -The id is one path segment that the dialer chooses, such as a random id or an -endpoint ID. Run one listener per path: two would both answer a dial, and the -dialer would follow only one of them. Both segments start with `.`, so sockets are -[hidden](/concept/moq-lite#hidden-broadcasts) from anyone listing `L`. The -listener discovers dialers with a one-segment watch of `L/.dial/*`, which a -relay serves without scanning everything beneath `L`. +The id is one path segment: a well-known name for a server, such as a game, or +a random id for a client. The prefix is shared by every endpoint that talks to +each other. The `.tcp` segment starts with `.`, so endpoints are +[hidden](/concept/moq-lite#hidden-broadcasts) from anyone listing the prefix, +and it names the mode: one reliable, ordered pipe per session, as TCP gives. + +Connections are never announced. An endpoint's broadcast holds only session +tracks, served on request, and its announcement is the only thing a relay +routes. + +## Sessions + +A session between a dialer `A` and an acceptor `B` is named by a session id +`S` that `A` picks: random and unguessable, at least 128 bits, such as 32 hex +digits. Both halves of the session are tracks with the same name, +`/`, one in each endpoint's broadcast: + +| Track | In broadcast | Written by | Read by | +| --- | --- | --- | --- | +| `A/S` | `/.tcp/A` | `A` | `B` | +| `A/S` | `/.tcp/B` | `B` | `A` | + +The dialer's id leads the name in both broadcasts, so an endpoint tells a dial +from an answer by the name alone: a request under another endpoint's id is that +endpoint dialing, and a request under its own id answers one of its own dials. + +### Dial + +1. `A` picks `S` and holds its own half, `A/S` in its broadcast, unanswered. +2. `A` subscribes to `/.tcp/B` for the track `A/S`, waiting for `B`'s + endpoint to be announced if it is not yet. +3. `B` receives that subscription as a request for a track it does not have: + an incoming dial, claiming to be from `A`, with session `S`. + +### Accept + +1. `B` subscribes to `/.tcp/A` for the track `A/S`, and holds `A`'s + subscription unanswered. +2. `A` answers that subscription only if it dialed `S` and is still waiting, + and resets any other. That makes the exchange a return-routability check: + a dial forged in `A`'s name, by anyone who can subscribe to `B`, reaches an + `A` that never dialed it, so `A` resets `B`'s subscription and `B` resets the + forger's. +3. Once `A` answers, `B` answers `A`'s subscription, and the session is open + for both. `B` writes nothing before `A` answers. + +### Reject + +`B` resets `A`'s subscription, with not found, as for a session that does not +exist. An endpoint that is not accepting dials resets them the same way. A dial +that gives up resets `B`'s subscription in turn. + +## Messages + +Each side writes its own half and reads the peer's. A message is one group, so a +slow message never holds up the next, and messages are ordered by group +sequence. A reader reads reliably and in order by default: oldest group first, +with no delay budget, so it never skips a message. An application may opt into a +lossy budget for a session whose messages go stale, such as positions, by +setting a delay budget on its subscription. + +Finishing a half closes the session that way, like a FIN. Resetting it is an +error. An endpoint closing, or its broadcast ending, resets every session it +holds. + +There is no multiplexing within a session: another channel is another session. +Several sessions between the same two endpoints are independent, and a +restarted endpoint starts new sessions. A channel that every client reads, such +as a game's status, stays an ordinary broadcast. + +## Hidden names + +The session id is the secret that keeps a session to its two ends: the token +grants below let any endpoint subscribe to any other, so whoever learns a +session's track name can read it, or answer it before its peer. A session +track therefore never appears where a third party can read it: + +- It is never announced. +- It is never listed: an endpoint serves its session tracks only on request, + so a moq-lite 07 track listing (`TRACKS`) of an endpoint's broadcast names + none of them, even to a reader allowed everything. +- Logs and metrics that a third party can read must not carry it. ## Authorization Sockets need nothing beyond ordinary [token grants](/bin/relay/auth#path-matching). With the token's root at the -listener's path `L`, each grant is a prefix: +socket prefix, each grant is a prefix: ```jsonc -// The listener: reads every dial and answers every id. -{ "root": "L", "subscribe": [".dial/**"], "publish": [".accept/**"] } - -// The dialer with id : its own dial and its own answer. -{ "root": "L", "publish": [".dial/"], "subscribe": [".accept/"] } +// The endpoint : publishes its own endpoint, reads every endpoint. +{ "root": "", "publish": [".tcp/"], "subscribe": [".tcp/**"] } ``` -The dialer's grant names its id, which is what stops one dialer from dialing as -another, and so replacing its socket, or reading another's answer. A pattern -matches `.`-prefixed segments like any other, so `**` would cover sockets too; -hiding narrows only discovery. Keep reads of `.dial` and `.accept` to the two -ends: a bi open counts any read of its track as the accept (see -[Streams](#streams)), so a third party that may read them, such as an archiver -granted `**` or anyone on an origin without auth, can resolve an open before -the peer accepts it. No mid-path wildcard is needed, because the root carries -`L`. It's the same shape as a request/response pair, such as -`request/` and `response/`. +The publish grant pins the id, so no endpoint can announce as another. The +subscribe grant lets it reach any endpoint, which is what dialing and accepting +need; the session id keeps it out of other endpoints' sessions. A narrower +subscribe grant, such as a client allowed only `.tcp/`, also works: it +can then dial only that server and accept only its dials. -`moq auth sign` writes these as, for example: +`moq auth sign` writes this as, for example: ```bash -moq auth sign --key key.jwk --root L --subscribe '.dial/**' --publish '.accept/**' > listener.jwt -moq auth sign --key key.jwk --root L --publish ".dial/$ID" --subscribe ".accept/$ID" > dialer.jwt +moq auth sign --key key.jwk --root "$PREFIX" --publish ".tcp/$ID" --subscribe '.tcp/**' > endpoint.jwt ``` -Each client then connects to the relay at `L` with its token, and its socket's -path is the session root. - -## Instances - -Each dial is one socket, named by its -[instance](/concept/moq-lite#publisher-instances): - -1. The dialer announces `L/.dial/` under a fresh random instance and - follows `L/.accept/` pinned to that instance. -2. When the application accepts, the listener requests `L/.dial/` pinned - to the instance it saw announced, then announces `L/.accept/` under the - same instance at epoch 0. -3. The dialer's follow resolves on that answer, and the socket is open. +Each client then connects to the relay at the prefix with its token, and its +endpoint's prefix is the session root. -An answer to another dial of the same id carries another instance, so a -dialer never takes it. Dialing again with the same id, as a restarted dialer -does, announces a new instance: the listener ends the socket it replaces and -answers the new one on its next accept. +Signing and encryption are not part of the convention: an endpoint id is a +name the token pins, not a key, and relays are trusted with session names. A +planned mesh layer makes ids public keys. -A dial waits for as long as it takes, so give up by dropping it. Nothing reads -a dialer before the application accepts it: dialers wait in the listener's -announcements without limit, and its broadcast is requested only on accept. - -## Streams - -A stream is a track. Like a QUIC stream it has no name of its own: it is -named by who opened it, which way it carries data, and its number. - -```text -.// -``` +## Later -`` is `dial` or `accept`, and `` is `uni` or `bi`. `` -counts the opener's streams in decimal from 0 with no leading zeros, uni and bi -alike, and is never reused within a socket, so each open is a new track. - -- **Uni.** The opener publishes the track in its own broadcast. The peer - accepts it and subscribes. -- **Bi.** The opener publishes the track in its own broadcast. The peer accepts - it by publishing the same name in its broadcast, with the opener's track - info, and subscribes to the opener's track until the opener drops it or the - peer closes the socket. That subscription is the accept: the opener's stream - resolves once its track is read. A listing can't carry it, because it drops an add its reader never - reached, so an answer dropped at once might never be seen. As with any - stream, the answer reaches the opener only while the peer holds it, finished - or not. - -The opener prefix and the number together work like a QUIC stream ID. The -prefix keeps the two sides' streams apart, the way the ID's low bit does, so -the dialer's first bi stream and the listener's are two streams. The number -keeps one side's opens apart, so a track name always means one stream: a -handle the peer kept to an earlier stream never reads a later one. A track's -info, priority included, is the opener's to choose. - -The peer learns of new streams from the broadcast's track listing (the Tracks -stream, `TRACKS_ADD`). A listing reports every track the broadcast holds, so -accept skips any name outside the scheme. - -A channel both sides know in advance, such as a viewer's `command` track or the -listener's acks back to it, isn't a stream. It's an ordinary track with a fixed -name in the socket's broadcast: the writer creates it in its own half, and the -reader subscribes to it by name in the peer's half, with no accept. Streams are -for what an application opens at run time, such as one per request or per -byte-stream connection, with any framing of its own. - -A stream can be accepted only while its opener lists it, which holds until the -opener drops the track, finished or not. One dropped before the peer accepts it -is never handed out, so a backlog of unaccepted streams costs the peer no more -than the tracks the opener keeps. - -## Ending - -A socket ends when either side's broadcast ends, which includes either side -closing it or a session on the way dropping, or when a newer dial of its id -replaces it. Each side then drops its socket, which retracts its broadcast and -ends the socket for the peer. A side that closed opens and accepts nothing -more, and its pending bi opens fail, even when the peer answers them later. - -## Versions - -Sockets need moq-lite 07 (`moq-lite-07-wip`, opt-in) on every hop: older -moq-lite versions and moq-transport carry neither instances nor track -listings. Over an older hop, accepting a stream fails with `Unsupported`, and -a dial never resolves, because the answer no longer carries the dial's -instance. +A multiplexed mode, with many streams per session like QUIC, would be its own +endpoint, `/.quic/`, beside `.tcp`, and could name its streams +beneath the session, such as a control stream under `//`. A +`.tcp` session name holds no `/` after the session id, so it leaves that room. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 1a1fc08693e..d7cad9f07e9 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Listener::new(&origin, path)` accepts the sockets that `socket::dial(&origin, path, id)` opens, and either side of a `Socket` opens and accepts unnamed uni and bi streams as tracks, the way QUIC does (`open_uni`, `open_bi`, `accept_uni`, `accept_bi`), or reads well-known channels from the peer's broadcast by fixed track name. A dial waits until accepted; drop it to give up. Needs moq-lite 07 (`moq-lite-07-wip`, opt-in) on every hop; see [sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.tcp/`. `dial(peer, info)` opens a session to another endpoint, and `accept()` yields the dials to this one, each a `socket::Stream`: this side's `track::Producer` and a reliable, ascending `track::Subscriber` of the peer's half, one message per group. Works over every moq-lite version and moq-transport; see [sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index 75c2802c972..e5a23a445d9 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -2,108 +2,123 @@ ## Goal -`moq-net` gains a generic dial/accept primitive. A `Listener` at a path accepts -the `Socket`s that dialers announce. Inside a socket either side opens and -accepts unnamed streams, as tracks, the way QUIC does. moq-boy, CARLA, -[byte streams](/quest/m2/stream/README.md), and the mesh -[Endpoint](/quest/m2/mesh/endpoint.md) all build on it instead of each +`moq-net` gains a generic dial/accept primitive. An `Endpoint` announces one +hidden broadcast, dials other endpoints, and accepts their dials, and each +connection is a session: one reliable, ordered pipe of messages between the +two ends. moq-boy, CARLA, [byte streams](/quest/m2/stream/README.md), and the +mesh [Endpoint](/quest/m2/mesh/endpoint.md) all build on it instead of each hand-rolling the announce, watch, and request loop. Rust, JS, and Python (through moq-ffi) expose the same surface and interoperate. Non-goals: a wire change to moq-lite (a socket is a path and track convention -over existing announce, Tracks, and subscribe streams); session caps, leases, -or idle timeouts, which stay application logic (boy's 60 s idle, CARLA's -one-driver lease and 1 s watchdog); announcement backpressure, which -[announce credit](/quest/m2/announce-credit.md) owns. +over existing announce and subscribe streams); session caps, leases, or idle +timeouts, which stay application logic (boy's 60 s idle, CARLA's one-driver +lease and 1 s watchdog); signing and encryption, which the m2 +[mesh](/quest/m2/mesh/README.md) owns by making endpoint ids public keys. ## Plan -Decided 2026-11-03 (maintainer, /quest-complete), on -[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677): - -- Copy QUIC's API: socket streams have no application name. Tracks are - `.//`, with `` `dial` or `accept`, `` - `uni` or `bi`, and `` the opener's never-reused counter. The surface - is quinn's: `open_uni`, `open_bi`, `accept_uni`, `accept_bi`, each taking - the track info an open needs. Why: no consumer needs the name. Well-known - channels (boy's `command`; CARLA's `control`, ack, and latency) are ordinary - fixed-name tracks on the socket's broadcasts, and Kixel T3 runs byte - streams with its own framing. Not chosen: a hidden number plus an app name, - or making name uniqueness the app's job. -- Auth uses moq.pro-style tokens with prefix grants only, rooted at the - listener's path `L`: the listener subscribes `.dial/**` and publishes - `.accept/**`, and a dialer publishes `.dial/` and subscribes - `.accept/`. No mid-path wildcard is needed; it's the shape of moq.pro's - voice demo (`request/`, `response/`). A relay test with real - `moq-auth` tokens covers it, and `doc/concept/socket.md` shows the tokens. +[`doc/concept/socket.md`](/doc/concept/socket.md) is the convention JS and +Python follow, and `rs/moq-net/src/model/socket.rs` implements it. + +Decided 2026-11-03 (maintainer, /quest-grill in /quest-complete), on +[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677), replacing the +listener layout and QUIC-style streams below: + +- Endpoint sockets, one pipe per session. Each endpoint announces one hidden + broadcast, `/.tcp/`, once, and connections are never announced. + `` is a well-known name for a server (a game) or a random id for a + client. Why: the old flow had the listener answer `L/.accept/` under the + dialer's instance, which broke instance uniqueness (lite-07 treats routes + sharing an instance as replicas), let two listeners accept one dial, + announced every connection publicly, and needed lite-07 instances and track + listings on every hop. +- The segment is `.tcp`, not `.socket`, so the path names the protocol: one + reliable, ordered pipe per session. A multiplexed mode would be its own + `.quic/` endpoint later. The Rust and JS module stays `socket` + (`socket::Endpoint`, `socket::Stream`), since `.tcp` and a later `.quic` are + modes of one socket API. +- Dial: `A` picks a random, unguessable session id `S` (128 bits) and + subscribes to `.tcp/`; `B` surfaces the requested track as an incoming + dial claiming to be from `A`. Accept: `B` subscribes to `.tcp/`, and `A` + answers only a session it dialed and resets any other, a return-routability + check that fails a dial forged in `A`'s name. `B` sends nothing until `A` + answers. Reject: `B` resets `A`'s subscription. +- The pipe is one track each way. One message per group, so messages don't + block each other, ordered by group sequence; reliable and in order by + default (no delay budget, ascending), and the app may opt into a lossy + budget. Finishing a track closes the pipe, resetting it is an error, and the + endpoint's broadcast ending disconnects. +- No multiplexing: another channel is another session, a pair of endpoints + holds several, and a restart is a new session. No control track and no + numbered streams now. +- Session track names are the secret, so they never appear in a lite-07 track + listing or anywhere else a third party reads. moq-net already lists only + tracks a producer creates or a handler's `Lister` answers, never one served + on request, so an endpoint serves its session tracks only on request. The + dialer's own half is reserved unlisted + (`broadcast::Producer::reserve_unlisted_track`, crate-private). +- Grants are prefix only, rooted at the prefix: publish `.tcp/`, + subscribe `.tcp/**`, or narrower subscribes. A relay test signs real + `moq-auth` tokens. +- It works over every moq-lite version and moq-transport, since the handshake + uses no instances or listings; lite-07 instances only affect relay route + choice. Tests cross moq-transport hops. +- Fixed channels become sessions or messages on one: boy's `command` and + per-viewer latency, CARLA's `control`, ack, and e-stop. Shared state such as + boy's `status` stays an ordinary broadcast. +- `socket::Stream` is a plain struct with public `producer` (this side's half) + and `consumer` (the peer's half) fields, plus read-only `peer()` and + `session()`, and no read, write, or message methods of its own: apps use the + track API directly (groups as messages, priority, budget), so JS, Python, and + the bindings mirror a minimal surface. + +Settled by the Rust implementation of that decision, for the maintainer to +confirm (see the PR's Waiting section): + +- Both halves are named `/`, not each after its reader. With + reader-named halves, an endpoint that both dials and accepts cannot tell a + peer's dial from a peer's answer, so it could not reset an answer to a + session it never dialed: a forged dial surfaced at the claimed dialer as a + dial from the victim and deadlocked both accepts. With the dialer's id + leading both names, a request under an endpoint's own id is always an answer. +- `Stream::consumer` is a `track::Subscriber`, already subscribed reliably and + in order, not a bare `track::Consumer`, whose default subscription is lossy + and newest first. The handshake's subscription becomes the stream's, so no + message falls between the accept and the app's first read; `update` makes it + lossy. +- An endpoint takes dials from its first `accept` on, like a TCP listener; until + then it has no request handler, so a dial to it is reset at once. A dial + reserves its own half, which the peer's answer finds without a handler, so a + dial-only endpoint never needs polling. A dial waits for the peer's endpoint + to be announced (drop it to give up); an accept fails at once when the + claimed dialer's endpoint is not announced. +- Accepted dials queue without limit while the app accepts, each holding the + dialer's subscription open. Announce backpressure is no longer a socket + concern, since connections are never announced. Decided 2026-11-02 (maintainer, Kixel planning): the whole epic moves from m2 to m1, so the next cut publishes sockets in `@moq/net` and moq.pro's relays carry them. Why: Kixel's T3 Code prototype ([moq-dev/kixel.ai T3 epic](https://github.com/moq-dev/kixel.ai/blob/main/quest/m0/t3/README.md)) sends T3's commands over sockets, one group per message, in place of byte -streams. `@moq/net` publishes from `release`, and accept relies on lite-07's -Tracks stream, which only `main` has, so a backport was declined and the cut -is the route. Byte streams, fastest-reader pacing, and the mesh libraries -keep their own milestones. +streams. `@moq/net` publishes from `release`. Byte streams, fastest-reader +pacing, and the mesh libraries keep their own milestones. Decided 2026-11-01 with the maintainer (planning interview from moq.pro's TCP -replacement arc): - -- Name and home: `Listener`, `dial`, and `Socket` in a `socket` module of - `moq-net` (`moq_net::socket`), the `socket` export of `@moq/net`, and - moq-ffi beside origin. Not a separate crate: it uses only the origin, - broadcast, and track APIs every binding already wraps. `Session` is taken by - the transport connection. -- Layout is owned by the listener and hidden. A listener at `L` keeps `L` for - whatever it shares with everyone (boy's game, CARLA's cameras), so fanout - is unchanged. A dialer announces `L/.dial/` and the listener answers on - `L/.accept/`. Discovery is a one-segment watch of `L/.dial/*`, which is - cheap for relays, unlike a globstar over the whole origin. Auth enforces the - id with prefix grants rooted at `L` (see the 2026-11-03 decision). The dot - keeps sessions out of ordinary discovery - ([hidden broadcasts](/doc/concept/moq-lite.md#hidden-broadcasts)), so boy's - game list never sees viewers. -- Identity is a dialer-chosen id plus a pinned instance (the epic first - called it an epoch; the code's term is the route's `Instance`). The id is the - dialer's own: an endpoint ID in the mesh, a random id in the demos. Each dial - instance is one socket: a new instance ends the old socket and starts a new - one, and the listener's request is pinned to the instance - (`origin::Consumer::request` with an owned `Instance`, as CARLA already - does), so a restarted dialer never inherits state. A socket ends when either - side's broadcast ends or its instance is replaced. Settled by the Rust - implementation: the listener's answer carries the dial's instance, and the - dialer follows its answer pinned to it, so it never takes an answer to an - older dial of its id. A dial waits indefinitely; callers bound it by dropping - it. -- Streams are tracks. A uni stream is a track the opener publishes in its own - socket broadcast. A bi stream is the same track name in both. The peer - learns of new tracks from the Lite07 Tracks stream (`TRACKS_ADD`, `REMOVE`, - `FINISH`), which pushes them like QUIC's accept. `requested_track` was - declined because it is pull-shaped: only the side being subscribed to can - accept. Settled by the Rust implementation and the 2026-11-03 decision: - tracks are named `.//` with `` `dial` or - `accept`, so the two sides' streams never collide, and `` counting - the opener's streams, so a track never carries two streams; accept skips - names outside the scheme, so fixed track names coexist; the opener chooses the track info (priority included) - and a bi answer mirrors it. [`doc/concept/socket.md`](/doc/concept/socket.md) - is the convention JS and Python follow. -- Accept is unbounded and lazy. Announced dialers queue without limit, and a - dialer's broadcast is requested only when the application accepts it, so an - application that stops accepting holds no subscriptions. Backpressure on the - announcements themselves (paced by the fastest consumer, made hard by merging - overlapping prefixes) is a separate design, settled as wire-level - [announce credit](/quest/m2/announce-credit.md). +replacement arc), still standing: + +- Home: a `socket` module of `moq-net` (`moq_net::socket`), the `socket` + export of `@moq/net`, and moq-ffi beside origin. Not a separate crate: it + uses only the origin, broadcast, and track APIs every binding already wraps. - The convention is specified on a doc.moq.dev concept page, - `doc/concept/socket.md`, written with the Rust implementation. It is not a - section of the moq-lite draft or a new draft: it is an application + `doc/concept/socket.md`, not in the moq-lite draft: it is an application convention over moq-lite. -- The demos move per-client channels onto the socket as fixed-name tracks on - its broadcasts, not streams: boy's `command` and CARLA's `control` on the - dialer's `.dial` broadcast, and boy's per-viewer latency and CARLA's driver - ack and sequence on the listener's `.accept` broadcast. State every viewer - needs (the game, who is driving, who holds the - e-stop) stays in the shared broadcast at `L`. + +Later, not built now: a multiplexed `.quic/` mode with many streams per +session, which could name a control stream and numbered streams beneath the +session (`//...`); and signed or encrypted sessions with ids +as public keys, in the m2 [mesh](/quest/m2/mesh/README.md). This epic owns removing the hand-rolled loops: after the ports, no announce-watch-request pairing remains in `rs/moq-boy`, `js/moq-boy`, or @@ -113,10 +128,10 @@ announce-watch-request pairing remains in `rs/moq-boy`, `js/moq-boy`, or - [JS sockets](/quest/m1/socket/js.md) - the same surface in `@moq/net`, interoperating with Rust - [Python sockets](/quest/m1/socket/py.md) - moq-ffi and `py/moq-rs` bindings, interoperating with Rust -- [moq-boy on sockets](/quest/m1/socket/boy.md) - the emulator accepts viewers through a listener and echoes latency per socket -- [CARLA on sockets](/quest/m1/socket/carla.md) - trucks accept drivers through a listener and ack them per socket +- [moq-boy on sockets](/quest/m1/socket/boy.md) - viewers dial the emulator's endpoint and send commands and receive latency on their session +- [CARLA on sockets](/quest/m1/socket/carla.md) - drivers dial the truck's endpoint and send control and receive acks on their session ## Related -- [Byte streams](/quest/m2/stream/README.md) - TCP over sockets, one bi track per connection -- [Endpoint API](/quest/m2/mesh/endpoint.md) - the mesh's `connect`/`accept`, a socket plus signing +- [Byte streams](/quest/m2/stream/README.md) - TCP over sockets, one session per connection +- [Endpoint API](/quest/m2/mesh/endpoint.md) - the mesh's `connect`/`accept`, a socket endpoint plus signing diff --git a/quest/m1/socket/boy.md b/quest/m1/socket/boy.md index c375e3beab6..6460b1d50c4 100644 --- a/quest/m1/socket/boy.md +++ b/quest/m1/socket/boy.md @@ -2,29 +2,33 @@ ## Goal -The moq-boy emulator accepts viewers through a `Listener` at its game path, -and each viewer dials it, so neither `rs/moq-boy` nor `js/moq-boy` watches -announcements or requests broadcasts by hand. Each viewer's latency comes back -on its own socket instead of the shared `status` map, and crowd play, pausing -when nobody watches, and the 60 s idle stop behave as before. +The moq-boy emulator accepts viewers on a socket endpoint named for its game, +and each viewer dials it from its own endpoint, so neither `rs/moq-boy` nor +`js/moq-boy` watches announcements or requests broadcasts by hand. Each +viewer's commands and latency travel on its own session instead of a viewer +broadcast and the shared `status` map, and crowd play, pausing when nobody +watches, and the 60 s idle stop behave as before. ## Plan -See the [epic](/quest/m1/socket/README.md) for the layout and the reply split. +See the [epic](/quest/m1/socket/README.md) for the convention. -- Viewers dial `boy/game/` with their random id. Their `command` track - moves from `{viewerPrefix}/{name}/{viewerId}` to the socket, as a fixed - track named `command` on the viewer's `.dial` broadcast (`Socket::local`), - which the emulator reads by name from `Socket::remote`. It isn't a socket - stream: streams are unnamed and for what opens at run time. Drop the +- The emulator runs an endpoint named for its game, and each viewer an + endpoint with its random id that dials it. Settle and record the prefix + (such as `boy`, so endpoints sit at `boy/.tcp/`). +- The viewer's `command` track at `{viewerPrefix}/{name}/{viewerId}` becomes + the viewer's messages on its session, one command per group. Drop the separate viewer-prefix configuration in `rs/moq-boy/src/main.rs`. -- Per-viewer latency moves from `status`'s `latency` map to a fixed track on - that viewer's `.accept` broadcast. Game state stays in `status`. -- Requests become pinned to the viewer's epoch (boy requests unpinned today), - so a reload is a new socket. -- The demo relay's anonymous publish grant (`demo/relay/prod.toml`, and the - `moq auth serve` line it documents) covers `demo/boy/game/*/.dial/*` - instead of boy's use of `demo/viewer/**`. Whichever of this and +- Per-viewer latency moves from `status`'s `latency` map to the emulator's + messages on that viewer's session. Game state stays in the ordinary + `status` broadcast. +- A reload is a new session; the emulator counts a viewer as gone when its + session ends. +- The demo relay's anonymous grants (`demo/relay/prod.toml`, and the + `moq auth serve` line it documents) let an anonymous viewer publish its own + endpoint and subscribe to the game endpoints, instead of boy's use of + `demo/viewer/**`. Anonymous viewers share one grant, so record how a viewer's + endpoint id is kept from being claimed by another. Whichever of this and [CARLA on sockets](/quest/m1/socket/carla.md) lands second removes `demo/viewer/**`. - Update `doc/bin/demo.md` and the boy README or CHANGELOG where they describe diff --git a/quest/m1/socket/carla.md b/quest/m1/socket/carla.md index 486cbe9d7f0..97385e996e2 100644 --- a/quest/m1/socket/carla.md +++ b/quest/m1/socket/carla.md @@ -2,37 +2,37 @@ ## Goal -Each CARLA truck accepts drivers through a `Listener` at `carla/`, and -the web driver dials it, so `demo/carla/sim` and `demo/carla/web` no longer -watch `viewer/carla//` or request driver broadcasts by hand. Each -driver's ack and sequence echo come back on its own socket. The one-driver -lease, takeover, e-stop, the 1 s watchdog, and handback behave as before. +Each CARLA truck runs a socket endpoint named for the truck and accepts +drivers on it, and the web driver dials it from its own endpoint, so +`demo/carla/sim` and `demo/carla/web` no longer watch `viewer/carla//` +or request driver broadcasts by hand. Each driver's control, ack, sequence +echo, and e-stop travel on its own sessions. The one-driver lease, takeover, +e-stop, the 1 s watchdog, and handback behave as before. ## Plan -See the [epic](/quest/m1/socket/README.md) for the layout and the reply split. +See the [epic](/quest/m1/socket/README.md) for the convention. -- The worker (`publish.py`) uses the Python `Listener`; `paths.ts` and - `truck.ts` dial `carla/` with the per-lease driver id. A new lease - is a new epoch and so a new socket, which matches today's lease binding. -- The `control` track moves onto the socket as a fixed track named `control` - on the driver's `.dial` broadcast, not a socket stream (streams are unnamed - and for what opens at run time). The driver's ack, sequence, and round-trip - echo move from the shared `status` to fixed tracks on the driver's `.accept` - broadcast. `driver` and `estop_by` stay in `status`, because every viewer - shows them. -- Decide in the PR whether the `safety-` e-stop broadcast becomes its own - socket or a second track on the driver's socket, and record why. -- The anonymous publish grant covers `demo/carla/*/.dial/*`. Whichever of - this and [moq-boy on sockets](/quest/m1/socket/boy.md) lands second removes +- The worker (`publish.py`) runs a Python endpoint per truck; `paths.ts` and + `truck.ts` dial it from an endpoint with the per-lease driver id. A new lease + is a new session, which matches today's lease binding. +- The `control` track becomes the driver's messages on its session. The + driver's ack, sequence, and round-trip echo move from the shared `status` to + the truck's messages on that session. `driver` and `estop_by` stay in the + ordinary `status` broadcast, because every viewer shows them. +- The `safety-` e-stop broadcast becomes messages on the driver's session + or a second session of its own; decide in the PR and record why, such as + keeping the e-stop off the control session's queue. +- The anonymous grants let a driver publish its own endpoint and subscribe to + the truck endpoints. Whichever of this and + [moq-boy on sockets](/quest/m1/socket/boy.md) lands second removes `demo/viewer/**` from `demo/relay/prod.toml` and its documented - `moq auth serve` line. The worker's token keeps publish on `carla/**` - and swaps its subscribe scope from `viewer/carla/**` to `carla/*/.dial/*`, - because publish and subscribe are granted separately; say so in - `demo/carla/README.md`. + `moq auth serve` line. The worker's token keeps publish on `carla/**`, which + covers its endpoints, and swaps its subscribe scope from `viewer/carla/**` + to the driver endpoints; say so in `demo/carla/README.md`. - Update `demo/carla/README.md` (paths, lease) and `doc/bin/demo.md`. ## Required -- [Python sockets](/quest/m1/socket/py.md) - the truck's listener +- [Python sockets](/quest/m1/socket/py.md) - the truck's endpoint - [JS sockets](/quest/m1/socket/js.md) - the driver's dial diff --git a/quest/m1/socket/js.md b/quest/m1/socket/js.md index ca8bf40c225..a0de86d5c15 100644 --- a/quest/m1/socket/js.md +++ b/quest/m1/socket/js.md @@ -2,16 +2,18 @@ ## Goal -`@moq/net` exports `socket` with the same `Listener`, `dial`, and `Socket` -surface as Rust, and a Rust dialer and a JS listener (and the reverse) open -and accept uni and bi streams through a relay. +`@moq/net` exports `socket` with the same `Endpoint` and `Stream` surface as +Rust, and a Rust endpoint and a JS endpoint dial and accept sessions with each +other through a relay, carrying messages both ways. ## Plan -Follow `doc/concept/socket.md` and mirror the -Rust surface (`rs/moq-net/src/model/socket.rs`), including its track-naming -scheme: streams are unnamed, like QUIC's (`openUni`, `openBi`, `acceptUni`, -`acceptBi`, in JS's casing), and well-known channels are fixed-name tracks on -the socket's `local` and `remote` broadcasts. -Add the Rust/JS pair to `just test interop`, and document the export in +Follow `doc/concept/socket.md` and mirror the Rust surface +(`rs/moq-net/src/model/socket.rs`): an `Endpoint` announcing +`/.tcp/`, `dial(peer)` and `accept()` in JS's casing, an incoming +dial to accept or reject, and a `Stream` with `producer` and `consumer` fields +plus `peer` and `session`, and no message methods of its own. Session tracks +are `/` in both endpoints, served only on request so no +listing names them, and the reader subscribes reliably and in order. Add the +Rust/JS pair to `just test interop`, and document the export in `doc/lib/js/net.md`. diff --git a/quest/m1/socket/py.md b/quest/m1/socket/py.md index b8706900cb2..ffdbb045289 100644 --- a/quest/m1/socket/py.md +++ b/quest/m1/socket/py.md @@ -2,14 +2,15 @@ ## Goal -moq-ffi exposes `Listener`, `dial`, and `Socket` beside origin, and -`py/moq-rs` wraps them, so a Python listener accepts a Rust dialer's socket -and the reverse, with uni and bi streams each way. +moq-ffi exposes the socket `Endpoint` and `Stream` beside origin, and +`py/moq-rs` wraps them, so a Python endpoint accepts a Rust endpoint's dial and +the reverse, carrying messages both ways. ## Plan -Bind the Rust surface (`moq_net::socket`), whose streams are unnamed like -QUIC's (`open_uni`, `open_bi`, `accept_uni`, `accept_bi`), through moq-ffi the way -origin's request and announce APIs are bound today (`rs/moq-ffi/src/origin.rs`, +Bind the Rust surface (`moq_net::socket`): `Endpoint::new`, `dial`, `accept`, +an incoming dial's `accept` and `reject`, and `Stream`'s `producer`, +`consumer`, `peer`, and `session`, through moq-ffi the way origin's request +and announce APIs are bound today (`rs/moq-ffi/src/origin.rs`, `py/moq-rs/moq/origin.py`). Add a Python/Rust interop test, and document the API wherever `py/moq-rs` documents origin. diff --git a/quest/m2/announce-credit.md b/quest/m2/announce-credit.md index 7ba0159a040..751173060a5 100644 --- a/quest/m2/announce-credit.md +++ b/quest/m2/announce-credit.md @@ -7,7 +7,7 @@ the wire: ANNOUNCE_REQUEST carries an initial route credit, the subscriber raises it later on its side of the stream, and the publisher never holds more live advertisements on the stream than the credit. A peer that floods distinct paths is held to the credit instead of closing the session at the 100k -announce cap, so a flood of `.dial/*` sockets cannot drop a cluster link or the +announce cap, so a flood of distinct paths cannot drop a cluster link or the other announcements it carries. Rust and JS implement it on the next unpublished lite version after lite-07. @@ -30,8 +30,8 @@ Decided 2026-11-01 with the maintainer (the follow-up to the budget. - Declined: per-route refusal (frees the subscriber but not the publisher, and the flood still crosses the wire once), and aggregating to a covering - prefix at the limit (loses the per-path inventory a socket `Listener` - needs). + prefix at the limit (loses the per-path inventory a reader watching many + broadcasts needs). - The publisher withholds routes beyond the credit, coalesced to the latest state per path as it already does, and sends them as an ANNOUNCE_END or a credit raise frees room. Which withheld route goes first is the diff --git a/quest/m2/mesh/README.md b/quest/m2/mesh/README.md index 993a27673cb..50020e7b274 100644 --- a/quest/m2/mesh/README.md +++ b/quest/m2/mesh/README.md @@ -52,9 +52,9 @@ Decided 2026-10-25 with the maintainer (planning interview), still in force: reaching an ID is subscribing there. There is no separate directory. - The token is the binding: enrollment proves the device key, then the coordinator issues a grant to publish `mesh//**` and subscribe - `mesh/**`, plus publish on `mesh/*/*/.dial/` so the node can dial - another's [socket](/quest/m1/socket/README.md) under its own id (amended - 2026-11-01). Relays need no new check. Membership is the coordinator's + `mesh/**`, plus publish on its own [socket](/quest/m1/socket/README.md) + endpoint, `.tcp/`, so it can dial and accept under its own id (amended + 2026-11-01, and 2026-11-03 when sockets became endpoints). Relays need no new check. Membership is the coordinator's token only; a Tailnet-Lock analog was declined. - Protection is per broadcast, with levels that nest: identity (the token only), authentication (per-frame Ed25519 signatures, the default), and diff --git a/quest/m2/mesh/endpoint.md b/quest/m2/mesh/endpoint.md index 3a4cb9c3aee..d4126196db2 100644 --- a/quest/m2/mesh/endpoint.md +++ b/quest/m2/mesh/endpoint.md @@ -15,17 +15,20 @@ Decided 2026-10-25 (maintainer): iroh-shaped, not drop-in compatible; tickets, `EndpointAddr`, and `Router` are out unless porting proves they are needed. - A service is the iroh ALPN analog: a name under `mesh//`. -- Decided 2026-11-01 (maintainer, socket planning interview): an `Endpoint` - wraps [sockets](/doc/concept/socket.md). A service is a `Listener` at - `mesh//`, `connect` dials it with the caller's own endpoint ID, - and `accept` reads the remote endpoint ID from the `.dial/` segment, - which the grant pins. Extend `moq auth`'s mesh grant with publish on - `mesh/*/*/.dial/`, and update `doc/bin/relay/auth.md`. +- Decided 2026-11-01 (maintainer, socket planning interview), and updated + 2026-11-03 when sockets became endpoints: an `Endpoint` wraps a + [socket](/doc/concept/socket.md) endpoint. Settle the layout here: such as + one socket endpoint per service at `mesh//.tcp/`, or one per + node with the service named in the session. `connect` dials with the + caller's own endpoint ID, and `accept` reads the remote ID from the dial, + which the socket's answer check proves and the grant pins (publish + `.tcp/` only). Update `moq auth`'s mesh grant and + `doc/bin/relay/auth.md` to match. - Streams are [byte streams](/quest/m2/stream/rust.md) at the protection level the service declares: identity reads `moq-net` directly, and authentication, the mesh default, reads through `moq_crypto::sign` against the - `PublicKey` parsed from the path it subscribes to: the listener's - `mesh//` for a dialer, and the `.dial/` segment for a listener. + `PublicKey` parsed from the peer's endpoint id, which names the endpoint + broadcast it subscribes to. - The secret key comes from `moq_crypto::sign::SecretKey::load_or_generate`. - A mesh daemon (Kixel's, outside this repository, decided 2026-11-02) is built on this API, so a process either embeds an `Endpoint` or talks to a diff --git a/quest/m2/stream/README.md b/quest/m2/stream/README.md index 7cbbcc25796..f789f8fef4d 100644 --- a/quest/m2/stream/README.md +++ b/quest/m2/stream/README.md @@ -17,9 +17,9 @@ fastest-reader pacing gives. Decided 2026-10-25 with the maintainer (mesh planning interview): - TCP-level proxying over groups, never IP over datagrams. -- A forwarded service is one [socket](/quest/m1/socket/README.md), and each - TCP connection is one bi track on it (amended 2026-11-01, below), one group - each way; each TCP `read()` becomes one frame, because a frame's length must +- A forwarded service is one [socket](/quest/m1/socket/README.md) endpoint, + and each TCP connection is one session on it (amended 2026-11-03, below), + one group each way; each TCP `read()` becomes one frame, because a frame's length must be known before it is written and a read has one. It coalesces whatever is buffered with no added timer (the local TCP socket's own Nagle or `TCP_NODELAY` still applies). @@ -39,13 +39,16 @@ Decided 2026-10-25 with the maintainer (mesh planning interview): identity, signing, and service names belong to the [mesh](/quest/m2/mesh/README.md), which builds on this. -Amended 2026-11-01 with the maintainer (socket planning interview): connection -setup is the [socket](/quest/m1/socket/README.md) epic's `Listener` and `dial`, -which settles the path layout (`L/.dial/`, `L/.accept/`). The earlier -one track per direction with a group per connection is replaced by one bi -track per connection. Each connection then gets its own priority, pacing, and -eviction, and a long-lived connection never sits in an old group that newer -ones crowd out of a track's window. +Amended 2026-11-01 with the maintainer (socket planning interview), and +2026-11-03 when sockets became endpoints: connection setup is the +[socket](/quest/m1/socket/README.md) epic's `Endpoint`, `dial`, and `accept`, +which settle the layout (`/.tcp/`, a `/` track +each way). Each TCP connection is one session, so it gets its own priority, +pacing, and eviction, and a long-lived connection never sits in an old group +that newer ones crowd out of a track's window. A session carries messages one +group each; a byte stream instead holds one group each way for the +connection's life, which the stream quests settle against the socket's +reliable reader. This epic owns the end-to-end proof: an unmodified TCP client and server (SSH and an HTTP bulk transfer) talk through a two-relay chain with loss injected on diff --git a/quest/m2/stream/rust.md b/quest/m2/stream/rust.md index a053e883b36..fb7aa9c9a75 100644 --- a/quest/m2/stream/rust.md +++ b/quest/m2/stream/rust.md @@ -2,22 +2,23 @@ ## Goal -A Rust application opens a duplex byte stream on a -[socket](/doc/concept/socket.md) and gets an `AsyncRead + AsyncWrite` that -carries TCP semantics over one bi track: ordered, -lossless, FIN as half-close, reset with a code. A socket's listener accepts the streams -its dialers open. `moq` gains TCP forwarding on both ends so an +A Rust application opens a duplex byte stream as a +[socket](/doc/concept/socket.md) session and gets an `AsyncRead + AsyncWrite` +that carries TCP semantics over the session's two tracks: ordered, lossless, +FIN as half-close, reset with a code. A socket endpoint accepts the streams +other endpoints dial. `moq` gains TCP forwarding on both ends so an unmodified client and server talk through a relay. ## Plan -See the [epic](/quest/m2/stream/README.md) for the mapping: a socket per -forwarded service, a bi track per connection with one group each way, a frame -per TCP read, no fill handler. +See the [epic](/quest/m2/stream/README.md) for the mapping: a socket endpoint +per forwarded service, a session per connection with one group each way, a +frame per TCP read, no fill handler. - Build on `moq_net::socket`: the path layout and connection naming are the - socket's (`open_bi` and `accept_bi`), so this quest adds only the - `AsyncRead`/`AsyncWrite` adapter and the TCP mapping. Place the adapter + socket's (`Endpoint::dial` and `accept`, each yielding a `Stream`), so this + quest adds only the `AsyncRead`/`AsyncWrite` adapter over a `Stream` and the + TCP mapping. Place the adapter beside the socket module unless it pulls in a dependency moq-net avoids; report the choice. - Each stream carries a subscription priority the caller sets; nothing is diff --git a/rs/moq-net/CHANGELOG.md b/rs/moq-net/CHANGELOG.md index 7e32e7ae11e..fb7fed2f450 100644 --- a/rs/moq-net/CHANGELOG.md +++ b/rs/moq-net/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `socket`: `Listener`, `dial`, and `Socket`, point-to-point connections over an origin whose streams are unnamed tracks, opened and accepted like QUIC's; see `doc/concept/socket.md`. +- `socket`: `Endpoint`, which announces a hidden `/.tcp/` broadcast, dials other endpoints, and accepts their dials, each session a `Stream` of two unlisted tracks carrying one message per group; see `doc/concept/socket.md`. - `transport::SETUP_RECEIVE_WINDOW`, the connection credit a server offers before it admits a session: enough for the SETUP and one in-band AUTH. ## [0.3.10](https://github.com/moq-dev/moq/compare/moq-net-v0.3.9...moq-net-v0.3.10) - 2026-10-05 diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index dd10680984e..ca5e3b502d9 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -460,6 +460,15 @@ impl Producer { })) } + /// [`Self::reserve_track`] for a track no listing ever names, so only a reader who + /// already knows the name finds it, as a socket session's track. + pub(super) fn reserve_unlisted_track(&self, name: impl Into>) -> Result { + let mut state = self.state.lock(); + let request = track::Request::new(self.info.clone(), name.into()).with_stats(self.stats.clone()); + state.insert_track(request.weak())?; + Ok(request) + } + /// Create a track with a unique name using the given suffix. /// /// Uses [`Self::unique_name`]; minted names are never reused, even after closure. diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 14a7080d753..de18db4893c 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -1,788 +1,586 @@ -//! Point-to-point sockets over an origin: a [`Listener`] accepts the [`Socket`]s that -//! [`dial`]ers open to its path. +//! Point-to-point sockets over an origin, the way TCP connects two hosts. //! -//! A socket is a convention over announcements and broadcasts, with no wire of its own. -//! A listener at `L` keeps `L` for whatever it shares with everyone. A dialer publishes -//! its half at `L/.dial/`, and the listener answers with its half at `L/.accept/` -//! once the application accepts. The answer carries the dial's [`Instance`], so a -//! dialer that dials again with the same id gets a new socket rather than an old answer. +//! An [`Endpoint`] announces one hidden broadcast, `/.tcp/`, and from it +//! [`dial`](Endpoint::dial)s other endpoints and [`accept`](Endpoint::accept)s their +//! dials. Each connection is a session: one [`Stream`], a reliable, ordered pipe made of +//! two tracks, one in each endpoint's broadcast. Connections are never announced. //! -//! Inside a socket either side opens and accepts streams the way QUIC does, and like -//! QUIC's they carry no name: a stream is told apart by who opened it and its number. A -//! uni stream is one track in the opener's broadcast, and a bi stream is the same track -//! name in both. Each open takes the next number of its opener, so a track never carries -//! two streams. The peer learns of new streams from the broadcast's track listing, which -//! needs moq-lite-07 on every hop. Channels both sides know in advance are ordinary -//! fixed-name tracks on [`Socket::local`] and [`Socket::remote`]. See -//! `doc/concept/socket.md` for the convention. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Weak}; -use std::task::{Poll, ready}; +//! A dialer `A` picks a random session id `S` and subscribes to `/.tcp/` for +//! the track `A/S`. Endpoint `B` surfaces that request as an [`Incoming`] dial from `A`. +//! To accept it, `B` subscribes to `/.tcp/` for the same track name, and `A` +//! answers only a session it dialed, so the exchange checks that `A` really asked: a +//! dial forged in `A`'s name is reset. Each side then writes its own half and reads the +//! peer's. Session tracks are served only on request, so no track listing names them. +//! +//! See `doc/concept/socket.md` for the convention other languages follow. -use hashlink::{LinkedHashMap, LinkedHashSet}; +use std::collections::VecDeque; +use std::sync::Arc; +use std::task::{Poll, ready}; -use crate::{Error, Instance, Path, Pattern, Patterns}; +use crate::{Error, Path}; -use super::broadcast::{self, TracksEvent}; +use super::WeakEntry; use super::origin::{self, Route}; -use super::{announce, track}; - -/// The segment beneath a listener's path that dialers publish under. -const DIAL: &str = ".dial"; -/// The segment beneath a listener's path that the listener answers under. -const ACCEPT: &str = ".accept"; - -/// Which end of a socket a side is. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Role { - Dialer, - Listener, -} +use super::track::{self, Delay, Order, Subscription}; -impl Role { - /// The tag naming the streams this role opens. - fn tag(self) -> &'static str { - match self { - Self::Dialer => "dial", - Self::Listener => "accept", - } - } -} +/// The segment an endpoint's broadcast sits under: one reliable, ordered pipe per +/// session, as TCP gives. A multiplexed mode would take a segment of its own. +const TCP: &str = ".tcp"; -/// Whether a stream carries data one way or both. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Kind { - Uni, - Bi, +/// Refuse an id that is not one plain path segment: empty, or holding a `/` or a `\`. +fn check_id(id: &str) -> Result<(), Error> { + if id.is_empty() || id.contains(['/', '\\']) { + return Err(crate::InvalidPattern::InvalidSegment(id.to_string()).into()); + } + Ok(()) } -impl Kind { - fn tag(self) -> &'static str { - match self { - Self::Uni => "uni", - Self::Bi => "bi", - } - } +/// `/.tcp/`, the broadcast of the endpoint `id`. +fn endpoint_path(prefix: &Path, id: &str) -> Result { + check_id(id)?; + Ok(prefix.join(&Path::from_static(TCP))?.join(&Path::new(id)?)?) } -/// The track name of the opener's `number`th stream, a `kind` one: -/// `.//`. -fn track_name(opener: Role, kind: Kind, number: u64) -> String { - format!(".{}/{}/{number}", opener.tag(), kind.tag()) +/// A fresh session id: 128 random bits as hex, so nobody can guess a session's tracks. +fn session_id() -> Arc { + use rand::RngExt; + format!("{:032x}", rand::rng().random::()).into() } -/// Split a track name into its opener and kind, or `None` for a track outside the -/// scheme, such as one the application names itself. -fn parse(track: &str) -> Option<(Role, Kind)> { - let (opener, rest) = track.strip_prefix('.')?.split_once('/')?; - let (kind, number) = rest.split_once('/')?; - // Decimal, as the opener writes it, so one stream has one name. - if number.parse::().ok()?.to_string() != number { - return None; - } - let opener = match opener { - "dial" => Role::Dialer, - "accept" => Role::Listener, - _ => return None, - }; - let kind = match kind { - "uni" => Kind::Uni, - "bi" => Kind::Bi, - _ => return None, - }; - Some((opener, kind)) +/// The track name of a session: `/`, the same in both endpoints. +fn track_name(dialer: &str, session: &str) -> String { + format!("{dialer}/{session}") } -/// `L//`, refusing an id that is not exactly one path segment. -fn side(path: &Path, side: &'static str, id: &str) -> Result { - let segment = Path::new(id)?; - if segment.parts().count() != 1 { - return Err(crate::InvalidPattern::InvalidSegment(id.to_string()).into()); - } - Ok(path.join(&Path::from_static(side))?.join(&segment)?) +/// Split a session's track name into its dialer and session, or `None` for a name +/// outside the scheme. +fn parse(name: &str) -> Option<(&str, &str)> { + let (dialer, session) = name.split_once('/')?; + check_id(dialer).ok()?; + check_id(session).ok()?; + Some((dialer, session)) } -/// Why a [`Socket`] ended. -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum Closed { - /// The peer's broadcast ended: it closed the socket, or a session on the way dropped. - #[error("the peer ended the socket")] - Remote, - /// This side's broadcast ended, through [`Socket::close`] or otherwise. - #[error("the socket was closed locally")] - Local, - /// The dialer dialed again with the same id, which replaces this socket. - #[error("a newer dial replaced the socket")] - Replaced, +/// What a session's reader asks for: every message, oldest first, however late it is. +fn reliable() -> Subscription { + Subscription::default() + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)) } -/// A bi stream: this side's track and the peer's, under one track name. -pub struct Bi { - /// This side's track. +/// One session: this side's half and the peer's. +/// +/// Each message is one group, so a slow message never holds up the next one, and messages +/// are ordered by group sequence. [`Self::consumer`] starts reliable and in order (oldest +/// first, with no delay budget); update its subscription to make it lossy. +/// +/// Finishing [`Self::producer`] closes the session for the peer, and aborting it is an +/// error. The endpoint closing aborts it too. +pub struct Stream { + /// This side's half: the session's track in this endpoint's broadcast. pub producer: track::Producer, - /// The peer's track. - pub consumer: track::Consumer, + /// The peer's half: the session's track in the peer's broadcast. + pub consumer: track::Subscriber, + peer: Arc, + session: Arc, } -/// What the peer's track listing has reported and nobody has taken yet. -#[derive(Default)] -struct Listing { - /// The peer's listing, opened by the first accept or bi open. - tracks: Option, - /// Uni streams the peer opened, not yet accepted, in order, by track name. A stream - /// leaves once the peer stops listing it, as the listing drops an add its reader - /// never reached, so these hold no more than the peer lists. - uni: LinkedHashSet>, - /// Bi streams the peer opened, with the opener's track info, not yet accepted. - bi: LinkedHashMap, track::Info>, - /// This side reading the opener's half of each bi stream it accepted, by track name, - /// until the opener's half leaves the listing or this side closes. The opener learns - /// of the accept from this demand, not from the answer's listing: a listing drops a - /// track that comes and goes before its reader reaches it, so an answer dropped at once - /// would never be seen, and the application may not read the opener's half before it - /// hears back. `None` while an accept is answering the stream, so a removal drained - /// meanwhile, by any poller, also drops the read the accept is about to register. - reading: HashMap, Option>>, - /// How the listing ended: `Ok` once the peer's broadcast ended, or why it failed. - end: Option>, -} +impl Stream { + /// The peer endpoint's id. + pub fn peer(&self) -> &str { + &self.peer + } -impl Listing { - /// Sort every event on hand into the queues, registering `waiter` for the next. - /// - /// Every poller drains the one listing, so each sorts what it reads for the others: - /// they all park on the listing, which wakes each of them on a change. - fn drain(&mut self, waiter: &kio::Waiter, role: Role, remote: &broadcast::Consumer) { - if self.end.is_some() { - return; - } - let tracks = match &mut self.tracks { - Some(tracks) => tracks, - None => match remote.tracks() { - Ok(tracks) => self.tracks.insert(tracks), - // The peer's broadcast already ended. - Err(Error::Unroutable) => { - self.end = Some(Ok(())); - return; - } - Err(err) => { - self.end = Some(Err(err)); - return; - } - }, - }; - loop { - let event = match tracks.poll_next(waiter) { - Poll::Pending => return, - Poll::Ready(Ok(Some(event))) => event, - Poll::Ready(Ok(None)) => { - self.end = Some(Ok(())); - self.reading.clear(); - return; - } - Poll::Ready(Err(err)) => { - self.end = Some(Err(err)); - self.reading.clear(); - return; - } - }; - let (name, info) = match event { - TracksEvent::Added { name, info } => (name, Some(info)), - TracksEvent::Removed { name } | TracksEvent::Finished { name } => (name, None), - TracksEvent::Synced => continue, - }; - // A track outside the scheme is the application's own. - let Some((opener, kind)) = parse(&name) else { - continue; - }; - match (opener == role, kind, info) { - (false, Kind::Uni, Some(_)) => { - self.uni.replace(name); - } - (false, Kind::Uni, None) => { - self.uni.remove(&name); - } - (false, Kind::Bi, Some(info)) => { - self.bi.replace(name, info); - } - (false, Kind::Bi, None) => { - self.bi.remove(&name); - // The opener dropped its half, so it resolved or gave up. - self.reading.remove(&name); - } - // An open learns of its answer from the peer reading it, not the listing. - (true, Kind::Bi, _) => {} - // This side's uni streams live only in its own broadcast. - (true, Kind::Uni, _) => {} - } - } + /// The session's id, which the dialer chose. + pub fn session(&self) -> &str { + &self.session } +} - /// The listing's end as what an accept returns: `None` once the peer is gone. - fn ended(&self) -> Option, Error>> { - match self.end.as_ref()? { - Ok(()) => Some(Ok(None)), - Err(err) => Some(Err(err.clone())), - } +/// What the endpoint's pollers share. +struct State { + /// The dials peers send, from the first accept on; `None` until then, so the endpoint + /// answers a request for a session it does not hold with [`Error::NotFound`] at once. + requests: Option, + /// Dials from peers, not yet taken by an accept, in order. + incoming: VecDeque, + /// This side's half of every session handed out, aborted when the endpoint closes. + streams: Vec, + /// Why the endpoint stopped, once it did. + end: Option, +} + +impl State { + /// Keep `producer`'s track, so the endpoint closing ends it. + fn hold(&mut self, producer: &track::Producer) { + self.streams.retain(|stream| !stream.is_closed()); + self.streams.push(producer.weak()); } } -/// What a socket shares with its pending bi opens and its listener. -struct Inner { - role: Role, +struct Shared { id: Arc, - /// This side's broadcast. - local: origin::Publication, - /// The peer's broadcast. - remote: broadcast::Consumer, - listing: kio::Lock, - /// Why the listener ended the socket, set before it closes [`Self::local`]. - closed: kio::Lock>, - /// The number of this side's next stream. - next: AtomicU64, + prefix: Path, + /// Where the peers' endpoints are requested. + origin: origin::Consumer, + /// This endpoint's broadcast. + publication: origin::Publication, + state: kio::Lock, } -impl Inner { - /// End the socket for `reason`, retracting this side's broadcast. - fn close(&self, reason: Closed) { - self.closed.lock().get_or_insert(reason); - self.retract(); +impl Shared { + /// Queue every dial on hand, registering `waiter` for the next, and fail with the + /// endpoint's end once it stopped. + fn drain(&self, state: &mut State, waiter: &kio::Waiter) -> Result<(), Error> { + while state.end.is_none() { + let requests = state.requests.get_or_insert_with(|| self.publication.requests()); + match requests.poll_requested_track(waiter) { + Poll::Pending => break, + Poll::Ready(Ok(request)) => self.sort(state, request), + Poll::Ready(Err(err)) => state.end = Some(err), + } + } + match &state.end { + Some(err) => Err(err.clone()), + None => Ok(()), + } } - /// Retract this side's broadcast and stop reading the opener's half of every bi stream - /// it accepted. Closed first, so an accept registering a read after the release sees - /// the close and drops it. - fn retract(&self) { - self.local.close(); - // Dropped outside the lock. - let reading = std::mem::take(&mut self.listing.lock().reading); - drop(reading); + /// Queue a peer's dial, or reset a request outside the scheme. + /// + /// A dial of this endpoint's reserves its own track, so a request reaching the handler + /// under this endpoint's id names a session it never dialed, or one it gave up: such + /// as an accept of a dial forged in this endpoint's name, which this reset fails. + fn sort(&self, state: &mut State, request: track::Request) { + match parse(request.name()) { + Some((dialer, _)) if dialer != &*self.id => state.incoming.push_back(request), + _ => { + // Not the name: a session's name is its secret. + tracing::debug!("reset a request for no session"); + request.reject(Error::NotFound); + } + } } - /// Whether this side's broadcast ended, registering `waiter` for when it does. - fn poll_local_closed(&self, waiter: &kio::Waiter) -> bool { - self.local.poll_closed(waiter).is_ready() + /// Poll for the endpoint's end, registering `waiter` for it. + fn poll_end(&self, waiter: &kio::Waiter) -> Result<(), Error> { + if let Some(err) = &self.state.lock().end { + return Err(err.clone()); + } + match self.publication.poll_closed(waiter) { + Poll::Ready(()) => Err(Error::Closed), + Poll::Pending => Ok(()), + } } - /// The track name of this side's next `kind` stream. - fn open(&self, kind: Kind) -> String { - track_name(self.role, kind, self.next.fetch_add(1, Ordering::Relaxed)) - } + fn close(&self) { + let mut state = self.state.lock(); + state.end.get_or_insert(Error::Closed); + let requests = state.requests.take(); + let incoming = std::mem::take(&mut state.incoming); + let streams = std::mem::take(&mut state.streams); + drop(state); - /// Create a track in this side's broadcast, refusing with [`Error::Closed`] once it - /// ended: a track in an ended broadcast reaches nobody. Checked after the create, so a - /// close racing it is either seen here or comes after the stream opened. - fn create_track(&self, name: String, info: impl Into>) -> Result { - let producer = self.local.create_track(name, info)?; - if self.poll_local_closed(&kio::Waiter::noop()) { - return Err(Error::Closed); + self.publication.close(); + for stream in streams { + stream.abort(Error::Cancel); } - Ok(producer) + // Reset outside the lock. + drop((requests, incoming)); } } -/// One end of a connection between a dialer and a listener. +/// A socket endpoint: announces `/.tcp/`, dials other endpoints, and accepts +/// their dials. /// -/// Holds this side's broadcast and the peer's. Drop it once it ends: dropping retracts -/// this side's broadcast, which ends the socket for the peer too. -pub struct Socket { - inner: Arc, +/// It accepts dials from its first [`Self::accept`] on, like a TCP listener: before that, a +/// dial to it is reset, and after it, dials queue until accepted. Dropping it, or +/// [`Self::close`], retracts the broadcast, resets the dials waiting to be accepted, and +/// aborts every session it handed out. +pub struct Endpoint { + shared: Arc, } -impl Socket { - fn new(role: Role, id: Arc, local: origin::Publication, remote: broadcast::Consumer) -> Self { - Self { - inner: Arc::new(Inner { - role, - id, - local, - remote, - listing: kio::Lock::new(Listing::default()), - closed: kio::Lock::new(None), - next: AtomicU64::new(0), +impl Endpoint { + /// Announce the endpoint `id` under `prefix` on `origin`, which also reaches the peers. + /// + /// `id` names the endpoint: a well-known name for a server, or a random one for a + /// client. It must be one path segment. Fails like + /// [`origin::Producer::create_broadcast`] when `origin` may not publish + /// `/.tcp/`. + pub fn new( + origin: &origin::Producer, + prefix: impl TryInto>, + id: &str, + ) -> Result { + let prefix = crate::path::convert(prefix)?; + let publication = origin.create_broadcast(endpoint_path(&prefix, id)?)?; + publication.announce(Route::default())?; + Ok(Self { + shared: Arc::new(Shared { + id: id.into(), + prefix, + origin: origin.consume(), + publication, + state: kio::Lock::new(State { + requests: None, + incoming: VecDeque::new(), + streams: Vec::new(), + end: None, + }), }), - } + }) } - /// The id the dialer chose. + /// This endpoint's id. pub fn id(&self) -> &str { - &self.inner.id + &self.shared.id } - /// This side's broadcast, for well-known channels: tracks with fixed names that both - /// sides agree on, beside the streams. - pub fn local(&self) -> &broadcast::Producer { - &self.inner.local + /// The prefix the endpoint's broadcast sits under, and where it finds its peers. + pub fn prefix(&self) -> &Path { + &self.shared.prefix } - /// The peer's broadcast, for well-known channels: tracks with fixed names that both - /// sides agree on, beside the streams. - pub fn remote(&self) -> &broadcast::Consumer { - &self.inner.remote - } - - /// Open a uni stream: a new track only this side writes, which the peer takes with - /// [`Self::accept_uni`]. - /// - /// `info` describes the track. Fails with [`Error::Closed`] once this side closed. - pub fn open_uni(&self, info: impl Into>) -> Result { - self.inner.create_track(self.inner.open(Kind::Uni), info) - } - - /// Open a bi stream, resolving once the peer accepts it with [`Self::accept_bi`]. + /// Dial the endpoint `peer` in a new session, resolving once the peer accepts it. /// - /// `info` describes this side's track, and the peer's answer mirrors it. Fails with - /// [`Error::Closed`] once this side closed. - pub fn open_bi(&self, info: impl Into>) -> Result, Error> { - let inner = &self.inner; - let track: Arc = inner.open(Kind::Bi).into(); - let producer = inner.create_track(track.to_string(), info)?; - Ok(kio::Pending::new(OpenBi { - inner: inner.clone(), - track, - demand: producer.demand(), - producer: Some(producer), + /// `info` describes this side's half. Waits for as long as the peer takes to accept: + /// drop the returned future to give up, which resets the session. That includes waiting + /// for the peer's endpoint to be announced. Fails with the peer's reset when it rejects + /// the dial, and with [`Error::Unsupported`] for a dial to this endpoint itself. + pub fn dial(&self, peer: &str, info: impl Into>) -> Result, Error> { + let shared = &self.shared; + let path = endpoint_path(&shared.prefix, peer)?; + if peer == &*shared.id { + return Err(Error::Unsupported); + } + if let Some(err) = &shared.state.lock().end { + return Err(err.clone()); + } + let session = session_id(); + // This side's half, waiting for the peer to ask for it, which only the peer can: + // nobody else knows the session. + let answer = shared + .publication + .reserve_unlisted_track(track_name(&shared.id, &session))?; + // Followed rather than requested, so a peer not announced yet is waited for. + let resolving = Resolving::Follow(shared.origin.follow(path, None)?); + Ok(kio::Pending::new(Dialing { + shared: shared.clone(), + peer: peer.into(), + session, + info: info.into(), + demand: answer.demand(), + answer: Some(Answer::Reserved(answer)), + step: Step::Resolving(resolving), })) } - /// Poll for the next uni stream the peer opened; `None` once the peer's broadcast ended, - /// and [`Error::Closed`] once this side closed. + /// Poll for the next dial from a peer; fails with [`Error::Closed`] once the endpoint + /// closed. /// - /// A stream is handed out only while the peer still lists it: one whose track the peer - /// dropped first, finished or not, is skipped. - pub fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll, Error>> { - let inner = &self.inner; - loop { - if inner.poll_local_closed(waiter) { - return Poll::Ready(Err(Error::Closed)); - } - let mut listing = inner.listing.lock(); - listing.drain(waiter, inner.role, &inner.remote); - let Some(track) = listing.uni.pop_front() else { - return match listing.ended() { - Some(end) => Poll::Ready(end), - None => Poll::Pending, - }; - }; - drop(listing); - return match inner.remote.track(&track) { - Ok(consumer) => Poll::Ready(Ok(Some(consumer))), - // The peer's broadcast ended. - Err(Error::Unroutable) => Poll::Ready(Ok(None)), - // The peer dropped the track since the listing read it. - Err(Error::NotFound) => continue, - Err(err) => Poll::Ready(Err(err)), - }; - } + /// Dials queue until taken, without limit, each holding the dialer's request open, so + /// keep accepting, and accept or reject each one. + pub fn poll_accept(&self, waiter: &kio::Waiter) -> Poll> { + let shared = &self.shared; + let mut state = shared.state.lock(); + shared.drain(&mut state, waiter)?; + let Some(request) = state.incoming.pop_front() else { + return Poll::Pending; + }; + drop(state); + let (peer, session) = parse(request.name()).expect("sorted as a dial"); + Poll::Ready(Ok(Incoming { + shared: shared.clone(), + peer: peer.into(), + session: session.into(), + request, + })) } - /// Wait for the next uni stream the peer opened; see [`Self::poll_accept_uni`]. + /// Wait for the next dial from a peer; see [`Self::poll_accept`]. #[cfg(feature = "async")] - pub async fn accept_uni(&self) -> Result, Error> { - kio::wait(|waiter| self.poll_accept_uni(waiter)).await + pub async fn accept(&self) -> Result { + kio::wait(|waiter| self.poll_accept(waiter)).await } - /// Poll for the next bi stream the peer opened, answering it with a track that mirrors - /// the opener's info; `None` once the peer's broadcast ended, and [`Error::Closed`] - /// once this side closed. Skips streams as [`Self::poll_accept_uni`] does. - /// - /// The socket reads the opener's half until the opener drops it or this side closes, - /// which is how the opener learns of the accept. As with any stream, the answer reaches the opener only - /// while this side holds it: finishing it at once is fine, dropping it before the - /// opener reads it is not. - pub fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll, Error>> { - let inner = &self.inner; - loop { - if inner.poll_local_closed(waiter) { - return Poll::Ready(Err(Error::Closed)); - } - let mut listing = inner.listing.lock(); - listing.drain(waiter, inner.role, &inner.remote); - let Some((track, info)) = listing.bi.pop_front() else { - return match listing.ended() { - Some(end) => Poll::Ready(end), - None => Poll::Pending, - }; - }; - // Already answered: the peer listed it again, which it does only by opening a - // number twice. - if listing.reading.contains_key(&track) { - continue; - } - listing.reading.insert(track.clone(), None); - drop(listing); - - let answered = self.answer_bi(&track, info); - let mut listing = inner.listing.lock(); - let (bi, read) = match answered { - Ok(Some(answered)) => answered, - Ok(None) => { - listing.reading.remove(&track); - continue; - } - Err(err) => { - listing.reading.remove(&track); - return match err { - // The peer's broadcast ended. - Error::Unroutable => Poll::Ready(Ok(None)), - err => Poll::Ready(Err(err)), - }; - } - }; - // Under the lock, so a close or a removal of the stream either comes after this - // and drops the read, or came first and is seen here. - let closed = inner.poll_local_closed(&kio::Waiter::noop()); - if !closed && let Some(slot) = listing.reading.get_mut(&track) { - // How the opener learns of the accept. - *slot = Some(read); - return Poll::Ready(Ok(Some(bi))); - } - // Dropped outside the lock. - let stale = listing.reading.remove(&track); - drop(listing); - drop((stale, read, bi)); - if closed { - return Poll::Ready(Err(Error::Closed)); - } - // The opener dropped its half while this answered it. - } + /// Close the endpoint; see [`Endpoint`]. + pub fn close(&self) { + self.shared.close(); } +} - /// Answer the peer's bi stream `track` with a track mirroring the opener's `info`, and - /// start reading the opener's half. `None` for a stream to skip. - fn answer_bi( - &self, - track: &str, - info: track::Info, - ) -> Result)>, Error> { - let inner = &self.inner; - let consumer = match inner.remote.track(track) { - Ok(consumer) => consumer, - // The peer dropped the track since the listing read it. - Err(Error::NotFound) => return Ok(None), - Err(err) => return Err(err), - }; - // The opener's track info, minus how its copy reached this hop. - let info = track::Info { - properties: None, - unchecked: false, - ..info - }; - // Answered under the opener's track name. - let producer = match inner.create_track(track.to_string(), info) { - Ok(producer) => producer, - // This side still holds an answer under the name, which the peer listed again - // only by opening a number twice. - Err(Error::Duplicate) => return Ok(None), - Err(err) => return Err(err), - }; - let read = consumer.subscribe(None); - Ok(Some((Bi { producer, consumer }, read))) +impl Drop for Endpoint { + fn drop(&mut self) { + self.shared.close(); } +} - /// Wait for the next bi stream the peer opened; see [`Self::poll_accept_bi`]. - #[cfg(feature = "async")] - pub async fn accept_bi(&self) -> Result, Error> { - kio::wait(|waiter| self.poll_accept_bi(waiter)).await +/// A dial from a peer, from [`Endpoint::accept`]: accept or reject it. +/// +/// Dropping it rejects it. +pub struct Incoming { + shared: Arc, + peer: Arc, + session: Arc, + request: track::Request, +} + +impl Incoming { + /// The id the dialer claims. [`Self::accept`] checks it: the dial connects only if that + /// endpoint really dialed this session. + pub fn peer(&self) -> &str { + &self.peer } - /// Poll for the socket to end, and why. - pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll { - let inner = &self.inner; - if let Some(reason) = *inner.closed.lock() { - return Poll::Ready(reason); - } - if inner.local.poll_closed(waiter).is_ready() { - // The listener may have ended it between the check above and the close. - return Poll::Ready(inner.closed.lock().unwrap_or(Closed::Local)); - } - ready!(inner.remote.poll_closed(waiter)); - Poll::Ready(Closed::Remote) + /// The session's id, which the dialer chose. + pub fn session(&self) -> &str { + &self.session } - /// Wait for the socket to end, and why. - #[cfg(feature = "async")] - pub async fn closed(&self) -> Closed { - kio::wait(|waiter| self.poll_closed(waiter)).await + /// Accept the dial, resolving once the dialer answers, which proves it dialed. + /// + /// `info` describes this side's half. Nothing is written before the dialer answers. + pub fn accept(self, info: impl Into>) -> kio::Pending { + let Self { + shared, + peer, + session, + request, + } = self; + // Requested, so a dial claiming an endpoint nobody announces fails at once: the + // dialer announced its endpoint before it dialed. + let resolving = endpoint_path(&shared.prefix, &peer) + .map(|path| Resolving::Request(shared.origin.request(path, None).into_inner())); + let step = match resolving { + Ok(resolving) => Step::Resolving(resolving), + Err(err) => Step::Failed(err), + }; + kio::Pending::new(Accepting { + shared, + peer, + session, + info: info.into(), + request: Some(request), + step, + }) } - /// End the socket, retracting this side's broadcast. - pub fn close(&self) { - self.inner.close(Closed::Local); + /// Reject the dial, resetting the dialer's request with [`Error::NotFound`], as for a + /// session that does not exist. + pub fn reject(self) { + self.request.reject(Error::NotFound); } } -impl Drop for Socket { - fn drop(&mut self) { - // A pending bi open still holds the shared half, but not the socket. - self.inner.retract(); +/// Resolving the peer's endpoint broadcast. +enum Resolving { + /// Failing once the endpoint is not announced. + Request(origin::Requesting), + /// Waiting for the endpoint to be announced. + Follow(origin::Following), +} + +impl Resolving { + fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + match self { + Self::Request(request) => request.poll_ok(waiter), + Self::Follow(follow) => follow.poll_available(waiter), + } } } -/// A bi stream being opened, from [`Socket::open_bi`]: resolves once the peer answers. -pub struct OpenBi { - inner: Arc, - /// The stream's track name, which the answer shares. - track: Arc, - /// This side's track, handed over once the peer answers. - producer: Option, - /// Whether anyone reads this side's track, which the peer does from its accept on. - demand: track::Demand, +/// How far a dial or an accept got in reading the peer's half. +enum Step { + /// Resolving the peer's endpoint broadcast. + Resolving(Resolving), + /// Subscribed to the peer's half, waiting for the peer to serve it. + Subscribing(track::Subscribing), + /// The peer serves its half. + Subscribed(track::Subscriber), + /// It failed before it started. + Failed(Error), + /// Handed out, or failed. + Done, } -impl OpenBi { - /// Poll for the peer's answer, which it gives by reading this side's half. Any reader - /// counts, so only the peer should be able to read this side's broadcast; see the - /// authorization section of `doc/concept/socket.md`. Fails with - /// [`Error::Unroutable`] once the peer's broadcast ends without one, and with - /// [`Error::Closed`] once this side closed, even if the answer came. - pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { - let inner = &self.inner; - // This side's track went with its broadcast, and a peer that keeps its own up - // without answering would otherwise hold this open forever. - if inner.poll_local_closed(waiter) { - return Poll::Ready(Err(Error::Closed)); - } - // Polled until it parks, so a later change wakes this. - while let Poll::Ready(Ok(_)) = self.demand.poll_wanted_changed(waiter) {} - if self.demand.wanted().is_some() { - let consumer = inner.remote.track(&self.track)?; - let producer = self.producer.take().ok_or(Error::Closed)?; - return Poll::Ready(Ok(Bi { producer, consumer })); +impl Step { + /// Advance until the peer serves its half, subscribing to `track` in its endpoint. + fn poll(&mut self, waiter: &kio::Waiter, track: &str) -> Poll> { + loop { + match self { + Self::Resolving(resolving) => { + let broadcast = ready!(resolving.poll(waiter))?; + let subscribing = broadcast.track(track)?.subscribe(reliable()).into_inner(); + *self = Self::Subscribing(subscribing); + } + Self::Subscribing(subscribing) => { + let subscriber = ready!(subscribing.poll_ok(waiter))?; + *self = Self::Subscribed(subscriber); + } + Self::Subscribed(_) => return Poll::Ready(Ok(())), + Self::Failed(err) => return Poll::Ready(Err(err.clone())), + Self::Done => return Poll::Ready(Err(Error::Closed)), + } } - let mut listing = inner.listing.lock(); - listing.drain(waiter, inner.role, &inner.remote); - match &listing.end { - Some(Ok(())) => Poll::Ready(Err(Error::Unroutable)), - Some(Err(err)) => Poll::Ready(Err(err.clone())), - None => Poll::Pending, + } + + fn take_subscriber(&mut self) -> track::Subscriber { + match std::mem::replace(self, Self::Done) { + Self::Subscribed(subscriber) => subscriber, + _ => unreachable!("taken once subscribed"), } } } -impl kio::Task for OpenBi { - type Output = Result; - - fn poll(&mut self, waiter: &kio::Waiter) -> Poll { - self.poll_ok(waiter) - } +/// This side's half of a dial. +enum Answer { + /// Reserved, until the peer asks for it. + Reserved(track::Request), + /// Served, once the peer asked. + Served(track::Producer), } -/// Dial the listener at `path` as `id`, from [`dial`]: resolves once the listener accepts. +/// A dial, from [`Endpoint::dial`]: resolves once the peer accepts. +/// +/// Dropping it gives up, resetting the session. pub struct Dialing { - id: Arc, - /// This side's broadcast, announced for the listener to find. - local: Option, - /// The listener's answer, pinned to this dial's instance. - answer: origin::Following, + shared: Arc, + peer: Arc, + session: Arc, + info: Option, + /// Whether the peer asked for this side's half, which it does on accept. + demand: track::Demand, + answer: Option, + step: Step, } impl Dialing { - /// Poll for the listener's answer. Waits for as long as it takes: drop this to give up. - pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { - let remote = ready!(self.answer.poll_available(waiter))?; - let local = self.local.take().ok_or(Error::Closed)?; - Poll::Ready(Ok(Socket::new(Role::Dialer, self.id.clone(), local, remote))) + /// The session's id. + pub fn session(&self) -> &str { + &self.session + } + + /// Poll for the peer to accept; see [`Endpoint::dial`]. + pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { + let result = self.poll_inner(waiter); + if let Poll::Ready(Err(_)) = &result { + // Resets the session for the peer. + self.answer = None; + self.step = Step::Done; + } + result + } + + fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { + let shared = &self.shared; + shared.poll_end(waiter)?; + + // The peer asks for this side's half when it accepts: serve it, which proves to the + // peer that this endpoint dialed. + if let Some(Answer::Reserved(_)) = &self.answer { + // Polled until it parks, so the next change wakes this. + while let Poll::Ready(changed) = self.demand.poll_wanted_changed(waiter) { + // Ends once the reservation does: the endpoint closed. + changed?; + } + if self.demand.wanted().is_some() + && let Some(Answer::Reserved(request)) = self.answer.take() + { + self.answer = Some(Answer::Served(request.accept(self.info.take()))); + } + } + + let track = track_name(&shared.id, &self.session); + ready!(self.step.poll(waiter, &track))?; + // The peer serves its half only after asking for this side's, so a conforming peer + // asked already; wait for its request regardless. + let Some(Answer::Served(_)) = &self.answer else { + return Poll::Pending; + }; + let Some(Answer::Served(producer)) = self.answer.take() else { + unreachable!("checked above"); + }; + + let mut state = shared.state.lock(); + if let Some(err) = &state.end { + return Poll::Ready(Err(err.clone())); + } + state.hold(&producer); + drop(state); + Poll::Ready(Ok(Stream { + producer, + consumer: self.step.take_subscriber(), + peer: self.peer.clone(), + session: self.session.clone(), + })) } } impl kio::Task for Dialing { - type Output = Result; + type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { self.poll_ok(waiter) } } -/// Dial the listener at `path` as `id`, resolving once it accepts. -/// -/// Publishes this side's broadcast at `path/.dial/` with a fresh [`Instance`], and -/// waits for the listener's answer at `path/.accept/` carrying that instance. There -/// is no timeout: drop the returned future to give up, which retracts the dial. -/// -/// `id` must be one path segment; an id with a `/` fails with [`Error::InvalidPath`]. -/// Fails like [`origin::Producer::create_broadcast`] when `origin` may not publish the -/// dial, and like [`origin::Consumer::follow`] when it may not read the answer. -pub fn dial( - origin: &origin::Producer, - path: impl TryInto>, - id: &str, -) -> Result, Error> { - let path = crate::path::convert(path)?; - let instance = Instance::random(); - let local = origin.create_broadcast(side(&path, DIAL, id)?)?; - let answer = origin.consume().follow(side(&path, ACCEPT, id)?, instance.clone())?; - local.announce(Route::default().with_instance(instance, 0))?; - Ok(kio::Pending::new(Dialing { - id: id.into(), - local: Some(local), - answer, - })) -} - -/// A socket the listener handed out, kept so a newer dial or the dialer leaving ends it. -struct Live { - instance: Instance, - socket: Weak, -} - -/// A dial being accepted: its broadcast requested, pinned to its instance. -struct Accepting { - id: Path, - instance: Instance, - request: origin::Requesting, -} - -/// Accepts the sockets dialed to a path. -/// -/// Dialers queue in the announcements until [`Self::accept`] takes them, without limit, -/// and a dialer's broadcast is requested only then: a listener that stops accepting -/// holds nothing for the dialers waiting on it. While it accepts, a dialer dialing again -/// with the same id ends the socket it replaces with [`Closed::Replaced`]. -/// -/// Run one listener per path. Two would both answer a dial under its instance, and the -/// dialer would follow only one of them, leaving the other's socket unread. -pub struct Listener { - origin: origin::Producer, - path: Path, - /// The dialers' broadcasts, rooted at `path/.dial`. - dials: origin::Consumer, - announced: announce::Consumer, - live: HashMap, - accepting: Option, +/// An accept, from [`Incoming::accept`]: resolves once the dialer answers. +pub struct Accepting { + shared: Arc, + peer: Arc, + session: Arc, + info: Option, + /// The dialer's request for this side's half, served once the dialer answers. + request: Option, + step: Step, } -impl Listener { - /// Listen at `path` on `origin`, which publishes the answers and reads the dials. +impl Accepting { + /// Poll for the dialer to answer; see [`Incoming::accept`]. /// - /// Fails with [`Error::Unauthorized`] when `origin` may not read `path/.dial`. - pub fn new(origin: &origin::Producer, path: impl TryInto>) -> Result { - let path = crate::path::convert(path)?; - let ids = Patterns::from("*".parse::()?); - // An id is any one segment, a dotted one included. - let dials = origin - .consume() - .scope(path.join(&Path::from_static(DIAL))?, &ids)? - .with_hidden(true); - let announced = dials.announced(); - Ok(Self { - origin: origin.clone(), - path, - dials, - announced, - live: HashMap::new(), - accepting: None, - }) - } - - /// The path this listener accepts dials at. - pub fn path(&self) -> &Path { - &self.path + /// Fails with the dialer's reset when it never dialed this session, such as a dial + /// forged in its name, and resets the dialer's request with the same error. + pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { + let result = self.poll_inner(waiter); + if let Poll::Ready(Err(err)) = &result { + self.step = Step::Done; + if let Some(request) = self.request.take() { + request.reject(err.clone()); + } + } + result } - /// Poll for the next dialer, requesting its broadcast and answering it. - /// - /// Dialers are taken one at a time, in order. A session resolves a request for a - /// broadcast its peer announced itself, without asking the peer, so a remote dialer - /// cannot hold up the ones behind it; only an [`origin::Requests`] handler on this - /// origin that holds a dial's request can. A dialer that leaves before its broadcast - /// resolves is skipped. Fails with [`Error::Closed`] once the origin closes. - pub fn poll_accept(&mut self, waiter: &kio::Waiter) -> Poll> { - loop { - if let Some(accepting) = &mut self.accepting { - let remote = ready!(accepting.request.poll_ok(waiter)); - let accepting = self.accepting.take().expect("accepting"); - match remote { - Ok(remote) => match self.answer(accepting, remote) { - Ok(Some(socket)) => return Poll::Ready(Ok(socket)), - Ok(None) => {} - Err(err) => return Poll::Ready(Err(err)), - }, - Err(err) => tracing::debug!(id = %accepting.id, %err, "dialer left before it was accepted"), - } - continue; - } + fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { + let shared = &self.shared; + shared.poll_end(waiter)?; + let track = track_name(&self.peer, &self.session); + ready!(self.step.poll(waiter, &track))?; - let Some(event) = ready!(self.announced.poll_next(waiter)) else { - return Poll::Ready(Err(Error::Closed)); - }; - let (announce, takeover) = match event { - announce::Event::Start(announce) | announce::Event::Restart(announce) => (announce, false), - announce::Event::Takeover { announce, .. } => (announce, true), - // The same instance, delivered in place. - announce::Event::Update(_) => continue, - announce::Event::End(announce) => { - // The dialer's routes are gone, and so is its broadcast; retract the - // answer too, so a return of the same instance never meets it. - if let Some(live) = self.live.remove(&announce.prefix) - && let Some(socket) = live.socket.upgrade() - { - socket.close(Closed::Remote); - } - continue; - } - }; - let Some(instance) = announce.route.instance else { - continue; - }; - if let Some(live) = self.live.get_mut(&announce.prefix) { - match live.socket.upgrade() { - Some(_) if live.instance == instance => continue, - // A takeover continues the stream the live socket already reads. - Some(_) if takeover => { - live.instance = instance; - continue; - } - Some(socket) => socket.close(Closed::Replaced), - None => {} - } - self.live.remove(&announce.prefix); - } - let request = self.dials.request(&announce.prefix, instance.clone()).into_inner(); - self.accepting = Some(Accepting { - id: announce.prefix, - instance, - request, - }); + let mut state = shared.state.lock(); + if let Some(err) = &state.end { + return Poll::Ready(Err(err.clone())); } + let request = self.request.take().ok_or(Error::Closed)?; + let producer = request.accept(self.info.take()); + state.hold(&producer); + drop(state); + Poll::Ready(Ok(Stream { + producer, + consumer: self.step.take_subscriber(), + peer: self.peer.clone(), + session: self.session.clone(), + })) } +} - /// Wait for the next dialer; see [`Self::poll_accept`]. - #[cfg(feature = "async")] - pub async fn accept(&mut self) -> Result { - kio::wait(|waiter| self.poll_accept(waiter)).await - } +impl kio::Task for Accepting { + type Output = Result; - /// Publish the answer to a resolved dial, or `None` when this dialer's answer can't be - /// published, such as an id this origin may not answer. - fn answer(&mut self, accepting: Accepting, remote: broadcast::Consumer) -> Result, Error> { - let Accepting { id, instance, .. } = accepting; - let local = side(&self.path, ACCEPT, id.as_str()) - .and_then(|path| self.origin.create_broadcast(path)) - .and_then(|local| { - local.announce(Route::default().with_instance(instance.clone(), 0))?; - Ok(local) - }); - let local = match local { - Ok(local) => local, - Err(Error::Closed) => return Err(Error::Closed), - Err(err) => { - tracing::warn!(%id, %err, "cannot answer the dialer"); - return Ok(None); - } - }; - let socket = Socket::new(Role::Listener, id.as_str().into(), local, remote); - self.live.insert( - id, - Live { - instance, - socket: Arc::downgrade(&socket.inner), - }, - ); - Ok(Some(socket)) + fn poll(&mut self, waiter: &kio::Waiter) -> Poll { + self.poll_ok(waiter) } } @@ -792,41 +590,31 @@ mod tests { #[test] fn track_names_round_trip() { - for opener in [Role::Dialer, Role::Listener] { - for kind in [Kind::Uni, Kind::Bi] { - let name = track_name(opener, kind, 7); - assert_eq!(parse(&name), Some((opener, kind))); - } - } - assert_eq!(track_name(Role::Dialer, Kind::Bi, 0), ".dial/bi/0"); - assert_eq!(track_name(Role::Listener, Kind::Uni, 12), ".accept/uni/12"); + let name = track_name("alice", "0123abcd"); + assert_eq!(name, "alice/0123abcd"); + assert_eq!(parse(&name), Some(("alice", "0123abcd"))); } #[test] fn foreign_track_names_are_skipped() { - for name in [ - "command", - ".dial", - ".dial/uni", - ".dial/uni/", - ".dial/other/0", - ".other/uni/0", - "dial/uni/0", - ".dial/uni/x", - ".dial/uni/0/x", - ".dial/uni/07", - ".dial/uni/-1", - ".dial/uni/+1", - ] { + for name in ["command", "alice/", "/s", "alice/s/0", "a\\b/s", "alice/s\\"] { assert_eq!(parse(name), None, "{name}"); } } #[test] - fn an_id_is_one_segment() { - let path = Path::from_static("room"); - assert_eq!(side(&path, DIAL, "alice").unwrap().as_str(), "room/.dial/alice"); - assert!(side(&path, DIAL, "a/b").is_err()); - assert!(side(&path, DIAL, "").is_err()); + fn an_endpoint_path_holds_one_segment() { + let prefix = Path::from_static("room"); + assert_eq!(endpoint_path(&prefix, "alice").unwrap().as_str(), "room/.tcp/alice"); + assert!(endpoint_path(&prefix, "a/b").is_err()); + assert!(endpoint_path(&prefix, "").is_err()); + assert!(endpoint_path(&prefix, "a\\b").is_err()); + } + + #[test] + fn session_ids_are_128_bits() { + let (a, b) = (session_id(), session_id()); + assert_eq!(a.len(), 32); + assert_ne!(a, b); } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index b592110afc9..dd75b9dc56a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -4270,6 +4270,21 @@ impl TrackWeak { true } + /// Abort the track with `err`, as [`Producer::abort`] does, unless it already closed. + /// + /// For an owner that hands a track's [`Producer`] out and must end it on its own + /// shutdown without keeping it alive. Leaves the listing alone: such a track is one + /// served on request, which is never listed. + pub(super) fn abort(&self, err: Error) { + let Some(producer) = self.state.produce() else { + return; + }; + let Ok(state) = producer.write() else { + return; + }; + commit_abort(state, err).abort(); + } + /// Whether its producer declared the track's end, which is how its listing ends too. pub(super) fn is_finished(&self) -> bool { self.state.read().end.is_some() diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 6c04585d354..5600a252fcc 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -1,12 +1,13 @@ #![cfg(feature = "async")] -//! Sockets through a relay: a dialer and a listener on their own origins, each with a -//! session to a relay, over moq-lite-07. +//! Sockets through a relay: endpoints on their own origins, each with a session to a +//! relay, dialing and accepting sessions over moq-lite-07 and moq-transport. use std::time::Duration; -use moq_net::socket::{self, Closed, Listener, Socket}; -use moq_net::{Hop, Instance, Version, origin, track}; +use moq_net::broadcast::TracksEvent; +use moq_net::socket::{Endpoint, Stream}; +use moq_net::{Error, Hop, Version, origin, track}; use crate::support::harness::{MockConnectOptions, MockPair, connect_mock, spawn}; @@ -15,597 +16,324 @@ const TIMEOUT: Duration = Duration::from_secs(10); /// Long enough for every hop to settle, in simulated time. const SETTLE: Duration = Duration::from_secs(1); +const LITE: &str = "moq-lite-07-wip"; +const IETF: &str = "moq-transport-18"; + fn origin(hop: u64) -> origin::Producer { let (origin, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); spawn(driver); origin } -/// A client publishing and subscribing through one origin, connected to `relay`. -async fn client(origin: &origin::Producer, relay: &origin::Producer) -> MockPair { - let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); - options.client_publish = Some(origin.consume()); - options.client_subscribe = Some(origin.clone()); - options.server_publish = Some(relay.consume()); - options.server_subscribe = Some(relay.clone()); - // Discovery follows the socket's own watches, as it does for an application. - options.client_discovers_all = false; - connect_mock(options).await -} - -/// A relay with a listener client and a dialer client. +/// A relay and the clients connected to it. struct Fixture { relay: origin::Producer, - dialer: origin::Producer, - listener: origin::Producer, + next: u64, sessions: Vec, } impl Fixture { - async fn new() -> Self { - let relay = origin(1); - let dialer = origin(2); - let listener = origin(3); - let sessions = vec![client(&dialer, &relay).await, client(&listener, &relay).await]; + fn new() -> Self { Self { - relay, - dialer, - listener, - sessions, + relay: origin(1), + next: 2, + sessions: Vec::new(), } } - /// Another dialer client of the same relay. - async fn dialer(&mut self, hop: u64) -> origin::Producer { - let dialer = origin(hop); - self.sessions.push(client(&dialer, &self.relay).await); - dialer + /// A client publishing and subscribing through one origin, connected to the relay over + /// `version`. + async fn client(&mut self, version: &str) -> origin::Producer { + let client = origin(self.next); + self.next += 1; + let mut options = MockConnectOptions::new(version.parse::().unwrap()); + options.client_publish = Some(client.consume()); + options.client_subscribe = Some(client.clone()); + options.server_publish = Some(self.relay.consume()); + options.server_subscribe = Some(self.relay.clone()); + // Discovery follows the endpoint's own requests, as it does for an application. + options.client_discovers_all = false; + self.sessions.push(connect_mock(options).await); + client + } + + /// An endpoint `id` on its own client of the relay. + async fn endpoint(&mut self, version: &str, id: &str) -> Endpoint { + let client = self.client(version).await; + Endpoint::new(&client, "room", id).unwrap() } } -/// Dial `id` from `origin` while `listener` accepts, returning both ends. -async fn connect(origin: &origin::Producer, listener: &mut Listener, id: &str) -> (Socket, Socket) { - let dial = socket::dial(origin, "room", id).unwrap(); - let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, listener.accept()) }) +/// Dial `acceptor` from `dialer` while `acceptor` accepts, returning both ends. +async fn connect(dialer: &Endpoint, acceptor: &Endpoint) -> (Stream, Stream) { + let dial = dialer.dial(acceptor.id(), None).unwrap(); + let accept = async { + let incoming = acceptor.accept().await?; + assert_eq!(incoming.peer(), dialer.id()); + incoming.accept(None).await + }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) .await .expect("connect timeout"); (dialed.expect("dial"), accepted.expect("accept")) } -/// Write `payload` as one group of `track`. +/// Write `payload` as one message: one group of `track`. fn send(track: &track::Producer, payload: &'static [u8]) { let mut group = track.append_group().unwrap(); group.write_frame(None, payload).unwrap(); group.finish().unwrap(); } -/// Subscribe to `track` and read its first frame. -async fn recv(track: &track::Consumer) -> Vec { +/// Read the next message from `subscriber`, or `None` once the peer finished. +async fn recv(subscriber: &mut track::Subscriber) -> Result>, Error> { kio::clock::timeout(TIMEOUT, async { - let mut subscriber = track.subscribe(None).await.expect("subscribe"); - let mut group = subscriber.recv_group().await.expect("group").expect("a group"); - group - .read_frame() - .await - .expect("frame") - .expect("a frame") - .payload - .to_vec() + let Some(mut group) = subscriber.recv_group().await? else { + return Ok(None); + }; + let frame = group.read_frame().await?.expect("a frame"); + Ok(Some(frame.payload.to_vec())) }) .await .expect("read timeout") } -/// Uni and bi streams each way, through the relay. +/// Messages flow both ways, in order, over every pairing of moq-lite-07 and +/// moq-transport: the handshake uses neither instances nor track listings. #[kio::test] -async fn streams_flow_both_ways() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - assert_eq!(dialer.id(), "alice"); - assert_eq!(accepted.id(), "alice"); - - // A uni stream each way. - let up = dialer.open_uni(None).unwrap(); - assert_eq!(up.name(), ".dial/uni/0"); - send(&up, b"from the dialer"); - let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) - .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - assert_eq!(uni.name(), up.name()); - assert_eq!(recv(&uni).await, b"from the dialer"); - - let down = accepted.open_uni(None).unwrap(); - assert_eq!(down.name(), ".accept/uni/0"); - send(&down, b"from the listener"); - let uni = kio::clock::timeout(TIMEOUT, dialer.accept_uni()) - .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - assert_eq!(uni.name(), down.name()); - assert_eq!(recv(&uni).await, b"from the listener"); - - // A bi stream each way, both the opener's second stream: the opener's role keeps - // them apart. - for (opener, acceptor) in [(&dialer, &accepted), (&accepted, &dialer)] { - let info = track::Info::default().with_priority(3); - let open = opener.open_bi(info).unwrap(); - let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(open, acceptor.accept_bi()) }) - .await - .expect("bi timeout"); - let opened = opened.expect("open"); - let answered = answered.expect("accept").expect("a stream"); - assert!(opened.producer.name().ends_with("/bi/1"), "{}", opened.producer.name()); - assert_eq!(answered.producer.name(), opened.producer.name()); - assert_eq!( - answered.producer.info().priority, - 3, - "the answer mirrors the opener's info" - ); - let info = kio::clock::timeout(TIMEOUT, opened.consumer.info()) - .await - .expect("info timeout") - .expect("the answer's info"); - assert_eq!(info.priority, 3, "the opener reads the mirrored info"); - - send(&opened.producer, b"ping"); - assert_eq!(recv(&answered.consumer).await, b"ping"); - send(&answered.producer, b"pong"); - assert_eq!(recv(&opened.consumer).await, b"pong"); - } +async fn sessions_carry_messages_both_ways() { + for (dialer, acceptor) in [(LITE, LITE), (LITE, IETF), (IETF, LITE), (IETF, IETF)] { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(dialer, "alice").await; + let bob = fixture.endpoint(acceptor, "bob").await; + let (mut dialed, mut accepted) = connect(&alice, &bob).await; + assert_eq!(dialed.peer(), "bob"); + assert_eq!(accepted.peer(), "alice"); + assert_eq!(dialed.session(), accepted.session()); + + for message in [b"one", b"two", b"six"] { + send(&dialed.producer, message); + } + for message in [b"one", b"two", b"six"] { + assert_eq!( + recv(&mut accepted.consumer).await.unwrap().unwrap(), + message, + "{dialer} to {acceptor}" + ); + } + send(&accepted.producer, b"pong"); + assert_eq!(recv(&mut dialed.consumer).await.unwrap().unwrap(), b"pong"); - // Neither side's own streams come back as the peer's. - assert!( - kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err(), - "the dialer accepted a stream it opened" - ); - assert!( - kio::clock::timeout(SETTLE, dialer.accept_uni()).await.is_err(), - "the dialer accepted a stream twice" - ); + // Finishing a half closes the pipe that way. + dialed.producer.finish().unwrap(); + assert_eq!(recv(&mut accepted.consumer).await.unwrap(), None); + } } -/// A dialer that dials again with the same id, as a restarted one does, ends the socket -/// it replaces, and the listener accepts the new one. +/// A pair of endpoints holds several sessions at once, each its own pipe. #[kio::test] -async fn a_new_dial_replaces_the_old_socket() { - let mut fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (old_dialer, old) = connect(&fixture.dialer, &mut listener, "alice").await; - - let restarted = fixture.dialer(4).await; - let (new_dialer, new) = connect(&restarted, &mut listener, "alice").await; - - assert_eq!( - kio::clock::timeout(TIMEOUT, old.closed()) - .await - .expect("old socket open"), - Closed::Replaced - ); - assert_eq!( - kio::clock::timeout(TIMEOUT, old_dialer.closed()) - .await - .expect("old dialer open"), - Closed::Remote - ); - - // The new socket is the new dialer's. - let up = new_dialer.open_uni(None).unwrap(); - send(&up, b"again"); - let uni = kio::clock::timeout(TIMEOUT, new.accept_uni()) - .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - assert_eq!(recv(&uni).await, b"again"); - assert!( - kio::clock::timeout(SETTLE, new.closed()).await.is_err(), - "the new socket ended" - ); +async fn several_sessions_share_a_pair() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (first, mut first_accepted) = connect(&alice, &bob).await; + let (second, mut second_accepted) = connect(&alice, &bob).await; + // Either side may dial. + let (mut third, third_accepted) = connect(&bob, &alice).await; + assert_ne!(first.session(), second.session()); + + send(&second.producer, b"second"); + send(&first.producer, b"first"); + send(&third_accepted.producer, b"third"); + assert_eq!(recv(&mut first_accepted.consumer).await.unwrap().unwrap(), b"first"); + assert_eq!(recv(&mut second_accepted.consumer).await.unwrap().unwrap(), b"second"); + assert_eq!(recv(&mut third.consumer).await.unwrap().unwrap(), b"third"); } -/// A dialer the listener has not accepted is not read: its tracks see no demand until -/// the application accepts it. +/// Rejecting a dial resets it. #[kio::test] -async fn an_unaccepted_dialer_holds_no_subscription() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - - // A dialer by hand, so the test can watch its broadcast. - let dial = fixture.dialer.create_broadcast("room/.dial/bob").unwrap(); - let probe = dial.create_track("probe", None).unwrap(); - dial.announce(origin::Route::default().with_instance(Instance::random(), 0)) - .unwrap(); - - let mut demand = probe.demand(); - assert!(demand.wanted_changed().await.unwrap().is_none()); - assert!( - kio::clock::timeout(SETTLE, demand.wanted_changed()).await.is_err(), - "an unaccepted dialer was read" - ); - - let accepted = kio::clock::timeout(TIMEOUT, listener.accept()) - .await - .expect("accept timeout") - .unwrap(); - assert_eq!(accepted.id(), "bob"); - let _subscriber = kio::clock::timeout(TIMEOUT, accepted.remote().track("probe").unwrap().subscribe(None)) +async fn a_rejected_dial_fails() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let dial = alice.dial("bob", None).unwrap(); + let reject = async { + bob.accept().await.unwrap().reject(); + }; + let (dialed, ()) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, reject) }) .await - .expect("subscribe timeout") - .unwrap(); - assert!(demand.wanted().is_some(), "the accepted dialer is read"); -} - -/// A listener going away ends the sockets it accepted. -#[kio::test] -async fn a_listener_restart_ends_its_sockets() { - let mut fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, _accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - // The listener's session drops, as when its process restarts. - drop(fixture.sessions.remove(1)); - assert_eq!( - kio::clock::timeout(TIMEOUT, dialer.closed()) - .await - .expect("socket open"), - Closed::Remote - ); + .expect("reject timeout"); + assert!(dialed.is_err(), "a rejected dial connected"); } -/// A dial resolves only on the answer to it: neither an answer carrying another instance -/// at its path nor another id's answer is taken for it. +/// A dial waits for its peer's endpoint to be announced, and a dial to itself fails. #[kio::test] -async fn a_dial_takes_only_its_own_answer() { - let mut fixture = Fixture::new().await; - - // An answer at alice's path for some other dial of alice, such as a stale one. - let stale = fixture - .listener - .publish( - "room/.accept/alice", - origin::Route::default().with_instance(Instance::random(), 0), - ) - .unwrap(); - let dial = socket::dial(&fixture.dialer, "room", "alice").unwrap(); - let mut dial = std::pin::pin!(dial); +async fn a_dial_waits_for_its_peer() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + assert!(matches!(alice.dial("alice", None), Err(Error::Unsupported))); + let mut dial = alice.dial("bob", None).unwrap(); assert!( - kio::clock::timeout(SETTLE, dial.as_mut()).await.is_err(), - "the dial took another dial's answer" + kio::clock::timeout(SETTLE, &mut dial).await.is_err(), + "a dial to nobody resolved" ); - drop(stale); - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let alice = kio::clock::timeout(TIMEOUT, listener.accept()) + let bob = fixture.endpoint(LITE, "bob").await; + let accept = async { bob.accept().await?.accept(None).await }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) .await - .expect("accept timeout") - .unwrap(); - assert_eq!(alice.id(), "alice"); - let dialer = kio::clock::timeout(TIMEOUT, dial).await.expect("dial timeout").unwrap(); - assert_eq!(dialer.id(), "alice"); - assert_eq!( - dialer.remote().info().instance, - alice.remote().info().instance, - "the answer carries the dial's instance" - ); - - // Another id is its own socket, never alice's. - let bob = fixture.dialer(4).await; - let (bob_dialer, bob_accepted) = connect(&bob, &mut listener, "bob").await; - assert_eq!(bob_dialer.id(), "bob"); - assert_eq!(bob_accepted.id(), "bob"); - assert_ne!(bob_accepted.remote().info().instance, alice.remote().info().instance); + .expect("connect timeout"); + assert_eq!(dialed.unwrap().session(), accepted.unwrap().session()); } -/// A stream is accepted only while its opener lists it: one whose producer the opener -/// dropped first, finished or not, is never handed out. +/// An accept of a dial claiming an endpoint nobody announces fails. #[kio::test] -async fn a_stream_is_accepted_only_while_listed() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - let gone = dialer.open_uni(None).unwrap(); - let done = dialer.open_uni(None).unwrap(); - let live = dialer.open_uni(None).unwrap(); - // Let the listener's listing hear of all three before any ends. - assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); - drop(gone); - done.finish().unwrap(); - drop(done); - // And of the ends. - assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); - - let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) +async fn a_dial_from_nobody_fails() { + let mut fixture = Fixture::new(); + let bob = fixture.endpoint(LITE, "bob").await; + let mallory = fixture.client(LITE).await; + let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.tcp/bob", None)) .await - .expect("accept timeout") + .expect("request timeout") + .expect("bob's endpoint"); + let forged = broadcast + .track("carol/0123456789abcdef0123456789abcdef") .unwrap() - .expect("a stream"); - assert_eq!(uni.name(), live.name()); - assert!( - kio::clock::timeout(SETTLE, accepted.accept_uni()).await.is_err(), - "a stream that left the listing was accepted" - ); - drop(live); -} - -/// Two concurrent bi opens are two streams, each answered on its own, and dropping the -/// socket ends it for the peer even while an open is pending. -#[kio::test] -async fn a_pending_open_neither_merges_nor_outlives_its_socket() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - let first = dialer.open_bi(None).unwrap(); - let second = dialer.open_bi(None).unwrap(); - let (first, second, answers) = kio::clock::timeout(TIMEOUT, async { - futures::join!(first, second, async { - (accepted.accept_bi().await, accepted.accept_bi().await) - }) - }) - .await - .expect("bi timeout"); - let (first, second) = (first.expect("first open"), second.expect("second open")); - let one = answers.0.expect("accept").expect("a stream"); - let two = answers.1.expect("accept").expect("a stream"); - assert_ne!(one.producer.name(), two.producer.name()); - send(&first.producer, b"one"); - send(&second.producer, b"two"); - assert_eq!(recv(&one.consumer).await, b"one"); - assert_eq!(recv(&two.consumer).await, b"two"); - send(&one.producer, b"one back"); - send(&two.producer, b"two back"); - assert_eq!(recv(&first.consumer).await, b"one back"); - assert_eq!(recv(&second.consumer).await, b"two back"); - - let pending = dialer.open_bi(None).unwrap(); - drop(dialer); - assert_eq!( - kio::clock::timeout(TIMEOUT, accepted.closed()) - .await - .expect("socket open"), - Closed::Remote - ); - drop(pending); -} - -/// Closing a socket ends its pending bi opens, both one the peer never answers and one -/// answered too late, and refuses new streams. -#[kio::test] -async fn closing_a_socket_ends_its_opens() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - let late = dialer.open_bi(None).unwrap(); - let stalled = dialer.open_bi(None).unwrap(); - // The peer answers the first and never the second, and keeps its broadcast up: the - // listener is not polled again, so nothing ends its socket. - let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) + .subscribe(None); + let incoming = kio::clock::timeout(TIMEOUT, bob.accept()) .await .expect("accept timeout") - .unwrap() - .expect("a stream"); - assert_eq!(answered.producer.name(), ".dial/bi/0", "the earlier open is answered"); - // Let the dialer's listing hear the answer before the close. - assert!(kio::clock::timeout(SETTLE, dialer.accept_bi()).await.is_err()); - - dialer.close(); - assert!( - matches!( - kio::clock::timeout(TIMEOUT, late).await.expect("late open"), - Err(moq_net::Error::Closed) - ), - "an answer after the close was taken" - ); + .unwrap(); + assert_eq!(incoming.peer(), "carol"); + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)) + .await + .expect("accept timeout"); + assert!(accepted.is_err(), "bob connected to nobody"); assert!( - matches!( - kio::clock::timeout(TIMEOUT, stalled).await.expect("stalled open"), - Err(moq_net::Error::Closed) - ), - "a stalled open outlived the close" + kio::clock::timeout(TIMEOUT, forged) + .await + .expect("forged timeout") + .is_err() ); - - assert!(matches!(dialer.open_uni(None), Err(moq_net::Error::Closed))); - assert!(matches!(dialer.open_bi(None), Err(moq_net::Error::Closed))); - assert!(matches!( - kio::clock::timeout(TIMEOUT, dialer.accept_uni()).await.expect("accept"), - Err(moq_net::Error::Closed) - )); - drop(answered); } -/// Each open is a new track, never an earlier one's: a held handle to an earlier stream -/// never reads a later one, and a bi open is answered while the peer still holds its -/// answer to an earlier one. +/// A third party asking bob for a session in alice's name surfaces at bob as a dial from +/// alice, but alice never dialed it, so she resets bob's accept, and the forger reads +/// nothing. #[kio::test] -async fn each_open_is_a_new_track() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - // Accepted and held without subscribing, then the opener moves on to a new stream. - let old = dialer.open_uni(None).unwrap(); - let held = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) - .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - drop(old); - let new = dialer.open_uni(None).unwrap(); - send(&new, b"new"); - let fresh = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) - .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - assert_ne!(fresh.name(), held.name()); - assert_eq!(recv(&fresh).await, b"new"); - let stale = kio::clock::timeout(SETTLE, async { - let mut subscriber = held.subscribe(None).await.ok()?; - let mut group = subscriber.recv_group().await.ok()??; - Some(group.read_frame().await.ok()??.payload.to_vec()) - }) - .await; - assert!( - !matches!(stale, Ok(Some(payload)) if payload == b"new"), - "the earlier stream's handle read the new one" - ); +async fn a_forged_dial_never_connects() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let mallory = fixture.client(LITE).await; - // A bi answer the peer keeps does not stop the next bi open. - let first = dialer.open_bi(None).unwrap(); - let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(first, accepted.accept_bi()) }) - .await - .expect("bi timeout"); - let _kept = answered.expect("accept").expect("a stream"); - drop(opened.expect("open")); - let again = dialer.open_bi(None).unwrap(); - let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(again, accepted.accept_bi()) }) + let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.tcp/bob", None)) .await - .expect("reopen timeout"); - let opened = opened.expect("open"); - let answered = answered.expect("accept").expect("a stream"); - send(&opened.producer, b"ping"); - assert_eq!(recv(&answered.consumer).await, b"ping"); - send(&answered.producer, b"pong"); - assert_eq!(recv(&opened.consumer).await, b"pong"); -} + .expect("request timeout") + .expect("bob's endpoint"); + let forged = broadcast + .track("alice/0123456789abcdef0123456789abcdef") + .unwrap() + .subscribe(None); -/// Open a bi stream from `dialer`, and have `accepted` accept it, write one group, and -/// finish its half before the open is polled, never reading the opener's half itself. -/// The opener still resolves and reads what the peer wrote. -async fn a_finished_answer_reaches_the_opener(dialer: &Socket, accepted: &Socket) { - let open = dialer.open_bi(None).unwrap(); - let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) + let incoming = kio::clock::timeout(TIMEOUT, bob.accept()) .await .expect("accept timeout") - .unwrap() - .expect("a stream"); - send(&answered.producer, b"greeting"); - answered.producer.finish().unwrap(); - // Held, finished, as any stream is until the peer reads it. - let answer = answered.producer; - assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); - - let opened = kio::clock::timeout(TIMEOUT, open) + .unwrap(); + assert_eq!(incoming.peer(), "alice"); + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)) .await - .expect("open timeout") - .expect("open"); - assert_eq!(recv(&opened.consumer).await, b"greeting"); - drop(answer); -} + .expect("accept timeout"); + assert!(accepted.is_err(), "bob connected to a forged dial"); + let forged = kio::clock::timeout(TIMEOUT, forged).await.expect("forged timeout"); + assert!(forged.is_err(), "the forger read bob's half"); -/// A finished answer reaches the opener through a relay. -#[kio::test] -async fn a_finished_answer_reaches_the_opener_through_a_relay() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - a_finished_answer_reaches_the_opener(&dialer, &accepted).await; + // Alice never heard of it as a dial either. + assert!(kio::clock::timeout(SETTLE, alice.accept()).await.is_err()); } -/// A finished answer reaches the opener on one origin, with no session between them. +/// A dial dropped before the peer accepts resets the accept. #[kio::test] -async fn a_finished_answer_reaches_the_opener_on_one_origin() { - let shared = origin(1); - let mut listener = Listener::new(&shared, "room").unwrap(); - let (dialer, accepted) = connect(&shared, &mut listener, "alice").await; - a_finished_answer_reaches_the_opener(&dialer, &accepted).await; +async fn a_dropped_dial_resets_the_accept() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let mut dial = alice.dial("bob", None).unwrap(); + // Polled until bob hears of it, then dropped. + let incoming = kio::clock::timeout(TIMEOUT, async { + match futures::future::select(std::pin::pin!(bob.accept()), &mut dial).await { + futures::future::Either::Left((incoming, _)) => incoming, + futures::future::Either::Right(_) => panic!("the dial resolved unaccepted"), + } + }) + .await + .expect("accept timeout") + .unwrap(); + drop(dial); + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)) + .await + .expect("accept timeout"); + assert!(accepted.is_err(), "bob connected to a dropped dial"); } -/// A peer that accepts a bi stream and drops its half at once, only reading, still -/// answers the open: its half left the listing before the opener could see it, and it -/// has not subscribed to the opener's half yet either. +/// Closing an endpoint aborts its sessions for the peer, and fails its pending work. #[kio::test] -async fn a_dropped_answer_still_answers_the_open() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - let open = dialer.open_bi(None).unwrap(); - let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) - .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - drop(answered.producer); - let request = answered.consumer; - assert!(kio::clock::timeout(SETTLE, accepted.accept_bi()).await.is_err()); +async fn closing_an_endpoint_ends_its_sessions() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (mut dialed, accepted) = connect(&alice, &bob).await; - let opened = kio::clock::timeout(TIMEOUT, open) - .await - .expect("open timeout") - .expect("open"); - send(&opened.producer, b"request"); - assert_eq!(recv(&request).await, b"request"); + bob.close(); + assert!( + recv(&mut dialed.consumer).await.is_err(), + "bob's half outlived his endpoint" + ); + assert!(accepted.producer.append_group().is_err()); + assert!(matches!(bob.accept().await, Err(Error::Closed))); + assert!(matches!(bob.dial("alice", None), Err(Error::Closed))); } -/// Closing or dropping a socket stops it reading the opener's half of the bi streams it -/// accepted, even while the opener keeps its half: the read that told the opener of the -/// accept is the socket's own, not the application's. +/// Session tracks are the secret that keeps third parties out, so no listing names them: +/// a reader allowed everything lists each endpoint's broadcast while a session runs and +/// finds no track. #[kio::test] -async fn closing_a_socket_releases_its_accepted_streams() { - for drop_it in [false, true] { - let shared = origin(1); - let mut listener = Listener::new(&shared, "room").unwrap(); - let (dialer, accepted) = connect(&shared, &mut listener, "alice").await; - - let open = dialer.open_bi(None).unwrap(); - let answered = kio::clock::timeout(TIMEOUT, accepted.accept_bi()) +async fn session_tracks_are_never_listed() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (dialed, mut accepted) = connect(&alice, &bob).await; + send(&dialed.producer, b"secret"); + assert_eq!(recv(&mut accepted.consumer).await.unwrap().unwrap(), b"secret"); + + let eve = fixture.client(LITE).await; + for id in ["alice", "bob"] { + let broadcast = kio::clock::timeout(TIMEOUT, eve.consume().request(format!("room/.tcp/{id}"), None)) .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - // The application is done with the stream; only the socket's own read is left. - drop(answered); - let opened = kio::clock::timeout(TIMEOUT, open) + .expect("request timeout") + .expect("the endpoint"); + let mut tracks = broadcast.tracks().unwrap(); + let event = kio::clock::timeout(TIMEOUT, tracks.next()) .await - .expect("open timeout") - .expect("open"); - let mut demand = opened.producer.demand(); - assert!(demand.wanted().is_some(), "the accept reads the opener's half"); - - match drop_it { - true => drop(accepted), - false => accepted.close(), - } - kio::clock::timeout(TIMEOUT, async { - while demand.wanted().is_some() { - demand.wanted_changed().await.unwrap(); - } - }) - .await - .expect("the closed socket still reads the opener's half"); + .expect("listing timeout") + .unwrap(); + assert!(matches!(event, Some(TracksEvent::Synced)), "{id} listed {event:?}"); + assert!( + kio::clock::timeout(SETTLE, tracks.next()).await.is_err(), + "{id} listed a track" + ); } } -/// A fixed-name track sits beside the streams: accept skips it, and the peer reads it by -/// name from [`Socket::remote`]. +/// An endpoint takes dials only from its first accept on, like a TCP listener: before it, +/// a dial is reset rather than left waiting. #[kio::test] -async fn fixed_tracks_sit_beside_streams() { - let fixture = Fixture::new().await; - let mut listener = Listener::new(&fixture.listener, "room").unwrap(); - let (dialer, accepted) = connect(&fixture.dialer, &mut listener, "alice").await; - - let command = dialer.local().create_track("command", None).unwrap(); - send(&command, b"press a"); - let stream = dialer.open_uni(None).unwrap(); - send(&stream, b"stream"); - - let uni = kio::clock::timeout(TIMEOUT, accepted.accept_uni()) +async fn an_endpoint_not_accepting_resets_dials() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let _bob = fixture.endpoint(LITE, "bob").await; + let dialed = kio::clock::timeout(TIMEOUT, alice.dial("bob", None).unwrap()) .await - .expect("accept timeout") - .unwrap() - .expect("a stream"); - assert_eq!(uni.name(), stream.name(), "accept skips the fixed track"); - assert_eq!(recv(&uni).await, b"stream"); - assert_eq!(recv(&accepted.remote().track("command").unwrap()).await, b"press a"); - assert!( - kio::clock::timeout(SETTLE, accepted.accept_uni()).await.is_err(), - "the fixed track was accepted as a stream" - ); + .expect("dial timeout"); + assert!(dialed.is_err(), "a dial to an endpoint not accepting connected"); } diff --git a/rs/moq-relay/tests/integration/socket_auth.rs b/rs/moq-relay/tests/integration/socket_auth.rs index 29746e74395..cb641faccb1 100644 --- a/rs/moq-relay/tests/integration/socket_auth.rs +++ b/rs/moq-relay/tests/integration/socket_auth.rs @@ -1,25 +1,30 @@ //! Sockets through a relay that admits with `moq-auth` tokens, verified by the reference //! auth server, granting only the prefixes `doc/concept/socket.md` lists. //! -//! The listener's token, rooted at the listener's path `room`, subscribes to `.dial/**` -//! and publishes `.accept/**`. A dialer's token publishes `.dial/` and subscribes to -//! `.accept/`, so it can neither dial as another id nor read another id's answer. +//! Every token is rooted at the socket prefix `room`. An endpoint's token publishes its own +//! endpoint, `.tcp/`, and subscribes to every endpoint, `.tcp/**`, so the session +//! track names are what keep third parties out: a forged dial never connects, and no +//! listing gives a session's name away. use std::time::Duration; -use moq_net::socket::{self, Closed, Listener, Socket}; +use moq_net::broadcast::TracksEvent; +use moq_net::socket::{Endpoint, Stream}; use crate::memory; const TIMEOUT: Duration = Duration::from_secs(10); -/// Long enough for a refused route to have arrived had it been allowed. The relay links +/// Long enough for a listing to have reported a track had it listed one. The relay links /// are in memory, so this is many round trips. const SETTLE: Duration = Duration::from_millis(500); -/// The listener's path, and every token's root. +/// The socket prefix, and every token's root. const ROOT: &str = "room"; +const LITE: &str = "moq-lite-07-wip"; +const IETF: &str = "moq-transport-18"; + /// A key, the reference auth server verifying tokens it signs, and a relay asking it. struct Fixture { key: moq_auth::Key, @@ -69,52 +74,57 @@ impl Fixture { } /// A client at [`ROOT`] presenting `token`, publishing and subscribing through one - /// origin, over moq-lite-07. - async fn client(&self, token: &str) -> (moq_net::origin::Producer, moq_net::Session) { + /// origin, over `version`. + async fn client(&self, token: &str, version: &str) -> (moq_net::origin::Producer, moq_net::Session) { let origin = moq_tokio::origin::spawn(); let url = format!("moql://relay.example.com/{ROOT}?jwt={token}").parse().unwrap(); let client = moq_net::Client::new() .with_publisher(origin.consume()) .with_subscriber(origin.clone()) .with_url(url); - let version = "moq-lite-07-wip".parse().expect("lite-07"); + let version = version.parse().expect("version"); let session = memory::connect(client, version, &self.cluster, &self.auth) .await .expect("connect"); (origin, session) } - async fn listener(&self) -> (moq_net::origin::Producer, moq_net::Session) { - self.client(&self.token(".accept/**", ".dial/**")).await - } - - async fn dialer(&self, id: &str) -> (moq_net::origin::Producer, moq_net::Session) { - let token = self.token(&format!(".dial/{id}"), &format!(".accept/{id}")); - self.client(&token).await + /// The endpoint `id`'s client, holding the grants `doc/concept/socket.md` lists: publish + /// its own endpoint, subscribe to every endpoint. + async fn endpoint(&self, id: &str, version: &str) -> (Endpoint, moq_net::origin::Producer, moq_net::Session) { + let (origin, session) = self + .client(&self.token(&format!(".tcp/{id}"), ".tcp/**"), version) + .await; + // The session is rooted at the prefix, so the endpoint's prefix is the root. + let endpoint = Endpoint::new(&origin, "", id).expect("endpoint"); + (endpoint, origin, session) } } -/// Dial as `id` from `origin` while `listener` accepts, returning both ends. -async fn connect(origin: &moq_net::origin::Producer, listener: &mut Listener, id: &str) -> (Socket, Socket) { - // The session is rooted at the listener's path, so the socket's path is the root. - let dial = socket::dial(origin, "", id).expect("dial"); - let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, listener.accept()) }) +/// Dial `acceptor` from `dialer` while `acceptor` accepts, returning both ends. +async fn connect(dialer: &Endpoint, acceptor: &Endpoint) -> (Stream, Stream) { + let dial = dialer.dial(acceptor.id(), None).expect("dial"); + let accept = async { + let incoming = acceptor.accept().await?; + assert_eq!(incoming.peer(), dialer.id()); + incoming.accept(None).await + }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) .await .expect("connect timeout"); (dialed.expect("dial"), accepted.expect("accept")) } -/// Write `payload` as one group of `track`. +/// Write `payload` as one message: one group of `track`. fn send(track: &moq_net::track::Producer, payload: &'static [u8]) { let mut group = track.append_group().expect("group"); group.write_frame(None, payload).expect("frame"); group.finish().expect("finish"); } -/// Subscribe to `track` and read its first frame. -async fn recv(track: &moq_net::track::Consumer) -> Vec { +/// Read the next message. +async fn recv(subscriber: &mut moq_net::track::Subscriber) -> Vec { kio::clock::timeout(TIMEOUT, async { - let mut subscriber = track.subscribe(None).await.expect("subscribe"); let mut group = subscriber.recv_group().await.expect("group").expect("a group"); let frame = group.read_frame().await.expect("frame").expect("a frame"); frame.payload.to_vec() @@ -123,56 +133,89 @@ async fn recv(track: &moq_net::track::Consumer) -> Vec { .expect("read timeout") } -/// The prefix grants admit a dial and its accept, and a bi stream carries data both -/// ways. A dialer holding another id's token can neither dial as that id, which would -/// replace its socket, nor read that id's answer. +/// The prefix grants admit a dial and its accept, and the session carries data both ways, +/// over moq-lite-07 and across a moq-transport hop. #[tokio::test] -async fn prefix_grants_scope_sockets_to_their_id() { - let fixture = Fixture::new().await; - let (listening, _listener_session) = fixture.listener().await; - let mut listener = Listener::new(&listening, "").expect("listen"); +async fn prefix_grants_admit_a_session() { + for (dialer, acceptor) in [(LITE, LITE), (LITE, IETF), (IETF, LITE)] { + let fixture = Fixture::new().await; + let (alice, _alice_origin, _alice_session) = fixture.endpoint("alice", dialer).await; + let (bob, _bob_origin, _bob_session) = fixture.endpoint("bob", acceptor).await; + let (mut dialed, mut accepted) = connect(&alice, &bob).await; + send(&dialed.producer, b"ping"); + assert_eq!(recv(&mut accepted.consumer).await, b"ping", "{dialer} to {acceptor}"); + send(&accepted.producer, b"pong"); + assert_eq!(recv(&mut dialed.consumer).await, b"pong", "{dialer} to {acceptor}"); + } +} - let (alice_origin, _alice_session) = fixture.dialer("alice").await; - let (alice, accepted) = connect(&alice_origin, &mut listener, "alice").await; - assert_eq!(accepted.id(), "alice"); +/// Mallory may read every endpoint, so she can ask bob for a session in alice's name. Bob +/// sees a dial from alice, but alice never dialed it and resets his accept, so it never +/// connects and mallory reads nothing. +#[tokio::test] +async fn a_forged_dial_never_connects() { + let fixture = Fixture::new().await; + let (alice, _alice_origin, _alice_session) = fixture.endpoint("alice", LITE).await; + let (bob, _bob_origin, _bob_session) = fixture.endpoint("bob", LITE).await; + let (_mallory, mallory_origin, _mallory_session) = fixture.endpoint("mallory", LITE).await; + // Alice keeps accepting, as a server would, which is how she resets the forged accept. + let alice_accepts = async { + let _ = alice.accept().await; + panic!("alice heard of a dial"); + }; + + let forge = async { + let broadcast = mallory_origin + .consume() + .request(".tcp/bob", None) + .await + .expect("bob's endpoint"); + let forged = broadcast + .track("alice/0123456789abcdef0123456789abcdef") + .expect("track") + .subscribe(None); + let incoming = bob.accept().await.expect("accept"); + assert_eq!(incoming.peer(), "alice"); + let (accepted, forged) = futures::join!(incoming.accept(None), forged); + assert!(accepted.is_err(), "bob connected to a forged dial"); + assert!(forged.is_err(), "mallory read bob's half"); + }; + kio::clock::timeout(TIMEOUT, async { + futures::select_biased! { + () = futures::FutureExt::fuse(Box::pin(forge)) => {} + () = futures::FutureExt::fuse(Box::pin(alice_accepts)) => {} + } + }) + .await + .expect("forge timeout"); +} - let open = alice.open_bi(None).expect("open"); - let (opened, answered) = kio::clock::timeout(TIMEOUT, async { futures::join!(open, accepted.accept_bi()) }) - .await - .expect("bi timeout"); - let opened = opened.expect("open"); - let answered = answered.expect("accept").expect("a stream"); - send(&opened.producer, b"ping"); - assert_eq!(recv(&answered.consumer).await, b"ping"); - send(&answered.producer, b"pong"); - assert_eq!(recv(&opened.consumer).await, b"pong"); - - // Mallory's token works for mallory's own socket. - let (mallory_origin, _mallory_session) = fixture.dialer("mallory").await; - let (_mallory, mallory_accepted) = connect(&mallory_origin, &mut listener, "mallory").await; - assert_eq!(mallory_accepted.id(), "mallory"); - - // It cannot read alice's answer. - let mut following = mallory_origin - .consume() - .follow(".accept/alice", None) - .expect("follow locally"); - let read = kio::clock::timeout(SETTLE, following.available()).await; - assert!(!matches!(read, Ok(Ok(_))), "mallory read alice's answer"); - - // Nor dial as alice. The dial is announced as it is made, so the listener would hear - // of it without the dial being polled, and would end alice's socket as replaced. - let dial = socket::dial(&mallory_origin, "", "alice").expect("dial locally"); - assert!( - kio::clock::timeout(SETTLE, listener.accept()).await.is_err(), - "the listener heard mallory dial as alice" - ); - assert!( - kio::clock::timeout(SETTLE, dial).await.is_err(), - "mallory's dial as alice resolved" - ); - for socket in [&alice, &accepted] { - let closed: Option = kio::clock::timeout(SETTLE, socket.closed()).await.ok(); - assert_eq!(closed, None, "alice's socket ended"); +/// A reader allowed every endpoint lists both ends of a running session and finds no +/// track, so it cannot learn a session's name to read or forge it. +#[tokio::test] +async fn a_reader_of_everything_lists_no_session() { + let fixture = Fixture::new().await; + let (alice, _alice_origin, _alice_session) = fixture.endpoint("alice", LITE).await; + let (bob, _bob_origin, _bob_session) = fixture.endpoint("bob", LITE).await; + let (dialed, mut accepted) = connect(&alice, &bob).await; + send(&dialed.producer, b"secret"); + assert_eq!(recv(&mut accepted.consumer).await, b"secret"); + + let (eve_origin, _eve_session) = fixture.client(&fixture.token(".tcp/eve", "**"), LITE).await; + for id in ["alice", "bob"] { + let broadcast = kio::clock::timeout(TIMEOUT, eve_origin.consume().request(format!(".tcp/{id}"), None)) + .await + .expect("request timeout") + .expect("the endpoint"); + let mut tracks = broadcast.tracks().expect("tracks"); + let event = kio::clock::timeout(TIMEOUT, tracks.next()) + .await + .expect("listing timeout") + .expect("listing"); + assert!(matches!(event, Some(TracksEvent::Synced)), "{id} listed {event:?}"); + assert!( + kio::clock::timeout(SETTLE, tracks.next()).await.is_err(), + "{id} listed a track" + ); } } From 58fc5be278d332ce9d9763d33fa73b09de01f3f7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 22:18:07 -0800 Subject: [PATCH 14/25] fix(net): reset and accept socket requests outside the endpoint lock Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/socket.rs | 75 ++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index de18db4893c..955524b1021 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -130,37 +130,41 @@ struct Shared { } impl Shared { - /// Queue every dial on hand, registering `waiter` for the next, and fail with the - /// endpoint's end once it stopped. - fn drain(&self, state: &mut State, waiter: &kio::Waiter) -> Result<(), Error> { + /// Queue every dial on hand and take the next, registering `waiter` for more, and + /// fail with the endpoint's end once it stopped. + /// + /// A dial of this endpoint's reserves its own track, so a request reaching the handler + /// under this endpoint's id names a session it never dialed, or one it gave up, such + /// as an accept of a dial forged in this endpoint's name: it is reset, which fails + /// that accept. Resets go out after the lock is released, since they wake readers. + fn poll_dial(&self, waiter: &kio::Waiter) -> Poll> { + let mut resets = Vec::new(); + let mut state = self.state.lock(); while state.end.is_none() { let requests = state.requests.get_or_insert_with(|| self.publication.requests()); match requests.poll_requested_track(waiter) { Poll::Pending => break, - Poll::Ready(Ok(request)) => self.sort(state, request), + Poll::Ready(Ok(request)) => match parse(request.name()) { + Some((dialer, _)) if dialer != &*self.id => state.incoming.push_back(request), + _ => resets.push(request), + }, Poll::Ready(Err(err)) => state.end = Some(err), } } - match &state.end { - Some(err) => Err(err.clone()), - None => Ok(()), - } - } - - /// Queue a peer's dial, or reset a request outside the scheme. - /// - /// A dial of this endpoint's reserves its own track, so a request reaching the handler - /// under this endpoint's id names a session it never dialed, or one it gave up: such - /// as an accept of a dial forged in this endpoint's name, which this reset fails. - fn sort(&self, state: &mut State, request: track::Request) { - match parse(request.name()) { - Some((dialer, _)) if dialer != &*self.id => state.incoming.push_back(request), - _ => { - // Not the name: a session's name is its secret. - tracing::debug!("reset a request for no session"); - request.reject(Error::NotFound); - } + let next = match &state.end { + Some(err) => Poll::Ready(Err(err.clone())), + None => match state.incoming.pop_front() { + Some(request) => Poll::Ready(Ok(request)), + None => Poll::Pending, + }, + }; + drop(state); + for request in resets { + // Not the name: a session's name is its secret. + tracing::debug!("reset a request for no session"); + request.reject(Error::NotFound); } + next } /// Poll for the endpoint's end, registering `waiter` for it. @@ -284,12 +288,7 @@ impl Endpoint { /// keep accepting, and accept or reject each one. pub fn poll_accept(&self, waiter: &kio::Waiter) -> Poll> { let shared = &self.shared; - let mut state = shared.state.lock(); - shared.drain(&mut state, waiter)?; - let Some(request) = state.incoming.pop_front() else { - return Poll::Pending; - }; - drop(state); + let request = ready!(shared.poll_dial(waiter))?; let (peer, session) = parse(request.name()).expect("sorted as a dial"); Poll::Ready(Ok(Incoming { shared: shared.clone(), @@ -504,8 +503,11 @@ impl Dialing { }; let mut state = shared.state.lock(); - if let Some(err) = &state.end { - return Poll::Ready(Err(err.clone())); + if let Some(err) = state.end.clone() { + // Aborted outside the lock. + drop(state); + let _ = producer.abort(err.clone()); + return Poll::Ready(Err(err)); } state.hold(&producer); drop(state); @@ -559,12 +561,15 @@ impl Accepting { let track = track_name(&self.peer, &self.session); ready!(self.step.poll(waiter, &track))?; - let mut state = shared.state.lock(); - if let Some(err) = &state.end { - return Poll::Ready(Err(err.clone())); - } + // Accepted outside the lock, since it wakes the dialer. let request = self.request.take().ok_or(Error::Closed)?; let producer = request.accept(self.info.take()); + let mut state = shared.state.lock(); + if let Some(err) = state.end.clone() { + drop(state); + let _ = producer.abort(err.clone()); + return Poll::Ready(Err(err)); + } state.hold(&producer); drop(state); Poll::Ready(Ok(Stream { From 2cc97ed56b4d4e7da61e6e9be4ac0d6d7ffa3f5f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 22:29:37 -0800 Subject: [PATCH 15/25] fix(net): read socket messages in sequence, reset handshakes explicitly `Stream::consumer` is now a `track::Ordered`, so messages are read in group sequence even when they arrive out of order. A dial, incoming dial, or accept that fails or is dropped now rejects or aborts its halves with the failure, or with `Cancel`, instead of leaving them to an implicit drop. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-net.md | 2 +- quest/m1/socket/README.md | 2 +- rs/moq-net/src/model/socket.rs | 102 ++++++++++++------ rs/moq-net/tests/integration/socket.rs | 31 +++++- rs/moq-relay/tests/integration/socket_auth.rs | 4 +- 5 files changed, 104 insertions(+), 37 deletions(-) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index d7cad9f07e9..1975561e69b 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.tcp/`. `dial(peer, info)` opens a session to another endpoint, and `accept()` yields the dials to this one, each a `socket::Stream`: this side's `track::Producer` and a reliable, ascending `track::Subscriber` of the peer's half, one message per group. Works over every moq-lite version and moq-transport; see [sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.tcp/`. `dial(peer, info)` opens a session to another endpoint, and `accept()` yields the dials to this one, each a `socket::Stream`: this side's `track::Producer` and a reliable, ascending, in-sequence `track::Ordered` of the peer's half, one message per group. Works over every moq-lite version and moq-transport; see [sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index e5a23a445d9..4c6addf361b 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -82,7 +82,7 @@ confirm (see the PR's Waiting section): session it never dialed: a forged dial surfaced at the claimed dialer as a dial from the victim and deadlocked both accepts. With the dialer's id leading both names, a request under an endpoint's own id is always an answer. -- `Stream::consumer` is a `track::Subscriber`, already subscribed reliably and +- `Stream::consumer` is a `track::Ordered`, already subscribed reliably and in order, not a bare `track::Consumer`, whose default subscription is lossy and newest first. The handshake's subscription becomes the stream's, so no message falls between the accept and the app's first read; `update` makes it diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 955524b1021..74e7a01c7fe 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -72,7 +72,7 @@ fn reliable() -> Subscription { /// One session: this side's half and the peer's. /// /// Each message is one group, so a slow message never holds up the next one, and messages -/// are ordered by group sequence. [`Self::consumer`] starts reliable and in order (oldest +/// are read in group sequence. [`Self::consumer`] starts reliable and in order (oldest /// first, with no delay budget); update its subscription to make it lossy. /// /// Finishing [`Self::producer`] closes the session for the peer, and aborting it is an @@ -80,8 +80,8 @@ fn reliable() -> Subscription { pub struct Stream { /// This side's half: the session's track in this endpoint's broadcast. pub producer: track::Producer, - /// The peer's half: the session's track in the peer's broadcast. - pub consumer: track::Subscriber, + /// The peer's half: the session's track in the peer's broadcast, read in sequence. + pub consumer: track::Ordered, peer: Arc, session: Arc, } @@ -187,11 +187,14 @@ impl Shared { drop(state); self.publication.close(); + // Reset outside the lock. for stream in streams { stream.abort(Error::Cancel); } - // Reset outside the lock. - drop((requests, incoming)); + for request in incoming { + request.reject(Error::Cancel); + } + drop(requests); } } @@ -269,7 +272,7 @@ impl Endpoint { .publication .reserve_unlisted_track(track_name(&shared.id, &session))?; // Followed rather than requested, so a peer not announced yet is waited for. - let resolving = Resolving::Follow(shared.origin.follow(path, None)?); + let resolving = Resolving::Follow(Box::new(shared.origin.follow(path, None)?)); Ok(kio::Pending::new(Dialing { shared: shared.clone(), peer: peer.into(), @@ -294,7 +297,7 @@ impl Endpoint { shared: shared.clone(), peer: peer.into(), session: session.into(), - request, + request: Some(request), })) } @@ -318,12 +321,13 @@ impl Drop for Endpoint { /// A dial from a peer, from [`Endpoint::accept`]: accept or reject it. /// -/// Dropping it rejects it. +/// Dropping it rejects it with [`Error::Cancel`]. pub struct Incoming { shared: Arc, peer: Arc, session: Arc, - request: track::Request, + /// The dialer's request for this side's half; taken by an accept or a reject. + request: Option, } impl Incoming { @@ -341,17 +345,14 @@ impl Incoming { /// Accept the dial, resolving once the dialer answers, which proves it dialed. /// /// `info` describes this side's half. Nothing is written before the dialer answers. - pub fn accept(self, info: impl Into>) -> kio::Pending { - let Self { - shared, - peer, - session, - request, - } = self; + pub fn accept(mut self, info: impl Into>) -> kio::Pending { + let shared = self.shared.clone(); + let (peer, session) = (self.peer.clone(), self.session.clone()); + let request = self.request.take(); // Requested, so a dial claiming an endpoint nobody announces fails at once: the // dialer announced its endpoint before it dialed. let resolving = endpoint_path(&shared.prefix, &peer) - .map(|path| Resolving::Request(shared.origin.request(path, None).into_inner())); + .map(|path| Resolving::Request(Box::new(shared.origin.request(path, None).into_inner()))); let step = match resolving { Ok(resolving) => Step::Resolving(resolving), Err(err) => Step::Failed(err), @@ -361,24 +362,34 @@ impl Incoming { peer, session, info: info.into(), - request: Some(request), + request, step, }) } /// Reject the dial, resetting the dialer's request with [`Error::NotFound`], as for a /// session that does not exist. - pub fn reject(self) { - self.request.reject(Error::NotFound); + pub fn reject(mut self) { + if let Some(request) = self.request.take() { + request.reject(Error::NotFound); + } + } +} + +impl Drop for Incoming { + fn drop(&mut self) { + if let Some(request) = self.request.take() { + request.reject(Error::Cancel); + } } } /// Resolving the peer's endpoint broadcast. enum Resolving { /// Failing once the endpoint is not announced. - Request(origin::Requesting), + Request(Box), /// Waiting for the endpoint to be announced. - Follow(origin::Following), + Follow(Box), } impl Resolving { @@ -397,7 +408,7 @@ enum Step { /// Subscribed to the peer's half, waiting for the peer to serve it. Subscribing(track::Subscribing), /// The peer serves its half. - Subscribed(track::Subscriber), + Subscribed(Box), /// It failed before it started. Failed(Error), /// Handed out, or failed. @@ -416,7 +427,7 @@ impl Step { } Self::Subscribing(subscribing) => { let subscriber = ready!(subscribing.poll_ok(waiter))?; - *self = Self::Subscribed(subscriber); + *self = Self::Subscribed(Box::new(subscriber)); } Self::Subscribed(_) => return Poll::Ready(Ok(())), Self::Failed(err) => return Poll::Ready(Err(err.clone())), @@ -427,7 +438,7 @@ impl Step { fn take_subscriber(&mut self) -> track::Subscriber { match std::mem::replace(self, Self::Done) { - Self::Subscribed(subscriber) => subscriber, + Self::Subscribed(subscriber) => *subscriber, _ => unreachable!("taken once subscribed"), } } @@ -441,6 +452,18 @@ enum Answer { Served(track::Producer), } +impl Answer { + /// Reset the half for the peer with `err`. + fn reset(self, err: Error) { + match self { + Self::Reserved(request) => request.reject(err), + Self::Served(producer) => { + let _ = producer.abort(err); + } + } + } +} + /// A dial, from [`Endpoint::dial`]: resolves once the peer accepts. /// /// Dropping it gives up, resetting the session. @@ -464,9 +487,10 @@ impl Dialing { /// Poll for the peer to accept; see [`Endpoint::dial`]. pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let result = self.poll_inner(waiter); - if let Poll::Ready(Err(_)) = &result { - // Resets the session for the peer. - self.answer = None; + if let Poll::Ready(Err(err)) = &result { + if let Some(answer) = self.answer.take() { + answer.reset(err.clone()); + } self.step = Step::Done; } result @@ -513,7 +537,7 @@ impl Dialing { drop(state); Poll::Ready(Ok(Stream { producer, - consumer: self.step.take_subscriber(), + consumer: self.step.take_subscriber().ordered(), peer: self.peer.clone(), session: self.session.clone(), })) @@ -528,6 +552,15 @@ impl kio::Task for Dialing { } } +impl Drop for Dialing { + fn drop(&mut self) { + // Given up: reset the session for the peer. + if let Some(answer) = self.answer.take() { + answer.reset(Error::Cancel); + } + } +} + /// An accept, from [`Incoming::accept`]: resolves once the dialer answers. pub struct Accepting { shared: Arc, @@ -574,7 +607,7 @@ impl Accepting { drop(state); Poll::Ready(Ok(Stream { producer, - consumer: self.step.take_subscriber(), + consumer: self.step.take_subscriber().ordered(), peer: self.peer.clone(), session: self.session.clone(), })) @@ -589,6 +622,15 @@ impl kio::Task for Accepting { } } +impl Drop for Accepting { + fn drop(&mut self) { + // Given up: reset the dialer's request. + if let Some(request) = self.request.take() { + request.reject(Error::Cancel); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 5600a252fcc..9b9757938d9 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -85,10 +85,10 @@ fn send(track: &track::Producer, payload: &'static [u8]) { group.finish().unwrap(); } -/// Read the next message from `subscriber`, or `None` once the peer finished. -async fn recv(subscriber: &mut track::Subscriber) -> Result>, Error> { +/// Read the next message from `reader`, or `None` once the peer finished. +async fn recv(reader: &mut track::Ordered) -> Result>, Error> { kio::clock::timeout(TIMEOUT, async { - let Some(mut group) = subscriber.recv_group().await? else { + let Some(mut group) = reader.next_group().await? else { return Ok(None); }; let frame = group.read_frame().await?.expect("a frame"); @@ -337,3 +337,28 @@ async fn an_endpoint_not_accepting_resets_dials() { .expect("dial timeout"); assert!(dialed.is_err(), "a dial to an endpoint not accepting connected"); } + +/// Messages are read in sequence even when they are written, and so arrive, out of order. +#[kio::test] +async fn messages_are_read_in_sequence() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (dialed, mut accepted) = connect(&alice, &bob).await; + + let write = |sequence: u64, payload: &'static [u8]| { + let mut group = dialed.producer.create_group(moq_net::group::Info { sequence }).unwrap(); + group.write_frame(None, payload).unwrap(); + group.finish().unwrap(); + }; + write(1, b"second"); + // Let the later message arrive first. + assert!( + kio::clock::timeout(SETTLE, accepted.consumer.next_group()) + .await + .is_err() + ); + write(0, b"first"); + assert_eq!(recv(&mut accepted.consumer).await.unwrap().unwrap(), b"first"); + assert_eq!(recv(&mut accepted.consumer).await.unwrap().unwrap(), b"second"); +} diff --git a/rs/moq-relay/tests/integration/socket_auth.rs b/rs/moq-relay/tests/integration/socket_auth.rs index cb641faccb1..af34d17cc9a 100644 --- a/rs/moq-relay/tests/integration/socket_auth.rs +++ b/rs/moq-relay/tests/integration/socket_auth.rs @@ -123,9 +123,9 @@ fn send(track: &moq_net::track::Producer, payload: &'static [u8]) { } /// Read the next message. -async fn recv(subscriber: &mut moq_net::track::Subscriber) -> Vec { +async fn recv(reader: &mut moq_net::track::Ordered) -> Vec { kio::clock::timeout(TIMEOUT, async { - let mut group = subscriber.recv_group().await.expect("group").expect("a group"); + let mut group = reader.next_group().await.expect("group").expect("a group"); let frame = group.read_frame().await.expect("frame").expect("a frame"); frame.payload.to_vec() }) From 5c76a1b76cff9598e8495406b0188c672bd1597f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 22:39:32 -0800 Subject: [PATCH 16/25] docs(net): say who ends each half of a socket session Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 3 ++- rs/moq-net/src/model/socket.rs | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 3490274df4e..76f348cd990 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -92,7 +92,8 @@ setting a delay budget on its subscription. Finishing a half closes the session that way, like a FIN. Resetting it is an error. An endpoint closing, or its broadcast ending, resets every session it -holds. +holds, and stops reading them. A side that sees the peer's half reset treats +the session as closed and resets its own half, so both halves end. There is no multiplexing within a session: another channel is another session. Several sessions between the same two endpoints are independent, and a diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 74e7a01c7fe..b5ac3eb074e 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -76,7 +76,8 @@ fn reliable() -> Subscription { /// first, with no delay budget); update its subscription to make it lossy. /// /// Finishing [`Self::producer`] closes the session for the peer, and aborting it is an -/// error. The endpoint closing aborts it too. +/// error. The endpoint closing aborts it too. When the peer's half ends with an error, +/// abort this side's half as well, and drop the stream, so both halves end. pub struct Stream { /// This side's half: the session's track in this endpoint's broadcast. pub producer: track::Producer, @@ -204,7 +205,9 @@ impl Shared { /// It accepts dials from its first [`Self::accept`] on, like a TCP listener: before that, a /// dial to it is reset, and after it, dials queue until accepted. Dropping it, or /// [`Self::close`], retracts the broadcast, resets the dials waiting to be accepted, and -/// aborts every session it handed out. +/// aborts this side's half of every session it handed out. The peer's half is read +/// through each [`Stream::consumer`], which the application owns: drop the streams too, +/// which releases the subscriptions. pub struct Endpoint { shared: Arc, } From 274d411e5aaa1fa9ed610c58caa588fe34457023 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 23:25:41 -0800 Subject: [PATCH 17/25] feat(net)!: socket endpoints at .socket/ with tcp and track protocols A tcp session half is one group: frames are writes, finishing is FIN, resetting is RST, and a second group resets the session. A track session hands out the whole tracks, groups as messages. Both share the dial, accept, reject, and return-routability handshake, named // under one hidden .socket/ endpoint. Co-Authored-By: Claude Opus 5.5 --- doc/concept/index.md | 2 +- doc/concept/socket.md | 150 ++-- doc/lib/rs/moq-net.md | 2 +- quest/m1/socket/README.md | 174 ++-- quest/m1/socket/boy.md | 6 +- quest/m1/socket/carla.md | 4 +- quest/m1/socket/js.md | 28 +- quest/m1/socket/py.md | 14 +- quest/m2/mesh/README.md | 5 +- quest/m2/mesh/endpoint.md | 9 +- quest/m2/stream/README.md | 19 +- quest/m2/stream/rust.md | 8 +- rs/moq-net/CHANGELOG.md | 2 +- rs/moq-net/src/model/origin.rs | 6 + rs/moq-net/src/model/socket.rs | 817 ++++++++++++++---- rs/moq-net/tests/integration/socket.rs | 506 ++++++++--- rs/moq-relay/tests/integration/socket_auth.rs | 91 +- 17 files changed, 1376 insertions(+), 467 deletions(-) diff --git a/doc/concept/index.md b/doc/concept/index.md index 3009b31f92c..1b1884eaca3 100644 --- a/doc/concept/index.md +++ b/doc/concept/index.md @@ -38,7 +38,7 @@ watched at 100 ms by one viewer and 10 s by another. - [Transport](/concept/transport): why QUIC, and the WebTransport, WebSocket, and raw QUIC paths onto it. - [moq-lite](/concept/moq-lite): the pub/sub protocol, discovery, path patterns, subscriptions, and congestion behavior. - [Identity and recovery](/concept/identity): choosing stream instances, following restarts, and answering work on demand without mixing streams. -- [Sockets](/concept/socket): dial and accept point-to-point sessions through relays, each a reliable, ordered pipe of messages. +- [Sockets](/concept/socket): dial and accept point-to-point sessions through relays, as reliable byte pipes or as tracks of messages. - [hang](/concept/hang): the media catalog, containers, encryption, and how to extend both. - [Archives](/concept/archives): record listed tracks into object storage and serve them back from a host. - [Audio jitter](/concept/audio-jitter): how a receiver sizes its audio playout target from arrival timing. diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 76f348cd990..16feb3cb73d 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -1,17 +1,18 @@ --- title: Sockets -description: Dial and accept point-to-point sessions over MoQ, each a reliable, ordered pipe of messages +description: Dial and accept point-to-point sessions over MoQ, as reliable byte pipes or as tracks of messages --- # Sockets A socket connects two endpoints through relays, the way a TCP connection connects two hosts. Each endpoint announces one hidden broadcast, dials other -endpoints, and accepts their dials. Each connection is a session: a reliable, -ordered, duplex pipe of messages that only the two ends can read. Use it for -per-client state that only the two ends need, such as a viewer's commands and -the replies to them, while whatever every client needs stays in an ordinary -broadcast. +endpoints, and accepts their dials. Each connection is a session that only the +two ends can read, in one of two protocols: `tcp`, a reliable, ordered byte +pipe each way, or `track`, a track each way whose groups are messages. Use a +socket for per-client state that only the two ends need, such as a viewer's +commands and the replies to them, while whatever every client needs stays in an +ordinary broadcast. A socket is a convention over announcements, broadcasts, and track subscriptions, not a wire extension. It uses no instances and no track @@ -26,47 +27,53 @@ it in Rust. An endpoint with id `` announces one broadcast, once: ```text -/.tcp/ +/.socket/ ``` The id is one path segment: a well-known name for a server, such as a game, or a random id for a client. The prefix is shared by every endpoint that talks to -each other. The `.tcp` segment starts with `.`, so endpoints are -[hidden](/concept/moq-lite#hidden-broadcasts) from anyone listing the prefix, -and it names the mode: one reliable, ordered pipe per session, as TCP gives. +each other. The `.socket` segment starts with `.`, so endpoints are +[hidden](/concept/moq-lite#hidden-broadcasts) from anyone listing the prefix. -Connections are never announced. An endpoint's broadcast holds only session -tracks, served on request, and its announcement is the only thing a relay -routes. +The announcement is used only for routing: it is how a relay finds an +endpoint. Connections are never announced. An endpoint's broadcast holds only +session tracks, served on request, whatever their protocol. ## Sessions -A session between a dialer `A` and an acceptor `B` is named by a session id -`S` that `A` picks: random and unguessable, at least 128 bits, such as 32 hex -digits. Both halves of the session are tracks with the same name, -`/`, one in each endpoint's broadcast: +A session between a dialer `A` and an acceptor `B` is named by its protocol `P` +and a session id `S` that `A` picks: random and unguessable, at least 128 bits, +such as 32 hex digits. Both halves of the session are tracks with the same +name, `//`, one in each endpoint's broadcast: | Track | In broadcast | Written by | Read by | | --- | --- | --- | --- | -| `A/S` | `/.tcp/A` | `A` | `B` | -| `A/S` | `/.tcp/B` | `B` | `A` | +| `P/A/S` | `/.socket/A` | `A` | `B` | +| `P/A/S` | `/.socket/B` | `B` | `A` | -The dialer's id leads the name in both broadcasts, so an endpoint tells a dial -from an answer by the name alone: a request under another endpoint's id is that -endpoint dialing, and a request under its own id answers one of its own dials. +The protocol is `tcp` or `track`; see [Protocols](#protocols). An endpoint +resets a request whose protocol it doesn't speak, or isn't accepting, as +unsupported. + +The dialer's id follows the protocol in both broadcasts, so an endpoint tells a +dial from an answer by the name alone: a request under another endpoint's id is +that endpoint dialing, and a request under its own id answers one of its own +dials. Since `S` is random per dial, every session is distinct, and one +endpoint can dial and accept any number of sessions at once, with the same peer +or others. ### Dial -1. `A` picks `S` and holds its own half, `A/S` in its broadcast, unanswered. -2. `A` subscribes to `/.tcp/B` for the track `A/S`, waiting for `B`'s - endpoint to be announced if it is not yet. +1. `A` picks `S` and holds its own half, `P/A/S` in its broadcast, unanswered. +2. `A` subscribes to `/.socket/B` for the track `P/A/S`, waiting for + `B`'s endpoint to be announced if it is not yet. 3. `B` receives that subscription as a request for a track it does not have: - an incoming dial, claiming to be from `A`, with session `S`. + an incoming dial in protocol `P`, claiming to be from `A`, with session `S`. ### Accept -1. `B` subscribes to `/.tcp/A` for the track `A/S`, and holds `A`'s - subscription unanswered. +1. `B` subscribes to `/.socket/A` for the track `P/A/S`, and holds + `A`'s subscription unanswered. 2. `A` answers that subscription only if it dialed `S` and is still waiting, and resets any other. That makes the exchange a return-routability check: a dial forged in `A`'s name, by anyone who can subscribe to `B`, reaches an @@ -78,22 +85,67 @@ endpoint dialing, and a request under its own id answers one of its own dials. ### Reject `B` resets `A`'s subscription, with not found, as for a session that does not -exist. An endpoint that is not accepting dials resets them the same way. A dial -that gives up resets `B`'s subscription in turn. +exist. An endpoint resets a dial in a protocol it is not accepting as +unsupported, and one that accepts nothing at all answers not found, as for any +track nobody serves. A dial that gives up resets `B`'s subscription in turn. + +## Protocols + +Each side writes its own half and reads the peer's. A reader subscribes +reliably and in order: oldest group first, with no delay budget, so it never +skips anything. The protocol decides what a half holds. + +### `tcp` + +Each half is exactly one group, sequence 0. In MoQ a group travels as one QUIC +stream on every hop, so it is reliable, strictly ordered, and flow controlled, +exactly like one direction of a TCP connection: + +- A frame is a write. The reader reads the frames in the order they were + written, with nothing skipped. +- Finishing the group is FIN: the reader reads to the end, then sees the end. +- Resetting the group, or its track, is RST: the reader fails with the error. +- A second group on a session track is a protocol error: the reader resets the + session, both halves. + +There are no track-level settings: no priority between messages, no delay +budget, no order, since the newest group is never stale. + +A session can last indefinitely: a group has no size cap, and each copy keeps +only a bounded tail of it. A writer that waits for room before each write (in +Rust, `group::Producer::ready`) runs at most a window ahead of the reader on +every hop, so a slow reader slows the writer, as TCP backpressure does, and +nothing is lost. A writer that doesn't wait sheds the front of its half, and a +reader that falls behind it fails with an eviction error instead of skipping +data. Pacing reaches across native QUIC and qmux hops; see +[unbounded groups](https://github.com/moq-dev/moq/tree/main/quest/m1/unbounded-groups) +for what remains, such as browsers, which read without pacing. + +Use `tcp` for a byte stream or a log of records that must all arrive in order, +such as a proxied TCP connection, a file transfer, or a request and its reply. + +### `track` + +Each half is a whole track, and a message is one group. A slow message never +holds up the next, and each group can carry its own priority. Messages are +ordered by group sequence. + +The reader's subscription starts reliable and in order, like `tcp`. An +application may make it lossy for messages that go stale, such as positions or +controls, by updating the subscription with a finite delay budget: a message +that is late past the budget is skipped rather than holding up later ones. It +may also read newest first. + +Finishing a half closes the session that way. Resetting it is an error. -## Messages +Use `track` for independent messages, where a late one should not block the +next: commands, controls, acknowledgements, or telemetry. -Each side writes its own half and reads the peer's. A message is one group, so a -slow message never holds up the next, and messages are ordered by group -sequence. A reader reads reliably and in order by default: oldest group first, -with no delay budget, so it never skips a message. An application may opt into a -lossy budget for a session whose messages go stale, such as positions, by -setting a delay budget on its subscription. +## Closing -Finishing a half closes the session that way, like a FIN. Resetting it is an -error. An endpoint closing, or its broadcast ending, resets every session it -holds, and stops reading them. A side that sees the peer's half reset treats -the session as closed and resets its own half, so both halves end. +An endpoint closing, or its broadcast ending, resets every session it holds, +and stops reading them. A side that sees the peer's half reset treats the +session as closed and resets its own half, so both halves end. There is no multiplexing within a session: another channel is another session. Several sessions between the same two endpoints are independent, and a @@ -121,19 +173,19 @@ socket prefix, each grant is a prefix: ```jsonc // The endpoint : publishes its own endpoint, reads every endpoint. -{ "root": "", "publish": [".tcp/"], "subscribe": [".tcp/**"] } +{ "root": "", "publish": [".socket/"], "subscribe": [".socket/**"] } ``` The publish grant pins the id, so no endpoint can announce as another. The subscribe grant lets it reach any endpoint, which is what dialing and accepting need; the session id keeps it out of other endpoints' sessions. A narrower -subscribe grant, such as a client allowed only `.tcp/`, also works: it -can then dial only that server and accept only its dials. +subscribe grant, such as a client allowed only `.socket/`, also works: +it can then dial only that server and accept only its dials. `moq auth sign` writes this as, for example: ```bash -moq auth sign --key key.jwk --root "$PREFIX" --publish ".tcp/$ID" --subscribe '.tcp/**' > endpoint.jwt +moq auth sign --key key.jwk --root "$PREFIX" --publish ".socket/$ID" --subscribe '.socket/**' > endpoint.jwt ``` Each client then connects to the relay at the prefix with its token, and its @@ -145,7 +197,7 @@ planned mesh layer makes ids public keys. ## Later -A multiplexed mode, with many streams per session like QUIC, would be its own -endpoint, `/.quic/`, beside `.tcp`, and could name its streams -beneath the session, such as a control stream under `//`. A -`.tcp` session name holds no `/` after the session id, so it leaves that room. +Later protocols are other prefixes on the same endpoint, with no new +announcement. A multiplexed `quic` mode, with many streams per session like +QUIC, would name its sessions `quic//` and bring back several +groups or streams per session. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 1975561e69b..b635412e44e 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.tcp/`. `dial(peer, info)` opens a session to another endpoint, and `accept()` yields the dials to this one, each a `socket::Stream`: this side's `track::Producer` and a reliable, ascending, in-sequence `track::Ordered` of the peer's half, one message per group. Works over every moq-lite version and moq-transport; see [sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept: `dial_tcp(peer)` / `accept_tcp()` yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` / `accept_track()` yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index 4c6addf361b..9d2763fa010 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -4,8 +4,8 @@ `moq-net` gains a generic dial/accept primitive. An `Endpoint` announces one hidden broadcast, dials other endpoints, and accepts their dials, and each -connection is a session: one reliable, ordered pipe of messages between the -two ends. moq-boy, CARLA, [byte streams](/quest/m2/stream/README.md), and the +connection is a session between the two ends: a reliable, ordered byte pipe +each way (`tcp`), or a track of messages each way (`track`). moq-boy, CARLA, [byte streams](/quest/m2/stream/README.md), and the mesh [Endpoint](/quest/m2/mesh/endpoint.md) all build on it instead of each hand-rolling the announce, watch, and request loop. Rust, JS, and Python (through moq-ffi) expose the same surface and interoperate. @@ -21,45 +21,103 @@ lease and 1 s watchdog); signing and encryption, which the m2 [`doc/concept/socket.md`](/doc/concept/socket.md) is the convention JS and Python follow, and `rs/moq-net/src/model/socket.rs` implements it. -Decided 2026-11-03 (maintainer, /quest-grill in /quest-complete), on -[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677), replacing the -listener layout and QUIC-style streams below: - -- Endpoint sockets, one pipe per session. Each endpoint announces one hidden - broadcast, `/.tcp/`, once, and connections are never announced. - `` is a well-known name for a server (a game) or a random id for a - client. Why: the old flow had the listener answer `L/.accept/` under the - dialer's instance, which broke instance uniqueness (lite-07 treats routes - sharing an instance as replicas), let two listeners accept one dial, - announced every connection publicly, and needed lite-07 instances and track - listings on every hop. -- The segment is `.tcp`, not `.socket`, so the path names the protocol: one - reliable, ordered pipe per session. A multiplexed mode would be its own - `.quic/` endpoint later. The Rust and JS module stays `socket` - (`socket::Endpoint`, `socket::Stream`), since `.tcp` and a later `.quic` are - modes of one socket API. -- Dial: `A` picks a random, unguessable session id `S` (128 bits) and - subscribes to `.tcp/`; `B` surfaces the requested track as an incoming - dial claiming to be from `A`. Accept: `B` subscribes to `.tcp/`, and `A` - answers only a session it dialed and resets any other, a return-routability - check that fails a dial forged in `A`'s name. `B` sends nothing until `A` - answers. Reject: `B` resets `A`'s subscription. -- The pipe is one track each way. One message per group, so messages don't - block each other, ordered by group sequence; reliable and in order by - default (no delay budget, ascending), and the app may opt into a lossy - budget. Finishing a track closes the pipe, resetting it is an error, and the - endpoint's broadcast ending disconnects. -- No multiplexing: another channel is another session, a pair of endpoints - holds several, and a restart is a new session. No control track and no - numbered streams now. +Decided 2026-11-04 (maintainer, /quest-complete), on +[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677), settling its +Waiting items and replacing the `.tcp/` layout and the one-message-per-group +pipe below: + +- Keep `/` session naming. A dial and an answer can't be confused, + the claimed dialer is named for the return-routability check, and `S` + (random per dial) makes every session unique, so one endpoint can dial and + accept any number of sessions. Not chosen: reader-named halves. +- One announcement per endpoint, used only for routing: + `/.socket/`. The session track name carries the protocol, the + dialer and the session: `//`. A dial is SUBSCRIBE + `/.socket/`, track `tcp//`; the answer is the same + name in the dialer's broadcast. A requested track whose protocol the + endpoint doesn't speak, or isn't accepting, is reset as unsupported. Grants: + publish `.socket/`, subscribe `.socket/**`. The module stays `socket`. +- Two protocols ship now over the same handshake, each with its own dial and + accept, and an incoming dial lands in its protocol's queue: + - `tcp//`: a session half is a single group, not a track of + messages. A group is one QUIC stream on every hop, so it's reliable, + strictly ordered and flow controlled, like a TCP direction. Frames are + writes, finishing the group is FIN, and resetting it is RST. There are no + track-level settings (budget, max-delay, order), since the newest group is + never stale. A second group on a session track is a protocol error that + resets the session. Rust: `Endpoint::dial_tcp` / `accept_tcp`, yielding + `socket::TcpStream { producer: group::Producer, consumer: group::Consumer }` + plus `peer()` and `session()`. Not chosen: `track::Ordered` with one message + per group. + - `track//`: the same dial, accept, reject and return-routability + check, but the app gets the whole session tracks, with groups as messages + and its own priority, budget and order, and no single-group rule. The + reader starts reliable and ascending (no delay budget) until the app + updates it. Rust: `Endpoint::dial_track` / `accept_track`, yielding + `socket::TrackStream { producer: track::Producer, consumer: track::Subscriber }`. +- Long-lived `tcp` sessions rely on unbounded groups (since + [moq-dev/moq#7411](https://github.com/moq-dev/moq/pull/7411) a group has no + size cap, and each copy keeps a ~32 MiB tail). With one reader per copy, + [fastest-reader pacing](/quest/m1/unbounded-groups/fastest-reader.md) acts + as TCP backpressure: a writer awaiting `group::Producer::ready` runs at most + 16 MiB past its reader on each hop, and a reader that falls off the tail + fails with `Evicted` rather than skipping data. Where pacing doesn't reach + yet, that `Evicted` is the failure mode, never silent loss: browsers and JS + read unpaced ([JS fastest-reader pacing](/quest/m2/js-fastest-reader.md)), + and a session past its paused-credit cap (about two paused group streams at + the client defaults) reads and sheds instead of pausing. That remaining + pacing work stays in the [unbounded groups](/quest/m1/unbounded-groups/README.md) + epic, which this depends on for lossless `tcp` in those cases. + +Implementation choices in +[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677): + +- A `tcp` session runs a small task on the endpoint origin's driver. It copies + the peer's group into the group the app reads, a frame at a time and paced by + that reader, so it can end that group itself: on a second group from the + peer, on the endpoint closing, or when the peer resets. It also finishes or + aborts this side's track the way the app ended its group, and resets the + session when the app drops its reader early. +- `TrackStream::consumer` is the handshake's own `track::Subscriber`, not a bare + `track::Consumer`: it is already subscribed reliably and in order, so no + message falls between the accept and the app's first read, and the app + overrides it with `update` or reads in sequence through `ordered()`. +- An endpoint takes dials in a protocol from its first accept in it, like a TCP + listener. Until its first accept in any protocol it has no request handler, + so a dial to it is reset with not found at once. A dial reserves its own + half, which the peer's answer finds without a handler, so a dial-only + endpoint never needs polling. A dial waits for the peer's endpoint to be + announced (drop it to give up); an accept fails at once when the claimed + dialer's endpoint is not announced. +- Accepted dials queue without limit while the app accepts, each holding the + dialer's subscription open. - Session track names are the secret, so they never appear in a lite-07 track - listing or anywhere else a third party reads. moq-net already lists only - tracks a producer creates or a handler's `Lister` answers, never one served - on request, so an endpoint serves its session tracks only on request. The + listing or anywhere else a third party reads. moq-net lists only tracks a + producer creates or a handler's `Lister` answers, never one served on + request, so an endpoint serves its session tracks only on request, and the dialer's own half is reserved unlisted (`broadcast::Producer::reserve_unlisted_track`, crate-private). -- Grants are prefix only, rooted at the prefix: publish `.tcp/`, - subscribe `.tcp/**`, or narrower subscribes. A relay test signs real + +Decided 2026-11-03 (maintainer, /quest-grill in /quest-complete), still +standing where the 2026-11-04 decisions above don't replace it: + +- Endpoint sockets, one session per connection. Each endpoint announces one + hidden broadcast once, and connections are never announced. `` is a + well-known name for a server (a game) or a random id for a client. Why: the + old flow had the listener answer `L/.accept/` under the dialer's + instance, which broke instance uniqueness (lite-07 treats routes sharing an + instance as replicas), let two listeners accept one dial, announced every + connection publicly, and needed lite-07 instances and track listings on + every hop. +- Dial: `A` picks a random, unguessable session id `S` (128 bits) and + subscribes to `B`'s endpoint; `B` surfaces the requested track as an + incoming dial claiming to be from `A`. Accept: `B` subscribes to `A`'s + endpoint, and `A` answers only a session it dialed and resets any other, a + return-routability check that fails a dial forged in `A`'s name. `B` sends + nothing until `A` answers. Reject: `B` resets `A`'s subscription. +- No multiplexing: another channel is another session, a pair of endpoints + holds several, and a restart is a new session. +- Grants are prefix only, rooted at the prefix. A relay test signs real `moq-auth` tokens. - It works over every moq-lite version and moq-transport, since the handshake uses no instances or listings; lite-07 instances only affect relay route @@ -67,35 +125,11 @@ listener layout and QUIC-style streams below: - Fixed channels become sessions or messages on one: boy's `command` and per-viewer latency, CARLA's `control`, ack, and e-stop. Shared state such as boy's `status` stays an ordinary broadcast. -- `socket::Stream` is a plain struct with public `producer` (this side's half) +- The stream types are plain structs with public `producer` (this side's half) and `consumer` (the peer's half) fields, plus read-only `peer()` and - `session()`, and no read, write, or message methods of its own: apps use the - track API directly (groups as messages, priority, budget), so JS, Python, and - the bindings mirror a minimal surface. - -Settled by the Rust implementation of that decision, for the maintainer to -confirm (see the PR's Waiting section): - -- Both halves are named `/`, not each after its reader. With - reader-named halves, an endpoint that both dials and accepts cannot tell a - peer's dial from a peer's answer, so it could not reset an answer to a - session it never dialed: a forged dial surfaced at the claimed dialer as a - dial from the victim and deadlocked both accepts. With the dialer's id - leading both names, a request under an endpoint's own id is always an answer. -- `Stream::consumer` is a `track::Ordered`, already subscribed reliably and - in order, not a bare `track::Consumer`, whose default subscription is lossy - and newest first. The handshake's subscription becomes the stream's, so no - message falls between the accept and the app's first read; `update` makes it - lossy. -- An endpoint takes dials from its first `accept` on, like a TCP listener; until - then it has no request handler, so a dial to it is reset at once. A dial - reserves its own half, which the peer's answer finds without a handler, so a - dial-only endpoint never needs polling. A dial waits for the peer's endpoint - to be announced (drop it to give up); an accept fails at once when the - claimed dialer's endpoint is not announced. -- Accepted dials queue without limit while the app accepts, each holding the - dialer's subscription open. Announce backpressure is no longer a socket - concern, since connections are never announced. + `session()`, and no read, write, or message methods of their own: apps use + the group and track APIs directly, so JS, Python, and the bindings mirror a + minimal surface. Decided 2026-11-02 (maintainer, Kixel planning): the whole epic moves from m2 to m1, so the next cut publishes sockets in `@moq/net` and moq.pro's @@ -115,10 +149,10 @@ replacement arc), still standing: `doc/concept/socket.md`, not in the moq-lite draft: it is an application convention over moq-lite. -Later, not built now: a multiplexed `.quic/` mode with many streams per -session, which could name a control stream and numbered streams beneath the -session (`//...`); and signed or encrypted sessions with ids -as public keys, in the m2 [mesh](/quest/m2/mesh/README.md). +Later, not built now: a multiplexed `quic` protocol, `quic//` on +the same endpoint with no new announcement, which brings back several groups +or streams per session; and signed or encrypted sessions with ids as public +keys, in the m2 [mesh](/quest/m2/mesh/README.md). This epic owns removing the hand-rolled loops: after the ports, no announce-watch-request pairing remains in `rs/moq-boy`, `js/moq-boy`, or diff --git a/quest/m1/socket/boy.md b/quest/m1/socket/boy.md index 6460b1d50c4..b7c484b96ce 100644 --- a/quest/m1/socket/boy.md +++ b/quest/m1/socket/boy.md @@ -15,9 +15,11 @@ See the [epic](/quest/m1/socket/README.md) for the convention. - The emulator runs an endpoint named for its game, and each viewer an endpoint with its random id that dials it. Settle and record the prefix - (such as `boy`, so endpoints sit at `boy/.tcp/`). + (such as `boy`, so endpoints sit at `boy/.socket/`). - The viewer's `command` track at `{viewerPrefix}/{name}/{viewerId}` becomes - the viewer's messages on its session, one command per group. Drop the + the viewer's messages on a `track` session, one command per group: commands + are independent messages, and a stale one shouldn't hold up the next, which + a `tcp` session's single group would. Drop the separate viewer-prefix configuration in `rs/moq-boy/src/main.rs`. - Per-viewer latency moves from `status`'s `latency` map to the emulator's messages on that viewer's session. Game state stays in the ordinary diff --git a/quest/m1/socket/carla.md b/quest/m1/socket/carla.md index 97385e996e2..2815c6c0f69 100644 --- a/quest/m1/socket/carla.md +++ b/quest/m1/socket/carla.md @@ -16,7 +16,9 @@ See the [epic](/quest/m1/socket/README.md) for the convention. - The worker (`publish.py`) runs a Python endpoint per truck; `paths.ts` and `truck.ts` dial it from an endpoint with the per-lease driver id. A new lease is a new session, which matches today's lease binding. -- The `control` track becomes the driver's messages on its session. The +- The `control` track becomes the driver's messages on a `track` session, one + control per group, so a late control never blocks a newer one; the driver + may read the truck's replies with a finite delay budget. The driver's ack, sequence, and round-trip echo move from the shared `status` to the truck's messages on that session. `driver` and `estop_by` stay in the ordinary `status` broadcast, because every viewer shows them. diff --git a/quest/m1/socket/js.md b/quest/m1/socket/js.md index a0de86d5c15..8d3294863f5 100644 --- a/quest/m1/socket/js.md +++ b/quest/m1/socket/js.md @@ -2,18 +2,26 @@ ## Goal -`@moq/net` exports `socket` with the same `Endpoint` and `Stream` surface as -Rust, and a Rust endpoint and a JS endpoint dial and accept sessions with each -other through a relay, carrying messages both ways. +`@moq/net` exports `socket` with the same `Endpoint`, `TcpStream`, and +`TrackStream` surface as Rust, and a Rust endpoint and a JS endpoint dial and +accept sessions in both protocols with each other through a relay, carrying +data both ways. ## Plan Follow `doc/concept/socket.md` and mirror the Rust surface (`rs/moq-net/src/model/socket.rs`): an `Endpoint` announcing -`/.tcp/`, `dial(peer)` and `accept()` in JS's casing, an incoming -dial to accept or reject, and a `Stream` with `producer` and `consumer` fields -plus `peer` and `session`, and no message methods of its own. Session tracks -are `/` in both endpoints, served only on request so no -listing names them, and the reader subscribes reliably and in order. Add the -Rust/JS pair to `just test interop`, and document the export in -`doc/lib/js/net.md`. +`/.socket/`; `dialTcp(peer)` / `acceptTcp()` and +`dialTrack(peer)` / `acceptTrack()` in JS's casing, each landing in its +protocol's queue; an incoming dial to accept or reject; a `TcpStream` with a +group `producer` and `consumer` (one group each way: frames are writes, finish +is FIN, abort is RST, and a second group resets the session); and a +`TrackStream` with a track `producer` and a reliably subscribed `consumer`. +Both carry `peer` and `session`, and no message methods of their own. Session +tracks are `//` in both endpoints, served only on +request so no listing names them; an unknown or unaccepted protocol is reset +as unsupported. JS reads without fastest-reader pacing until +[JS fastest-reader pacing](/quest/m2/js-fastest-reader.md), so a JS `tcp` +reader that falls a group tail behind fails with an eviction error; say so in +the docs. Add the Rust/JS pairs to `just test interop`, and document the +export in `doc/lib/js/net.md`. diff --git a/quest/m1/socket/py.md b/quest/m1/socket/py.md index ffdbb045289..efc7e42eb6c 100644 --- a/quest/m1/socket/py.md +++ b/quest/m1/socket/py.md @@ -2,15 +2,17 @@ ## Goal -moq-ffi exposes the socket `Endpoint` and `Stream` beside origin, and -`py/moq-rs` wraps them, so a Python endpoint accepts a Rust endpoint's dial and -the reverse, carrying messages both ways. +moq-ffi exposes the socket `Endpoint`, `TcpStream`, and `TrackStream` beside +origin, and `py/moq-rs` wraps them, so a Python endpoint accepts a Rust +endpoint's dial and the reverse, in both protocols, carrying data both ways. ## Plan -Bind the Rust surface (`moq_net::socket`): `Endpoint::new`, `dial`, `accept`, -an incoming dial's `accept` and `reject`, and `Stream`'s `producer`, -`consumer`, `peer`, and `session`, through moq-ffi the way origin's request +Bind the Rust surface (`moq_net::socket`): `Endpoint::new`, `dial_tcp`, +`accept_tcp`, `dial_track`, `accept_track`, an incoming dial's `accept` and +`reject`, and each stream's `producer`, `consumer`, `peer`, and `session` +(group handles for `TcpStream`, track handles for `TrackStream`), through +moq-ffi the way origin's request and announce APIs are bound today (`rs/moq-ffi/src/origin.rs`, `py/moq-rs/moq/origin.py`). Add a Python/Rust interop test, and document the API wherever `py/moq-rs` documents origin. diff --git a/quest/m2/mesh/README.md b/quest/m2/mesh/README.md index 50020e7b274..54f91ed6266 100644 --- a/quest/m2/mesh/README.md +++ b/quest/m2/mesh/README.md @@ -53,8 +53,9 @@ Decided 2026-10-25 with the maintainer (planning interview), still in force: - The token is the binding: enrollment proves the device key, then the coordinator issues a grant to publish `mesh//**` and subscribe `mesh/**`, plus publish on its own [socket](/quest/m1/socket/README.md) - endpoint, `.tcp/`, so it can dial and accept under its own id (amended - 2026-11-01, and 2026-11-03 when sockets became endpoints). Relays need no new check. Membership is the coordinator's + endpoint, `.socket/`, so it can dial and accept under its own id + (amended 2026-11-01, 2026-11-03 when sockets became endpoints, and + 2026-11-04 when the segment became `.socket`). Relays need no new check. Membership is the coordinator's token only; a Tailnet-Lock analog was declined. - Protection is per broadcast, with levels that nest: identity (the token only), authentication (per-frame Ed25519 signatures, the default), and diff --git a/quest/m2/mesh/endpoint.md b/quest/m2/mesh/endpoint.md index d4126196db2..97073c5085e 100644 --- a/quest/m2/mesh/endpoint.md +++ b/quest/m2/mesh/endpoint.md @@ -16,13 +16,14 @@ Decided 2026-10-25 (maintainer): iroh-shaped, not drop-in compatible; tickets, - A service is the iroh ALPN analog: a name under `mesh//`. - Decided 2026-11-01 (maintainer, socket planning interview), and updated - 2026-11-03 when sockets became endpoints: an `Endpoint` wraps a - [socket](/doc/concept/socket.md) endpoint. Settle the layout here: such as - one socket endpoint per service at `mesh//.tcp/`, or one per + 2026-11-03 when sockets became endpoints and 2026-11-04 when the segment + became `.socket` with the protocol in the session name: an `Endpoint` wraps + a [socket](/doc/concept/socket.md) endpoint. Settle the layout here: such as + one socket endpoint per service at `mesh//.socket/`, or one per node with the service named in the session. `connect` dials with the caller's own endpoint ID, and `accept` reads the remote ID from the dial, which the socket's answer check proves and the grant pins (publish - `.tcp/` only). Update `moq auth`'s mesh grant and + `.socket/` only). Update `moq auth`'s mesh grant and `doc/bin/relay/auth.md` to match. - Streams are [byte streams](/quest/m2/stream/rust.md) at the protection level the service declares: identity reads `moq-net` directly, and diff --git a/quest/m2/stream/README.md b/quest/m2/stream/README.md index f789f8fef4d..e9b8ef11ba1 100644 --- a/quest/m2/stream/README.md +++ b/quest/m2/stream/README.md @@ -39,16 +39,17 @@ Decided 2026-10-25 with the maintainer (mesh planning interview): identity, signing, and service names belong to the [mesh](/quest/m2/mesh/README.md), which builds on this. -Amended 2026-11-01 with the maintainer (socket planning interview), and -2026-11-03 when sockets became endpoints: connection setup is the -[socket](/quest/m1/socket/README.md) epic's `Endpoint`, `dial`, and `accept`, -which settle the layout (`/.tcp/`, a `/` track -each way). Each TCP connection is one session, so it gets its own priority, +Amended 2026-11-01 with the maintainer (socket planning interview), +2026-11-03 when sockets became endpoints, and 2026-11-04 when the socket's +`tcp` protocol became one group each way: connection setup is the +[socket](/quest/m1/socket/README.md) epic's `Endpoint`, `dial_tcp`, and +`accept_tcp`, which settle the layout (`/.socket/`, a +`tcp//` track each way). Each TCP connection is one `tcp` +session, whose halves are a single group each way for the connection's life: +frames are writes, finishing is FIN, resetting is RST, and the writer awaits +`group::Producer::ready` for backpressure. Each session gets its own priority, pacing, and eviction, and a long-lived connection never sits in an old group -that newer ones crowd out of a track's window. A session carries messages one -group each; a byte stream instead holds one group each way for the -connection's life, which the stream quests settle against the socket's -reliable reader. +that newer ones crowd out of a track's window. This epic owns the end-to-end proof: an unmodified TCP client and server (SSH and an HTTP bulk transfer) talk through a two-relay chain with loss injected on diff --git a/quest/m2/stream/rust.md b/quest/m2/stream/rust.md index fb7aa9c9a75..7f001cb5dfe 100644 --- a/quest/m2/stream/rust.md +++ b/quest/m2/stream/rust.md @@ -15,10 +15,10 @@ See the [epic](/quest/m2/stream/README.md) for the mapping: a socket endpoint per forwarded service, a session per connection with one group each way, a frame per TCP read, no fill handler. -- Build on `moq_net::socket`: the path layout and connection naming are the - socket's (`Endpoint::dial` and `accept`, each yielding a `Stream`), so this - quest adds only the `AsyncRead`/`AsyncWrite` adapter over a `Stream` and the - TCP mapping. Place the adapter +- Build on `moq_net::socket`: the path layout, connection naming, FIN and RST + are the socket's (`Endpoint::dial_tcp` and `accept_tcp`, each yielding a + `TcpStream` of one group each way), so this quest adds only the + `AsyncRead`/`AsyncWrite` adapter over a `TcpStream` and the TCP mapping. Place the adapter beside the socket module unless it pulls in a dependency moq-net avoids; report the choice. - Each stream carries a subscription priority the caller sets; nothing is diff --git a/rs/moq-net/CHANGELOG.md b/rs/moq-net/CHANGELOG.md index fb7fed2f450..678445b1b56 100644 --- a/rs/moq-net/CHANGELOG.md +++ b/rs/moq-net/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `socket`: `Endpoint`, which announces a hidden `/.tcp/` broadcast, dials other endpoints, and accepts their dials, each session a `Stream` of two unlisted tracks carrying one message per group; see `doc/concept/socket.md`. +- `socket`: `Endpoint`, which announces a hidden `/.socket/` broadcast, dials other endpoints, and accepts their dials in two protocols: `tcp` sessions (`dial_tcp`, `accept_tcp`, `TcpStream`), one group each way as a reliable, ordered byte pipe, and `track` sessions (`dial_track`, `accept_track`, `TrackStream`), a track each way with one message per group; see `doc/concept/socket.md`. - `transport::SETUP_RECEIVE_WINDOW`, the connection credit a server offers before it admits a session: enough for the SETUP and one in-band AUTH. ## [0.3.10](https://github.com/moq-dev/moq/compare/moq-net-v0.3.9...moq-net-v0.3.10) - 2026-10-05 diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 7b1a6a9181f..ee1e07ebc1a 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2087,6 +2087,12 @@ pub struct Producer { } impl Producer { + /// A handle that queues work on this origin's [`Driver`] without keeping the origin + /// alive: work queued once the driver is gone is dropped. + pub(super) fn tasks(&self) -> crate::util::TasksWeak { + self.tasks.downgrade() + } + /// Build a producer from a [`Config`] (identity + cache pool) with no scoped /// prefix and no pre-existing broadcasts, paired with the [`Driver`] that runs /// the origin's lifecycle work. diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index b5ac3eb074e..72df1b89c22 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -1,20 +1,30 @@ -//! Point-to-point sockets over an origin, the way TCP connects two hosts. +//! Point-to-point sessions over an origin, the way TCP connects two hosts. //! -//! An [`Endpoint`] announces one hidden broadcast, `/.tcp/`, and from it -//! [`dial`](Endpoint::dial)s other endpoints and [`accept`](Endpoint::accept)s their -//! dials. Each connection is a session: one [`Stream`], a reliable, ordered pipe made of -//! two tracks, one in each endpoint's broadcast. Connections are never announced. +//! An [`Endpoint`] announces one hidden broadcast, `/.socket/`, used only to +//! route dials to it. From it, an endpoint dials other endpoints and accepts their dials. +//! Each connection is a session: two tracks, one in each endpoint's broadcast, both named +//! `//`. Sessions are never announced or listed. //! -//! A dialer `A` picks a random session id `S` and subscribes to `/.tcp/` for -//! the track `A/S`. Endpoint `B` surfaces that request as an [`Incoming`] dial from `A`. -//! To accept it, `B` subscribes to `/.tcp/` for the same track name, and `A` -//! answers only a session it dialed, so the exchange checks that `A` really asked: a -//! dial forged in `A`'s name is reset. Each side then writes its own half and reads the -//! peer's. Session tracks are served only on request, so no track listing names them. +//! A session speaks one of two protocols: +//! +//! - `tcp`: one reliable, ordered byte pipe each way, as TCP gives. Each half is a single +//! group. Dial with [`Endpoint::dial_tcp`], accept with [`Endpoint::accept_tcp`], and +//! read and write a [`TcpStream`]. +//! - `track`: each half is a whole track, so messages are groups, with the track's own +//! priority, delay budget and order. Dial with [`Endpoint::dial_track`], accept with +//! [`Endpoint::accept_track`], and read and write a [`TrackStream`]. +//! +//! A dialer `A` picks a random session id `S` and subscribes to `/.socket/` for +//! the track `/A/S`. Endpoint `B` surfaces that request as an [`Incoming`] dial +//! from `A` in its protocol's queue. To accept it, `B` subscribes to `/.socket/` +//! for the same track name, and `A` answers only a session it dialed, so the exchange +//! checks that `A` really asked: a dial forged in `A`'s name is reset. Each side then +//! writes its own half and reads the peer's. //! //! See `doc/concept/socket.md` for the convention other languages follow. use std::collections::VecDeque; +use std::marker::PhantomData; use std::sync::Arc; use std::task::{Poll, ready}; @@ -23,10 +33,10 @@ use crate::{Error, Path}; use super::WeakEntry; use super::origin::{self, Route}; use super::track::{self, Delay, Order, Subscription}; +use super::{frame, group}; -/// The segment an endpoint's broadcast sits under: one reliable, ordered pipe per -/// session, as TCP gives. A multiplexed mode would take a segment of its own. -const TCP: &str = ".tcp"; +/// The segment every endpoint's broadcast sits under. +const SOCKET: &str = ".socket"; /// Refuse an id that is not one plain path segment: empty, or holding a `/` or a `\`. fn check_id(id: &str) -> Result<(), Error> { @@ -36,10 +46,10 @@ fn check_id(id: &str) -> Result<(), Error> { Ok(()) } -/// `/.tcp/`, the broadcast of the endpoint `id`. +/// `/.socket/`, the broadcast of the endpoint `id`. fn endpoint_path(prefix: &Path, id: &str) -> Result { check_id(id)?; - Ok(prefix.join(&Path::from_static(TCP))?.join(&Path::new(id)?)?) + Ok(prefix.join(&Path::from_static(SOCKET))?.join(&Path::new(id)?)?) } /// A fresh session id: 128 random bits as hex, so nobody can guess a session's tracks. @@ -48,46 +58,217 @@ fn session_id() -> Arc { format!("{:032x}", rand::rng().random::()).into() } -/// The track name of a session: `/`, the same in both endpoints. -fn track_name(dialer: &str, session: &str) -> String { - format!("{dialer}/{session}") +/// The track name of a session: `//`, the same in both +/// endpoints. +fn track_name(protocol: &str, dialer: &str, session: &str) -> String { + format!("{protocol}/{dialer}/{session}") +} + +/// A session's track name, split. +#[derive(Debug, PartialEq, Eq)] +struct Name<'a> { + protocol: &'a str, + dialer: &'a str, + session: &'a str, } -/// Split a session's track name into its dialer and session, or `None` for a name -/// outside the scheme. -fn parse(name: &str) -> Option<(&str, &str)> { - let (dialer, session) = name.split_once('/')?; - check_id(dialer).ok()?; - check_id(session).ok()?; - Some((dialer, session)) +/// Split a session's track name, or `None` for a name outside the scheme. +fn parse(name: &str) -> Option> { + let (protocol, rest) = name.split_once('/')?; + let (dialer, session) = rest.split_once('/')?; + for segment in [protocol, dialer, session] { + check_id(segment).ok()?; + } + Some(Name { + protocol, + dialer, + session, + }) } -/// What a session's reader asks for: every message, oldest first, however late it is. +/// What a session's reader asks for: every group, oldest first, however late it is. fn reliable() -> Subscription { Subscription::default() .with_order(Order::Ascending) .with_delay(Delay::default().with_max(None)) } -/// One session: this side's half and the peer's. +mod sealed { + use super::*; + + /// The protocols an endpoint speaks. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Protocol { + Tcp, + Track, + } + + /// What an endpoint's handles share. + pub struct Shared { + pub(super) id: Arc, + pub(super) prefix: Path, + /// Where the peers' endpoints are requested. + pub(super) origin: origin::Consumer, + /// Where the `tcp` sessions' pumps run: the origin's driver. + pub(super) tasks: crate::util::TasksWeak, + /// This endpoint's broadcast. + pub(super) publication: origin::Publication, + pub(super) state: kio::Lock, + } + + /// A session's two halves, once the handshake is done. + pub struct Parts { + pub shared: Arc, + /// This side's half, served to the peer. + pub producer: track::Producer, + /// The peer's half, subscribed reliably and in order. + pub subscriber: track::Subscriber, + pub peer: Arc, + pub session: Arc, + } + + /// What a protocol does with a session's halves. + pub trait Open: Sized { + /// The protocol's segment, the first of a session's track name. + const PROTOCOL: Protocol; + + /// Make the stream an application reads and writes. + fn open(parts: Parts) -> Self; + } +} + +use sealed::{Parts, Protocol, Shared}; + +impl Protocol { + fn as_str(self) -> &'static str { + match self { + Self::Tcp => "tcp", + Self::Track => "track", + } + } + + fn parse(protocol: &str) -> Option { + match protocol { + "tcp" => Some(Self::Tcp), + "track" => Some(Self::Track), + _ => None, + } + } +} + +/// A session's stream: a [`TcpStream`] or a [`TrackStream`]. Names the protocol a dial, +/// an incoming dial, or an accept is for. +pub trait Stream: sealed::Open {} + +/// A `tcp` session: one reliable, ordered byte pipe each way, as TCP gives. +/// +/// Each half is the session track's only group, which is one QUIC stream on every hop, +/// so it is reliable, ordered and flow controlled like a TCP direction. A frame is a +/// write. Finishing [`Self::producer`] is FIN, and aborting or dropping it is RST; the +/// peer reads the same from [`Self::consumer`]. Reading ends once the peer finished, and +/// fails with its reset. A second group on a peer's half is a protocol error that resets +/// the session. +/// +/// Await [`group::Producer::poll_ready`] (or `ready`) before each write: it waits until the peer reads, so +/// the writer runs at most a window ahead of the reader and nothing is lost. A writer +/// that never waits sheds the front of its half, and a reader that falls behind it fails +/// with [`Error::Evicted`]. +/// +/// Dropping [`Self::consumer`] before the peer finished resets the session, since nobody +/// reads what the peer writes. The endpoint closing resets it too. +pub struct TcpStream { + /// This side's half: the frames written here reach the peer in order. + pub producer: group::Producer, + /// The peer's half: its frames, in order. + pub consumer: group::Consumer, + peer: Arc, + session: Arc, +} + +impl TcpStream { + /// The peer endpoint's id. + pub fn peer(&self) -> &str { + &self.peer + } + + /// The session's id, which the dialer chose. + pub fn session(&self) -> &str { + &self.session + } +} + +impl Stream for TcpStream {} + +impl sealed::Open for TcpStream { + const PROTOCOL: Protocol = Protocol::Tcp; + + fn open(parts: Parts) -> Self { + let Parts { + shared, + producer: track, + subscriber, + peer, + session, + } = parts; + let (producer, ours) = match track.create_group(group::Info { sequence: 0 }) { + Ok(group) => { + let handle = group.handle(); + (group, Some(handle)) + } + // The track already ended, such as by the endpoint closing: hand out a half + // that reports it. + Err(err) => { + let group = group::Producer::new(group::Info { sequence: 0 }, track::Info::default(), Default::default()); + let _ = group.handle().abort(err); + (group, None) + } + }; + // The application reads a copy of the peer's half that the pump fills, so the + // session can end it: the peer's own copy is the session's, not this side's. + let local = group::Producer::new(group::Info { sequence: 0 }, track::Info::default(), Default::default()); + let consumer = local.consume(); + let pump = Pump { + shared: shared.clone(), + track: Some(track), + ours, + subscriber: Some(subscriber), + remote: None, + local_handle: local.handle(), + local: Some(local), + pending: None, + }; + shared.tasks.push(kio::Pending::new(pump)); + Self { + producer, + consumer, + peer, + session, + } + } +} + +/// A `track` session: each half is a whole track, and each message a group. /// -/// Each message is one group, so a slow message never holds up the next one, and messages -/// are read in group sequence. [`Self::consumer`] starts reliable and in order (oldest -/// first, with no delay budget); update its subscription to make it lossy. +/// [`Self::consumer`] starts subscribed reliably and in order: oldest first, with no +/// delay budget, so every group arrives. Update its subscription, such as with a finite +/// [`Delay`] or [`Order::Descending`], to make it lossy or newest first, and read it in +/// group order with [`track::Subscriber::ordered`]. Groups are written with +/// [`track::Producer::append_group`], each with its own priority on the wire. /// /// Finishing [`Self::producer`] closes the session for the peer, and aborting it is an /// error. The endpoint closing aborts it too. When the peer's half ends with an error, /// abort this side's half as well, and drop the stream, so both halves end. -pub struct Stream { +pub struct TrackStream { /// This side's half: the session's track in this endpoint's broadcast. pub producer: track::Producer, - /// The peer's half: the session's track in the peer's broadcast, read in sequence. - pub consumer: track::Ordered, + /// The peer's half: the session's track in the peer's broadcast, subscribed reliably + /// and in order until updated. + pub consumer: track::Subscriber, peer: Arc, session: Arc, } -impl Stream { +impl TrackStream { /// The peer endpoint's id. pub fn peer(&self) -> &str { &self.peer @@ -99,13 +280,38 @@ impl Stream { } } +impl Stream for TrackStream {} + +impl sealed::Open for TrackStream { + const PROTOCOL: Protocol = Protocol::Track; + + fn open(parts: Parts) -> Self { + Self { + producer: parts.producer, + consumer: parts.subscriber, + peer: parts.peer, + session: parts.session, + } + } +} + +/// Dials from peers in one protocol, waiting to be accepted. +#[derive(Default)] +struct Queue { + /// Whether the protocol is accepted: from its first accept on. + listening: bool, + dials: VecDeque, + /// The accepts waiting for a dial. + waiting: kio::WaiterList, +} + /// What the endpoint's pollers share. struct State { /// The dials peers send, from the first accept on; `None` until then, so the endpoint /// answers a request for a session it does not hold with [`Error::NotFound`] at once. requests: Option, - /// Dials from peers, not yet taken by an accept, in order. - incoming: VecDeque, + tcp: Queue, + track: Queue, /// This side's half of every session handed out, aborted when the endpoint closes. streams: Vec, /// Why the endpoint stopped, once it did. @@ -113,6 +319,13 @@ struct State { } impl State { + fn queue(&mut self, protocol: Protocol) -> &mut Queue { + match protocol { + Protocol::Tcp => &mut self.tcp, + Protocol::Track => &mut self.track, + } + } + /// Keep `producer`'s track, so the endpoint closing ends it. fn hold(&mut self, producer: &track::Producer) { self.streams.retain(|stream| !stream.is_closed()); @@ -120,50 +333,66 @@ impl State { } } -struct Shared { - id: Arc, - prefix: Path, - /// Where the peers' endpoints are requested. - origin: origin::Consumer, - /// This endpoint's broadcast. - publication: origin::Publication, - state: kio::Lock, -} - impl Shared { - /// Queue every dial on hand and take the next, registering `waiter` for more, and - /// fail with the endpoint's end once it stopped. + /// Queue every dial on hand and take the next in `protocol`, registering `waiter` for + /// more, and fail with the endpoint's end once it stopped. /// /// A dial of this endpoint's reserves its own track, so a request reaching the handler /// under this endpoint's id names a session it never dialed, or one it gave up, such - /// as an accept of a dial forged in this endpoint's name: it is reset, which fails - /// that accept. Resets go out after the lock is released, since they wake readers. - fn poll_dial(&self, waiter: &kio::Waiter) -> Poll> { + /// as an accept of a dial forged in this endpoint's name: it is reset with + /// [`Error::NotFound`], which fails that accept. A request in a protocol nobody accepts + /// is reset with [`Error::Unsupported`]. Resets and wakes go out after the lock is + /// released. + fn poll_dial(&self, protocol: Protocol, waiter: &kio::Waiter) -> Poll> { let mut resets = Vec::new(); + let mut wakes = Vec::new(); let mut state = self.state.lock(); + state.queue(protocol).listening = true; while state.end.is_none() { let requests = state.requests.get_or_insert_with(|| self.publication.requests()); match requests.poll_requested_track(waiter) { Poll::Pending => break, - Poll::Ready(Ok(request)) => match parse(request.name()) { - Some((dialer, _)) if dialer != &*self.id => state.incoming.push_back(request), - _ => resets.push(request), - }, + Poll::Ready(Ok(request)) => { + // `None` for a name naming no dial: outside the scheme, or this + // endpoint's own. + let target = match parse(request.name()) { + Some(name) if name.dialer != &*self.id => Some(Protocol::parse(name.protocol)), + _ => None, + }; + match target { + Some(Some(protocol)) if state.queue(protocol).listening => { + let queue = state.queue(protocol); + queue.dials.push_back(request); + wakes.push(queue.waiting.take()); + } + Some(_) => resets.push((request, Error::Unsupported)), + None => resets.push((request, Error::NotFound)), + } + } Poll::Ready(Err(err)) => state.end = Some(err), } } let next = match &state.end { Some(err) => Poll::Ready(Err(err.clone())), - None => match state.incoming.pop_front() { - Some(request) => Poll::Ready(Ok(request)), - None => Poll::Pending, - }, + None => { + let queue = state.queue(protocol); + match queue.dials.pop_front() { + Some(request) => Poll::Ready(Ok(request)), + None => { + queue.waiting.register(waiter); + Poll::Pending + } + } + } }; drop(state); - for request in resets { + for mut wake in wakes { + wake.wake(); + } + for (request, err) in resets { // Not the name: a session's name is its secret. - tracing::debug!("reset a request for no session"); - request.reject(Error::NotFound); + tracing::debug!(%err, "reset a request for no session"); + request.reject(err); } next } @@ -180,34 +409,49 @@ impl Shared { } fn close(&self) { - let mut state = self.state.lock(); + let mut guard = self.state.lock(); + let state = &mut *guard; state.end.get_or_insert(Error::Closed); let requests = state.requests.take(); - let incoming = std::mem::take(&mut state.incoming); + let mut queued = Vec::new(); + let mut wakes = Vec::new(); + for queue in [&mut state.tcp, &mut state.track] { + queued.extend(queue.dials.drain(..)); + wakes.push(queue.waiting.take()); + } let streams = std::mem::take(&mut state.streams); - drop(state); + drop(guard); self.publication.close(); // Reset outside the lock. for stream in streams { stream.abort(Error::Cancel); } - for request in incoming { + for request in queued { request.reject(Error::Cancel); } + for mut wake in wakes { + wake.wake(); + } drop(requests); } } -/// A socket endpoint: announces `/.tcp/`, dials other endpoints, and accepts -/// their dials. +/// A socket endpoint: announces `/.socket/`, dials other endpoints, and +/// accepts their dials. +/// +/// It accepts dials in a protocol from that protocol's first accept on, like a TCP +/// listener: before that, a dial in it is reset with [`Error::Unsupported`], and after it, +/// dials queue until accepted. Until the endpoint's first accept in any protocol, a dial +/// to it is reset with [`Error::NotFound`], as for any track nobody serves. /// -/// It accepts dials from its first [`Self::accept`] on, like a TCP listener: before that, a -/// dial to it is reset, and after it, dials queue until accepted. Dropping it, or -/// [`Self::close`], retracts the broadcast, resets the dials waiting to be accepted, and -/// aborts this side's half of every session it handed out. The peer's half is read -/// through each [`Stream::consumer`], which the application owns: drop the streams too, -/// which releases the subscriptions. +/// Dropping it, or [`Self::close`], retracts the broadcast, resets the dials waiting to be +/// accepted, and resets every session it handed out: a [`TcpStream`] entirely, and this +/// side's half of a [`TrackStream`]. The peer's half of a [`TrackStream`] is read through +/// its [`TrackStream::consumer`], which the application owns: drop the streams too, which +/// releases the subscriptions. +/// +/// `tcp` sessions run a task on the origin's driver, which must keep running. pub struct Endpoint { shared: Arc, } @@ -218,7 +462,7 @@ impl Endpoint { /// `id` names the endpoint: a well-known name for a server, or a random one for a /// client. It must be one path segment. Fails like /// [`origin::Producer::create_broadcast`] when `origin` may not publish - /// `/.tcp/`. + /// `/.socket/`. pub fn new( origin: &origin::Producer, prefix: impl TryInto>, @@ -232,10 +476,12 @@ impl Endpoint { id: id.into(), prefix, origin: origin.consume(), + tasks: origin.tasks(), publication, state: kio::Lock::new(State { requests: None, - incoming: VecDeque::new(), + tcp: Queue::default(), + track: Queue::default(), streams: Vec::new(), end: None, }), @@ -253,13 +499,28 @@ impl Endpoint { &self.shared.prefix } - /// Dial the endpoint `peer` in a new session, resolving once the peer accepts it. + /// Dial the endpoint `peer` in a new `tcp` session, resolving once the peer accepts it. /// - /// `info` describes this side's half. Waits for as long as the peer takes to accept: - /// drop the returned future to give up, which resets the session. That includes waiting - /// for the peer's endpoint to be announced. Fails with the peer's reset when it rejects - /// the dial, and with [`Error::Unsupported`] for a dial to this endpoint itself. - pub fn dial(&self, peer: &str, info: impl Into>) -> Result, Error> { + /// Waits for as long as the peer takes to accept: drop the returned future to give up, + /// which resets the session. That includes waiting for the peer's endpoint to be + /// announced. Fails with the peer's reset when it rejects the dial, or does not accept + /// `tcp` sessions, and with [`Error::Unsupported`] for a dial to this endpoint itself. + pub fn dial_tcp(&self, peer: &str) -> Result>, Error> { + self.dial(peer, None) + } + + /// Dial the endpoint `peer` in a new `track` session; see [`Self::dial_tcp`]. + /// + /// `info` describes this side's half. + pub fn dial_track( + &self, + peer: &str, + info: impl Into>, + ) -> Result>, Error> { + self.dial(peer, info.into()) + } + + fn dial(&self, peer: &str, info: Option) -> Result>, Error> { let shared = &self.shared; let path = endpoint_path(&shared.prefix, peer)?; if peer == &*shared.id { @@ -269,45 +530,63 @@ impl Endpoint { return Err(err.clone()); } let session = session_id(); + let name = track_name(S::PROTOCOL.as_str(), &shared.id, &session); // This side's half, waiting for the peer to ask for it, which only the peer can: // nobody else knows the session. - let answer = shared - .publication - .reserve_unlisted_track(track_name(&shared.id, &session))?; + let answer = shared.publication.reserve_unlisted_track(name.clone())?; // Followed rather than requested, so a peer not announced yet is waited for. let resolving = Resolving::Follow(Box::new(shared.origin.follow(path, None)?)); Ok(kio::Pending::new(Dialing { shared: shared.clone(), peer: peer.into(), session, - info: info.into(), + name, + info, demand: answer.demand(), answer: Some(Answer::Reserved(answer)), step: Step::Resolving(resolving), + stream: PhantomData, })) } - /// Poll for the next dial from a peer; fails with [`Error::Closed`] once the endpoint - /// closed. - /// - /// Dials queue until taken, without limit, each holding the dialer's request open, so - /// keep accepting, and accept or reject each one. - pub fn poll_accept(&self, waiter: &kio::Waiter) -> Poll> { + fn poll_accept(&self, waiter: &kio::Waiter) -> Poll, Error>> { let shared = &self.shared; - let request = ready!(shared.poll_dial(waiter))?; - let (peer, session) = parse(request.name()).expect("sorted as a dial"); + let request = ready!(shared.poll_dial(S::PROTOCOL, waiter))?; + let name = parse(request.name()).expect("sorted as a dial"); Poll::Ready(Ok(Incoming { shared: shared.clone(), - peer: peer.into(), - session: session.into(), + peer: name.dialer.into(), + session: name.session.into(), request: Some(request), + stream: PhantomData, })) } - /// Wait for the next dial from a peer; see [`Self::poll_accept`]. + /// Poll for the next `tcp` dial from a peer; fails with [`Error::Closed`] once the + /// endpoint closed. + /// + /// The first call starts accepting `tcp` dials. They queue until taken, without limit, + /// each holding the dialer's request open, so keep accepting, and accept or reject + /// each one. + pub fn poll_accept_tcp(&self, waiter: &kio::Waiter) -> Poll, Error>> { + self.poll_accept(waiter) + } + + /// Wait for the next `tcp` dial from a peer; see [`Self::poll_accept_tcp`]. #[cfg(feature = "async")] - pub async fn accept(&self) -> Result { - kio::wait(|waiter| self.poll_accept(waiter)).await + pub async fn accept_tcp(&self) -> Result, Error> { + kio::wait(|waiter| self.poll_accept_tcp(waiter)).await + } + + /// Poll for the next `track` dial from a peer; see [`Self::poll_accept_tcp`]. + pub fn poll_accept_track(&self, waiter: &kio::Waiter) -> Poll, Error>> { + self.poll_accept(waiter) + } + + /// Wait for the next `track` dial from a peer; see [`Self::poll_accept_tcp`]. + #[cfg(feature = "async")] + pub async fn accept_track(&self) -> Result, Error> { + kio::wait(|waiter| self.poll_accept_track(waiter)).await } /// Close the endpoint; see [`Endpoint`]. @@ -322,19 +601,21 @@ impl Drop for Endpoint { } } -/// A dial from a peer, from [`Endpoint::accept`]: accept or reject it. +/// A dial from a peer, from [`Endpoint::accept_tcp`] or [`Endpoint::accept_track`]: accept +/// or reject it. /// /// Dropping it rejects it with [`Error::Cancel`]. -pub struct Incoming { +pub struct Incoming { shared: Arc, peer: Arc, session: Arc, /// The dialer's request for this side's half; taken by an accept or a reject. request: Option, + stream: PhantomData S>, } -impl Incoming { - /// The id the dialer claims. [`Self::accept`] checks it: the dial connects only if that +impl Incoming { + /// The id the dialer claims. An accept checks it: the dial connects only if that /// endpoint really dialed this session. pub fn peer(&self) -> &str { &self.peer @@ -345,10 +626,15 @@ impl Incoming { &self.session } - /// Accept the dial, resolving once the dialer answers, which proves it dialed. - /// - /// `info` describes this side's half. Nothing is written before the dialer answers. - pub fn accept(mut self, info: impl Into>) -> kio::Pending { + /// Reject the dial, resetting the dialer's request with [`Error::NotFound`], as for a + /// session that does not exist. + pub fn reject(mut self) { + if let Some(request) = self.request.take() { + request.reject(Error::NotFound); + } + } + + fn accept_with(mut self, info: Option) -> kio::Pending> { let shared = self.shared.clone(); let (peer, session) = (self.peer.clone(), self.session.clone()); let request = self.request.take(); @@ -362,24 +648,36 @@ impl Incoming { }; kio::Pending::new(Accepting { shared, + name: track_name(S::PROTOCOL.as_str(), &peer, &session), peer, session, - info: info.into(), + info, request, step, + stream: PhantomData, }) } +} - /// Reject the dial, resetting the dialer's request with [`Error::NotFound`], as for a - /// session that does not exist. - pub fn reject(mut self) { - if let Some(request) = self.request.take() { - request.reject(Error::NotFound); - } +impl Incoming { + /// Accept the dial, resolving once the dialer answers, which proves it dialed. + /// + /// Nothing is written before the dialer answers. + pub fn accept(self) -> kio::Pending> { + self.accept_with(None) + } +} + +impl Incoming { + /// Accept the dial, resolving once the dialer answers, which proves it dialed. + /// + /// `info` describes this side's half. Nothing is written before the dialer answers. + pub fn accept(self, info: impl Into>) -> kio::Pending> { + self.accept_with(info.into()) } } -impl Drop for Incoming { +impl Drop for Incoming { fn drop(&mut self) { if let Some(request) = self.request.take() { request.reject(Error::Cancel); @@ -447,6 +745,32 @@ impl Step { } } +/// Hand out a session's stream once its handshake is done, unless the endpoint stopped, +/// in which case this side's half is aborted, outside the lock. +fn open( + shared: &Arc, + producer: track::Producer, + step: &mut Step, + peer: &Arc, + session: &Arc, +) -> Result { + let mut state = shared.state.lock(); + if let Some(err) = state.end.clone() { + drop(state); + let _ = producer.abort(err.clone()); + return Err(err); + } + state.hold(&producer); + drop(state); + Ok(S::open(Parts { + shared: shared.clone(), + producer, + subscriber: step.take_subscriber(), + peer: peer.clone(), + session: session.clone(), + })) +} + /// This side's half of a dial. enum Answer { /// Reserved, until the peer asks for it. @@ -467,28 +791,32 @@ impl Answer { } } -/// A dial, from [`Endpoint::dial`]: resolves once the peer accepts. +/// A dial, from [`Endpoint::dial_tcp`] or [`Endpoint::dial_track`]: resolves once the peer +/// accepts. /// /// Dropping it gives up, resetting the session. -pub struct Dialing { +pub struct Dialing { shared: Arc, peer: Arc, session: Arc, + /// The session's track name. + name: String, info: Option, /// Whether the peer asked for this side's half, which it does on accept. demand: track::Demand, answer: Option, step: Step, + stream: PhantomData S>, } -impl Dialing { +impl Dialing { /// The session's id. pub fn session(&self) -> &str { &self.session } - /// Poll for the peer to accept; see [`Endpoint::dial`]. - pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { + /// Poll for the peer to accept; see [`Endpoint::dial_tcp`]. + pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let result = self.poll_inner(waiter); if let Poll::Ready(Err(err)) = &result { if let Some(answer) = self.answer.take() { @@ -499,9 +827,8 @@ impl Dialing { result } - fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { - let shared = &self.shared; - shared.poll_end(waiter)?; + fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { + self.shared.poll_end(waiter)?; // The peer asks for this side's half when it accepts: serve it, which proves to the // peer that this endpoint dialed. @@ -518,8 +845,7 @@ impl Dialing { } } - let track = track_name(&shared.id, &self.session); - ready!(self.step.poll(waiter, &track))?; + ready!(self.step.poll(waiter, &self.name))?; // The peer serves its half only after asking for this side's, so a conforming peer // asked already; wait for its request regardless. let Some(Answer::Served(_)) = &self.answer else { @@ -528,34 +854,19 @@ impl Dialing { let Some(Answer::Served(producer)) = self.answer.take() else { unreachable!("checked above"); }; - - let mut state = shared.state.lock(); - if let Some(err) = state.end.clone() { - // Aborted outside the lock. - drop(state); - let _ = producer.abort(err.clone()); - return Poll::Ready(Err(err)); - } - state.hold(&producer); - drop(state); - Poll::Ready(Ok(Stream { - producer, - consumer: self.step.take_subscriber().ordered(), - peer: self.peer.clone(), - session: self.session.clone(), - })) + Poll::Ready(open(&self.shared, producer, &mut self.step, &self.peer, &self.session)) } } -impl kio::Task for Dialing { - type Output = Result; +impl kio::Task for Dialing { + type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { self.poll_ok(waiter) } } -impl Drop for Dialing { +impl Drop for Dialing { fn drop(&mut self) { // Given up: reset the session for the peer. if let Some(answer) = self.answer.take() { @@ -564,23 +875,26 @@ impl Drop for Dialing { } } -/// An accept, from [`Incoming::accept`]: resolves once the dialer answers. -pub struct Accepting { +/// An accept, from an [`Incoming`] dial: resolves once the dialer answers. +pub struct Accepting { shared: Arc, peer: Arc, session: Arc, + /// The session's track name. + name: String, info: Option, /// The dialer's request for this side's half, served once the dialer answers. request: Option, step: Step, + stream: PhantomData S>, } -impl Accepting { - /// Poll for the dialer to answer; see [`Incoming::accept`]. +impl Accepting { + /// Poll for the dialer to answer, which proves it dialed. /// /// Fails with the dialer's reset when it never dialed this session, such as a dial /// forged in its name, and resets the dialer's request with the same error. - pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { + pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let result = self.poll_inner(waiter); if let Poll::Ready(Err(err)) = &result { self.step = Step::Done; @@ -591,41 +905,26 @@ impl Accepting { result } - fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { - let shared = &self.shared; - shared.poll_end(waiter)?; - let track = track_name(&self.peer, &self.session); - ready!(self.step.poll(waiter, &track))?; + fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { + self.shared.poll_end(waiter)?; + ready!(self.step.poll(waiter, &self.name))?; // Accepted outside the lock, since it wakes the dialer. let request = self.request.take().ok_or(Error::Closed)?; let producer = request.accept(self.info.take()); - let mut state = shared.state.lock(); - if let Some(err) = state.end.clone() { - drop(state); - let _ = producer.abort(err.clone()); - return Poll::Ready(Err(err)); - } - state.hold(&producer); - drop(state); - Poll::Ready(Ok(Stream { - producer, - consumer: self.step.take_subscriber().ordered(), - peer: self.peer.clone(), - session: self.session.clone(), - })) + Poll::Ready(open(&self.shared, producer, &mut self.step, &self.peer, &self.session)) } } -impl kio::Task for Accepting { - type Output = Result; +impl kio::Task for Accepting { + type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { self.poll_ok(waiter) } } -impl Drop for Accepting { +impl Drop for Accepting { fn drop(&mut self) { // Given up: reset the dialer's request. if let Some(request) = self.request.take() { @@ -634,20 +933,184 @@ impl Drop for Accepting { } } +/// A `tcp` session's task on the origin's driver. +/// +/// It copies the peer's half into the group the application reads, a frame at a time and +/// paced by that reader, so backpressure reaches the peer. It ends this side's half the +/// way the application ended its group, and resets the session on a second group from the +/// peer, the application dropping its reader, or the endpoint closing. +struct Pump { + shared: Arc, + /// This side's half, held so the application can write only its one group. + track: Option, + /// This side's group, watched for how the application ends it. + ours: Option, + /// The peer's half, watched for its one group, and for a second. + subscriber: Option, + /// The peer's group, once it arrived. + remote: Option, + /// The application's copy of the peer's group, until it ends. + local: Option, + local_handle: group::Handle, + /// A frame read from the peer, waiting for the application's reader to make room. + pending: Option, +} + +impl Pump { + /// Reset everything still open with `err`. + fn reset(&mut self, err: Error) { + if let Some(local) = self.local.take() { + let _ = local.abort(err.clone()); + } + if let Some(track) = self.track.take() { + let _ = track.abort(err); + } + self.ours = None; + self.subscriber = None; + self.remote = None; + self.pending = None; + } + + fn poll_session(&mut self, waiter: &kio::Waiter) -> Result<(), Error> { + self.shared.poll_end(waiter)?; + + // This side's half ends the way the application ended its group. + if let Some(ours) = &self.ours + && let Poll::Ready(outcome) = ours.poll_closed(waiter) + { + self.ours = None; + match outcome { + Ok(()) => { + if let Some(track) = &self.track { + let _ = track.finish(); + } + } + Err(err) => { + if let Some(track) = self.track.take() { + let _ = track.abort(err); + } + } + } + } + + if self.local.is_none() { + return Ok(()); + } + // Nobody reads what the peer writes. + if self.local_handle.poll_unwanted(waiter).is_ready() { + return Err(Error::Cancel); + } + + // The peer's half holds one group: a second is a protocol error. + if let Some(subscriber) = &mut self.subscriber { + loop { + match subscriber.poll_recv_group(waiter) { + Poll::Pending => break, + Poll::Ready(Ok(Some(group))) => { + if self.remote.is_some() || group.sequence != 0 { + tracing::debug!(sequence = group.sequence, "a second group on a tcp session"); + return Err(Error::ProtocolViolation); + } + self.remote = Some(group); + } + Poll::Ready(Ok(None)) => { + self.subscriber = None; + break; + } + Poll::Ready(Err(err)) => { + // Ended short of its group: the peer's reset. + if self.remote.is_none() { + return Err(err); + } + self.subscriber = None; + break; + } + } + } + } + + let Some(remote) = &mut self.remote else { + // Finished without a group: an empty half. + if self.subscriber.is_none() + && let Some(mut local) = self.local.take() + { + let _ = local.finish(); + } + return Ok(()); + }; + let local = self.local.as_mut().expect("checked above"); + loop { + if let Some(frame) = &self.pending { + let info = frame::Info { + size: frame.payload.len() as u64, + timestamp: None, + }; + if local.poll_ready(waiter, &info).is_pending() { + return Ok(()); + } + let frame = self.pending.take().expect("checked above"); + local.write_frame(None, frame.payload)?; + } + match remote.poll_read_frame(waiter) { + Poll::Pending => return Ok(()), + Poll::Ready(Ok(Some(frame))) => self.pending = Some(frame), + Poll::Ready(Ok(None)) => { + local.finish()?; + self.local = None; + self.remote = None; + return Ok(()); + } + Poll::Ready(Err(err)) => return Err(err), + } + } + } +} + +impl kio::Task for Pump { + type Output = (); + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + if let Err(err) = self.poll_session(waiter) { + self.reset(err); + } + // Done once both halves ended; dropping the track then leaves its end in place. + if self.ours.is_none() && self.local.is_none() { + return Poll::Ready(()); + } + Poll::Pending + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn track_names_round_trip() { - let name = track_name("alice", "0123abcd"); - assert_eq!(name, "alice/0123abcd"); - assert_eq!(parse(&name), Some(("alice", "0123abcd"))); + let name = track_name("tcp", "alice", "0123abcd"); + assert_eq!(name, "tcp/alice/0123abcd"); + assert_eq!( + parse(&name), + Some(Name { + protocol: "tcp", + dialer: "alice", + session: "0123abcd" + }) + ); } #[test] fn foreign_track_names_are_skipped() { - for name in ["command", "alice/", "/s", "alice/s/0", "a\\b/s", "alice/s\\"] { + for name in [ + "command", + "alice/s", + "tcp/alice/", + "tcp//s", + "/alice/s", + "tcp/alice/s/0", + "tcp/a\\b/s", + "tcp/alice/s\\", + ] { assert_eq!(parse(name), None, "{name}"); } } @@ -655,7 +1118,7 @@ mod tests { #[test] fn an_endpoint_path_holds_one_segment() { let prefix = Path::from_static("room"); - assert_eq!(endpoint_path(&prefix, "alice").unwrap().as_str(), "room/.tcp/alice"); + assert_eq!(endpoint_path(&prefix, "alice").unwrap().as_str(), "room/.socket/alice"); assert!(endpoint_path(&prefix, "a/b").is_err()); assert!(endpoint_path(&prefix, "").is_err()); assert!(endpoint_path(&prefix, "a\\b").is_err()); diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 9b9757938d9..a4b7ab5f73b 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -1,13 +1,17 @@ #![cfg(feature = "async")] //! Sockets through a relay: endpoints on their own origins, each with a session to a -//! relay, dialing and accepting sessions over moq-lite-07 and moq-transport. +//! relay, dialing and accepting `tcp` and `track` sessions over moq-lite-07 and +//! moq-transport. +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; +use bytes::Bytes; use moq_net::broadcast::TracksEvent; -use moq_net::socket::{Endpoint, Stream}; -use moq_net::{Error, Hop, Version, origin, track}; +use moq_net::socket::{Endpoint, TcpStream, TrackStream}; +use moq_net::{Error, Hop, Version, frame, group, origin, track}; use crate::support::harness::{MockConnectOptions, MockPair, connect_mock, spawn}; @@ -64,11 +68,29 @@ impl Fixture { } } -/// Dial `acceptor` from `dialer` while `acceptor` accepts, returning both ends. -async fn connect(dialer: &Endpoint, acceptor: &Endpoint) -> (Stream, Stream) { - let dial = dialer.dial(acceptor.id(), None).unwrap(); + +const MB: usize = 1 << 20; + +/// Dial `acceptor` from `dialer` in a `tcp` session while `acceptor` accepts, returning +/// both ends. +async fn connect_tcp(dialer: &Endpoint, acceptor: &Endpoint) -> (TcpStream, TcpStream) { + let dial = dialer.dial_tcp(acceptor.id()).unwrap(); + let accept = async { + let incoming = acceptor.accept_tcp().await?; + assert_eq!(incoming.peer(), dialer.id()); + incoming.accept().await + }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) + .await + .expect("connect timeout"); + (dialed.expect("dial"), accepted.expect("accept")) +} + +/// Dial `acceptor` from `dialer` in a `track` session; see [`connect_tcp`]. +async fn connect_track(dialer: &Endpoint, acceptor: &Endpoint) -> (TrackStream, TrackStream) { + let dial = dialer.dial_track(acceptor.id(), None).unwrap(); let accept = async { - let incoming = acceptor.accept().await?; + let incoming = acceptor.accept_track().await?; assert_eq!(incoming.peer(), dialer.id()); incoming.accept(None).await }; @@ -78,14 +100,22 @@ async fn connect(dialer: &Endpoint, acceptor: &Endpoint) -> (Stream, Stream) { (dialed.expect("dial"), accepted.expect("accept")) } -/// Write `payload` as one message: one group of `track`. +/// Read the next write from a `tcp` session, or `None` once the peer finished. +async fn read(reader: &mut group::Consumer) -> Result>, Error> { + kio::clock::timeout(TIMEOUT, reader.read_frame()) + .await + .expect("read timeout") + .map(|frame| frame.map(|frame| frame.payload.to_vec())) +} + +/// Write `payload` as one message of a `track` session: one group. fn send(track: &track::Producer, payload: &'static [u8]) { let mut group = track.append_group().unwrap(); group.write_frame(None, payload).unwrap(); group.finish().unwrap(); } -/// Read the next message from `reader`, or `None` once the peer finished. +/// Read the next message from a `track` session, or `None` once the peer finished. async fn recv(reader: &mut track::Ordered) -> Result>, Error> { kio::clock::timeout(TIMEOUT, async { let Some(mut group) = reader.next_group().await? else { @@ -98,56 +128,282 @@ async fn recv(reader: &mut track::Ordered) -> Result>, Error> { .expect("read timeout") } -/// Messages flow both ways, in order, over every pairing of moq-lite-07 and -/// moq-transport: the handshake uses neither instances nor track listings. +/// Writes flow both ways, in order, over every pairing of moq-lite-07 and moq-transport, +/// and finishing a half is FIN: the handshake uses neither instances nor track listings. #[kio::test] -async fn sessions_carry_messages_both_ways() { +async fn tcp_sessions_carry_writes_both_ways() { for (dialer, acceptor) in [(LITE, LITE), (LITE, IETF), (IETF, LITE), (IETF, IETF)] { let mut fixture = Fixture::new(); let alice = fixture.endpoint(dialer, "alice").await; let bob = fixture.endpoint(acceptor, "bob").await; - let (mut dialed, mut accepted) = connect(&alice, &bob).await; + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; assert_eq!(dialed.peer(), "bob"); assert_eq!(accepted.peer(), "alice"); assert_eq!(dialed.session(), accepted.session()); + for write in [b"one", b"two", b"six"] { + dialed.producer.write_frame(None, &write[..]).unwrap(); + } + for write in [b"one", b"two", b"six"] { + assert_eq!( + read(&mut accepted.consumer).await.unwrap().unwrap(), + write, + "{dialer} to {acceptor}" + ); + } + accepted.producer.write_frame(None, &b"pong"[..]).unwrap(); + assert_eq!(read(&mut dialed.consumer).await.unwrap().unwrap(), b"pong"); + + // FIN, each way. + dialed.producer.finish().unwrap(); + assert_eq!(read(&mut accepted.consumer).await.unwrap(), None, "{dialer} to {acceptor}"); + accepted.producer.finish().unwrap(); + assert_eq!(read(&mut dialed.consumer).await.unwrap(), None, "{dialer} to {acceptor}"); + } +} + +/// A long session is one group of many writes, read in order. +#[kio::test] +async fn a_long_tcp_session_stays_in_order() { + const WRITES: u32 = 2000; + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; + + let write = async { + for i in 0..WRITES { + let payload = Bytes::copy_from_slice(&i.to_be_bytes()); + dialed.producer.ready(&frame::Info { size: 4, timestamp: None }).await; + dialed.producer.write_frame(None, payload).unwrap(); + } + dialed.producer.finish().unwrap(); + }; + let read_all = async { + for i in 0..WRITES { + assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), i.to_be_bytes()); + } + assert_eq!(read(&mut accepted.consumer).await.unwrap(), None); + }; + futures::join!(write, read_all); +} + +/// Aborting a half is RST: the peer's read fails instead of ending. +#[kio::test] +async fn a_reset_tcp_half_fails_the_read() { + for version in [LITE, IETF] { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(version, "alice").await; + let bob = fixture.endpoint(version, "bob").await; + let (dialed, mut accepted) = connect_tcp(&alice, &bob).await; + let mut producer = dialed.producer; + producer.write_frame(None, &b"partial"[..]).unwrap(); + assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), b"partial"); + producer.abort(Error::Cancel).unwrap(); + assert!(read(&mut accepted.consumer).await.is_err(), "{version}: a reset read as FIN"); + } +} + +/// A slow reader paces the writer through every hop, and a session longer than a group's +/// live tail loses nothing: the writer waits instead of running ahead. +#[kio::test] +async fn a_slow_tcp_reader_paces_the_writer() { + // Well past the 32 MiB a group copy holds. + const WRITES: usize = 64; + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; + + let written = Arc::new(AtomicUsize::new(0)); + let write = { + let written = written.clone(); + async move { + for i in 0..WRITES { + let payload = Bytes::from(vec![i as u8; MB]); + let info = frame::Info { + size: MB as u64, + timestamp: None, + }; + dialed.producer.ready(&info).await; + dialed.producer.write_frame(None, payload).unwrap(); + written.fetch_add(1, Ordering::SeqCst); + } + dialed.producer.finish().unwrap(); + } + }; + let read_all = async { + // Not reading for a while: the writer stalls well short of the end. + kio::clock::sleep(Duration::from_secs(5)).await; + let ahead = written.load(Ordering::SeqCst); + assert!(ahead < WRITES, "the writer ran {ahead} writes ahead of an idle reader"); + for i in 0..WRITES { + let frame = read(&mut accepted.consumer).await.unwrap().expect("a write"); + assert_eq!(frame.len(), MB); + assert!(frame.iter().all(|byte| *byte == i as u8), "write {i} out of order"); + // Slowly. + kio::clock::sleep(Duration::from_millis(10)).await; + } + assert_eq!(read(&mut accepted.consumer).await.unwrap(), None); + }; + kio::clock::timeout(Duration::from_secs(120), async { futures::join!(write, read_all) }) + .await + .expect("session timeout"); +} + +/// A peer that writes a second group in a `tcp` session breaks the protocol: the session +/// is reset, both halves. +#[kio::test] +async fn a_second_tcp_group_resets_the_session() { + let mut fixture = Fixture::new(); + let bob = fixture.endpoint(LITE, "bob").await; + // Carol dials by hand, as a nonconforming implementation would. + let carol = fixture.client(LITE).await; + let endpoint = carol.create_broadcast("room/.socket/carol").unwrap(); + endpoint.announce(Default::default()).unwrap(); + let mut requests = endpoint.requests(); + let name = "tcp/carol/0123456789abcdef0123456789abcdef"; + let broadcast = kio::clock::timeout(TIMEOUT, carol.consume().request("room/.socket/bob", None)) + .await + .expect("request timeout") + .expect("bob's endpoint"); + let dial = broadcast.track(name).unwrap().subscribe(None); + + let accept = async { + let incoming = bob.accept_tcp().await.unwrap(); + incoming.accept().await + }; + let answer = async { + let request = kio::wait(|waiter| requests.poll_requested_track(waiter)).await.unwrap(); + assert_eq!(request.name(), name); + request.accept(None) + }; + let (accepted, half, dialed) = kio::clock::timeout(TIMEOUT, async { futures::join!(accept, answer, dial) }) + .await + .expect("connect timeout"); + let mut accepted = accepted.expect("accept"); + let _dialed = dialed.expect("bob's half"); + + for sequence in [0, 1] { + let mut group = half.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, &b"data"[..]).unwrap(); + group.finish().unwrap(); + } + // Bob reads at most the first group's write, then the reset. + let mut reads = 0; + loop { + match read(&mut accepted.consumer).await { + Ok(Some(_)) => reads += 1, + Ok(None) => panic!("a second group read as FIN"), + Err(_) => break, + } + assert!(reads <= 1, "read past the first group"); + } + // And bob's own half was reset with it. + kio::clock::sleep(SETTLE).await; + assert!(accepted.producer.write_frame(None, &b"late"[..]).is_err()); +} + +/// Messages flow both ways, in order, over every pairing of moq-lite-07 and +/// moq-transport. +#[kio::test] +async fn track_sessions_carry_messages_both_ways() { + for (dialer, acceptor) in [(LITE, LITE), (LITE, IETF), (IETF, LITE), (IETF, IETF)] { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(dialer, "alice").await; + let bob = fixture.endpoint(acceptor, "bob").await; + let (dialed, accepted) = connect_track(&alice, &bob).await; + assert_eq!(dialed.peer(), "bob"); + assert_eq!(dialed.session(), accepted.session()); + let mut reader = accepted.consumer.ordered(); + let mut replies = dialed.consumer.ordered(); + for message in [b"one", b"two", b"six"] { send(&dialed.producer, message); } for message in [b"one", b"two", b"six"] { assert_eq!( - recv(&mut accepted.consumer).await.unwrap().unwrap(), + recv(&mut reader).await.unwrap().unwrap(), message, "{dialer} to {acceptor}" ); } send(&accepted.producer, b"pong"); - assert_eq!(recv(&mut dialed.consumer).await.unwrap().unwrap(), b"pong"); + assert_eq!(recv(&mut replies).await.unwrap().unwrap(), b"pong"); // Finishing a half closes the pipe that way. dialed.producer.finish().unwrap(); - assert_eq!(recv(&mut accepted.consumer).await.unwrap(), None); + assert_eq!(recv(&mut reader).await.unwrap(), None); } } -/// A pair of endpoints holds several sessions at once, each its own pipe. +/// Messages are read in sequence even when they are written, and so arrive, out of order. +#[kio::test] +async fn track_messages_are_read_in_sequence() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (dialed, accepted) = connect_track(&alice, &bob).await; + let mut reader = accepted.consumer.ordered(); + + let write = |sequence: u64, payload: &'static [u8]| { + let mut group = dialed.producer.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, payload).unwrap(); + group.finish().unwrap(); + }; + write(1, b"second"); + // Let the later message arrive first. + assert!(kio::clock::timeout(SETTLE, reader.next_group()).await.is_err()); + write(0, b"first"); + assert_eq!(recv(&mut reader).await.unwrap().unwrap(), b"first"); + assert_eq!(recv(&mut reader).await.unwrap().unwrap(), b"second"); +} + +/// With a finite delay budget, a message that never arrives stops blocking the ones after +/// it once the budget runs out. +#[kio::test] +async fn a_lost_track_message_does_not_block_later_ones() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let (dialed, accepted) = connect_track(&alice, &bob).await; + let lossy = accepted.consumer.subscription().with_delay( + track::Delay::default().with_max(Duration::from_millis(500)), + ); + accepted.consumer.control().update(lossy).unwrap(); + let mut reader = accepted.consumer.ordered(); + + // Message 0 is never written. + for (sequence, payload) in [(1, &b"one"[..]), (2, &b"two"[..])] { + let mut group = dialed.producer.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, payload).unwrap(); + group.finish().unwrap(); + } + assert_eq!(recv(&mut reader).await.unwrap().unwrap(), b"one"); + assert_eq!(recv(&mut reader).await.unwrap().unwrap(), b"two"); +} + +/// A pair of endpoints holds several sessions at once in either protocol, each its own +/// pipe. #[kio::test] async fn several_sessions_share_a_pair() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; - let (first, mut first_accepted) = connect(&alice, &bob).await; - let (second, mut second_accepted) = connect(&alice, &bob).await; - // Either side may dial. - let (mut third, third_accepted) = connect(&bob, &alice).await; + let (mut first, mut first_accepted) = connect_tcp(&alice, &bob).await; + let (mut second, mut second_accepted) = connect_tcp(&alice, &bob).await; + // Either side may dial, in either protocol. + let (third, third_accepted) = connect_track(&bob, &alice).await; assert_ne!(first.session(), second.session()); - send(&second.producer, b"second"); - send(&first.producer, b"first"); + second.producer.write_frame(None, &b"second"[..]).unwrap(); + first.producer.write_frame(None, &b"first"[..]).unwrap(); send(&third_accepted.producer, b"third"); - assert_eq!(recv(&mut first_accepted.consumer).await.unwrap().unwrap(), b"first"); - assert_eq!(recv(&mut second_accepted.consumer).await.unwrap().unwrap(), b"second"); - assert_eq!(recv(&mut third.consumer).await.unwrap().unwrap(), b"third"); + assert_eq!(read(&mut first_accepted.consumer).await.unwrap().unwrap(), b"first"); + assert_eq!(read(&mut second_accepted.consumer).await.unwrap().unwrap(), b"second"); + assert_eq!(recv(&mut third.consumer.ordered()).await.unwrap().unwrap(), b"third"); + first.producer.finish().unwrap(); + second.producer.finish().unwrap(); } /// Rejecting a dial resets it. @@ -156,9 +412,9 @@ async fn a_rejected_dial_fails() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; - let dial = alice.dial("bob", None).unwrap(); + let dial = alice.dial_tcp("bob").unwrap(); let reject = async { - bob.accept().await.unwrap().reject(); + bob.accept_tcp().await.unwrap().reject(); }; let (dialed, ()) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, reject) }) .await @@ -166,24 +422,58 @@ async fn a_rejected_dial_fails() { assert!(dialed.is_err(), "a rejected dial connected"); } +/// A dial in a protocol the peer does not accept is reset, while the peer still accepts +/// the protocol it listens on. +#[kio::test] +async fn a_protocol_nobody_accepts_is_reset() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let dial = alice.dial_track("bob", None).unwrap(); + let accept = async { kio::clock::timeout(SETTLE, bob.accept_tcp()).await }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) + .await + .expect("dial timeout"); + assert!(dialed.is_err(), "a track dial reached a tcp listener"); + assert!(accepted.is_err(), "a track dial surfaced as a tcp dial"); + + // An unknown protocol is reset too. + let mallory = fixture.client(LITE).await; + let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.socket/bob", None)) + .await + .expect("request timeout") + .expect("bob's endpoint"); + let unknown = broadcast + .track("quic/mallory/0123456789abcdef0123456789abcdef") + .unwrap() + .subscribe(None); + let unknown = async { kio::clock::timeout(TIMEOUT, unknown).await }; + let accept = async { kio::clock::timeout(SETTLE, bob.accept_tcp()).await }; + let (unknown, accepted) = futures::join!(unknown, accept); + assert!(unknown.expect("unknown timeout").is_err()); + assert!(accepted.is_err(), "an unknown protocol surfaced as a dial"); +} + /// A dial waits for its peer's endpoint to be announced, and a dial to itself fails. #[kio::test] async fn a_dial_waits_for_its_peer() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; - assert!(matches!(alice.dial("alice", None), Err(Error::Unsupported))); - let mut dial = alice.dial("bob", None).unwrap(); + assert!(matches!(alice.dial_tcp("alice"), Err(Error::Unsupported))); + let mut dial = alice.dial_tcp("bob").unwrap(); assert!( kio::clock::timeout(SETTLE, &mut dial).await.is_err(), "a dial to nobody resolved" ); let bob = fixture.endpoint(LITE, "bob").await; - let accept = async { bob.accept().await?.accept(None).await }; + let accept = async { bob.accept_tcp().await?.accept().await }; let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) .await .expect("connect timeout"); - assert_eq!(dialed.unwrap().session(), accepted.unwrap().session()); + let (mut dialed, accepted) = (dialed.unwrap(), accepted.unwrap()); + assert_eq!(dialed.session(), accepted.session()); + dialed.producer.finish().unwrap(); } /// An accept of a dial claiming an endpoint nobody announces fails. @@ -192,20 +482,20 @@ async fn a_dial_from_nobody_fails() { let mut fixture = Fixture::new(); let bob = fixture.endpoint(LITE, "bob").await; let mallory = fixture.client(LITE).await; - let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.tcp/bob", None)) + let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.socket/bob", None)) .await .expect("request timeout") .expect("bob's endpoint"); let forged = broadcast - .track("carol/0123456789abcdef0123456789abcdef") + .track("tcp/carol/0123456789abcdef0123456789abcdef") .unwrap() .subscribe(None); - let incoming = kio::clock::timeout(TIMEOUT, bob.accept()) + let incoming = kio::clock::timeout(TIMEOUT, bob.accept_tcp()) .await .expect("accept timeout") .unwrap(); assert_eq!(incoming.peer(), "carol"); - let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)) + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept()) .await .expect("accept timeout"); assert!(accepted.is_err(), "bob connected to nobody"); @@ -219,37 +509,53 @@ async fn a_dial_from_nobody_fails() { /// A third party asking bob for a session in alice's name surfaces at bob as a dial from /// alice, but alice never dialed it, so she resets bob's accept, and the forger reads -/// nothing. +/// nothing. The same holds in either protocol. #[kio::test] async fn a_forged_dial_never_connects() { - let mut fixture = Fixture::new(); - let alice = fixture.endpoint(LITE, "alice").await; - let bob = fixture.endpoint(LITE, "bob").await; - let mallory = fixture.client(LITE).await; - - let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.tcp/bob", None)) - .await - .expect("request timeout") - .expect("bob's endpoint"); - let forged = broadcast - .track("alice/0123456789abcdef0123456789abcdef") - .unwrap() - .subscribe(None); - - let incoming = kio::clock::timeout(TIMEOUT, bob.accept()) - .await - .expect("accept timeout") - .unwrap(); - assert_eq!(incoming.peer(), "alice"); - let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)) - .await - .expect("accept timeout"); - assert!(accepted.is_err(), "bob connected to a forged dial"); - let forged = kio::clock::timeout(TIMEOUT, forged).await.expect("forged timeout"); - assert!(forged.is_err(), "the forger read bob's half"); + for protocol in ["tcp", "track"] { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let mallory = fixture.client(LITE).await; - // Alice never heard of it as a dial either. - assert!(kio::clock::timeout(SETTLE, alice.accept()).await.is_err()); + let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.socket/bob", None)) + .await + .expect("request timeout") + .expect("bob's endpoint"); + let forged = broadcast + .track(&format!("{protocol}/alice/0123456789abcdef0123456789abcdef")) + .unwrap() + .subscribe(None); + + let (incoming_peer, accepted) = match protocol { + "tcp" => { + let incoming = kio::clock::timeout(TIMEOUT, bob.accept_tcp()) + .await + .expect("accept timeout") + .unwrap(); + let peer = incoming.peer().to_string(); + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept()).await; + (peer, accepted.expect("accept timeout").map(drop)) + } + _ => { + let incoming = kio::clock::timeout(TIMEOUT, bob.accept_track()) + .await + .expect("accept timeout") + .unwrap(); + let peer = incoming.peer().to_string(); + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)).await; + (peer, accepted.expect("accept timeout").map(drop)) + } + }; + assert_eq!(incoming_peer, "alice"); + assert!(accepted.is_err(), "bob connected to a forged {protocol} dial"); + let forged = kio::clock::timeout(TIMEOUT, forged).await.expect("forged timeout"); + assert!(forged.is_err(), "the forger read bob's {protocol} half"); + + // Alice never heard of it as a dial either. + assert!(kio::clock::timeout(SETTLE, alice.accept_tcp()).await.is_err()); + assert!(kio::clock::timeout(SETTLE, alice.accept_track()).await.is_err()); + } } /// A dial dropped before the peer accepts resets the accept. @@ -258,10 +564,10 @@ async fn a_dropped_dial_resets_the_accept() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; - let mut dial = alice.dial("bob", None).unwrap(); + let mut dial = alice.dial_tcp("bob").unwrap(); // Polled until bob hears of it, then dropped. let incoming = kio::clock::timeout(TIMEOUT, async { - match futures::future::select(std::pin::pin!(bob.accept()), &mut dial).await { + match futures::future::select(std::pin::pin!(bob.accept_tcp()), &mut dial).await { futures::future::Either::Left((incoming, _)) => incoming, futures::future::Either::Right(_) => panic!("the dial resolved unaccepted"), } @@ -270,28 +576,38 @@ async fn a_dropped_dial_resets_the_accept() { .expect("accept timeout") .unwrap(); drop(dial); - let accepted = kio::clock::timeout(TIMEOUT, incoming.accept(None)) + let accepted = kio::clock::timeout(TIMEOUT, incoming.accept()) .await .expect("accept timeout"); assert!(accepted.is_err(), "bob connected to a dropped dial"); } -/// Closing an endpoint aborts its sessions for the peer, and fails its pending work. +/// Closing an endpoint resets its sessions for the peer, and fails its pending work. #[kio::test] async fn closing_an_endpoint_ends_its_sessions() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; - let (mut dialed, accepted) = connect(&alice, &bob).await; + let (mut tcp_dialed, mut tcp_accepted) = connect_tcp(&alice, &bob).await; + let (track_dialed, track_accepted) = connect_track(&alice, &bob).await; bob.close(); assert!( - recv(&mut dialed.consumer).await.is_err(), - "bob's half outlived his endpoint" + read(&mut tcp_dialed.consumer).await.is_err(), + "bob's tcp half outlived his endpoint" + ); + // Both halves of bob's own tcp session end. + assert!(read(&mut tcp_accepted.consumer).await.is_err()); + assert!(tcp_accepted.producer.write_frame(None, &b"late"[..]).is_err()); + assert!( + recv(&mut track_dialed.consumer.ordered()).await.is_err(), + "bob's track half outlived his endpoint" ); - assert!(accepted.producer.append_group().is_err()); - assert!(matches!(bob.accept().await, Err(Error::Closed))); - assert!(matches!(bob.dial("alice", None), Err(Error::Closed))); + assert!(track_accepted.producer.append_group().is_err()); + assert!(matches!(bob.accept_tcp().await, Err(Error::Closed))); + assert!(matches!(bob.accept_track().await, Err(Error::Closed))); + assert!(matches!(bob.dial_tcp("alice"), Err(Error::Closed))); + tcp_dialed.producer.finish().unwrap(); } /// Session tracks are the secret that keeps third parties out, so no listing names them: @@ -302,13 +618,19 @@ async fn session_tracks_are_never_listed() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; - let (dialed, mut accepted) = connect(&alice, &bob).await; - send(&dialed.producer, b"secret"); - assert_eq!(recv(&mut accepted.consumer).await.unwrap().unwrap(), b"secret"); + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; + let (track_dialed, track_accepted) = connect_track(&alice, &bob).await; + dialed.producer.write_frame(None, &b"secret"[..]).unwrap(); + assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), b"secret"); + send(&track_dialed.producer, b"secret"); + assert_eq!( + recv(&mut track_accepted.consumer.ordered()).await.unwrap().unwrap(), + b"secret" + ); let eve = fixture.client(LITE).await; for id in ["alice", "bob"] { - let broadcast = kio::clock::timeout(TIMEOUT, eve.consume().request(format!("room/.tcp/{id}"), None)) + let broadcast = kio::clock::timeout(TIMEOUT, eve.consume().request(format!("room/.socket/{id}"), None)) .await .expect("request timeout") .expect("the endpoint"); @@ -323,6 +645,7 @@ async fn session_tracks_are_never_listed() { "{id} listed a track" ); } + dialed.producer.finish().unwrap(); } /// An endpoint takes dials only from its first accept on, like a TCP listener: before it, @@ -332,33 +655,8 @@ async fn an_endpoint_not_accepting_resets_dials() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let _bob = fixture.endpoint(LITE, "bob").await; - let dialed = kio::clock::timeout(TIMEOUT, alice.dial("bob", None).unwrap()) + let dialed = kio::clock::timeout(TIMEOUT, alice.dial_tcp("bob").unwrap()) .await .expect("dial timeout"); assert!(dialed.is_err(), "a dial to an endpoint not accepting connected"); } - -/// Messages are read in sequence even when they are written, and so arrive, out of order. -#[kio::test] -async fn messages_are_read_in_sequence() { - let mut fixture = Fixture::new(); - let alice = fixture.endpoint(LITE, "alice").await; - let bob = fixture.endpoint(LITE, "bob").await; - let (dialed, mut accepted) = connect(&alice, &bob).await; - - let write = |sequence: u64, payload: &'static [u8]| { - let mut group = dialed.producer.create_group(moq_net::group::Info { sequence }).unwrap(); - group.write_frame(None, payload).unwrap(); - group.finish().unwrap(); - }; - write(1, b"second"); - // Let the later message arrive first. - assert!( - kio::clock::timeout(SETTLE, accepted.consumer.next_group()) - .await - .is_err() - ); - write(0, b"first"); - assert_eq!(recv(&mut accepted.consumer).await.unwrap().unwrap(), b"first"); - assert_eq!(recv(&mut accepted.consumer).await.unwrap().unwrap(), b"second"); -} diff --git a/rs/moq-relay/tests/integration/socket_auth.rs b/rs/moq-relay/tests/integration/socket_auth.rs index af34d17cc9a..a30005bf982 100644 --- a/rs/moq-relay/tests/integration/socket_auth.rs +++ b/rs/moq-relay/tests/integration/socket_auth.rs @@ -2,14 +2,14 @@ //! auth server, granting only the prefixes `doc/concept/socket.md` lists. //! //! Every token is rooted at the socket prefix `room`. An endpoint's token publishes its own -//! endpoint, `.tcp/`, and subscribes to every endpoint, `.tcp/**`, so the session -//! track names are what keep third parties out: a forged dial never connects, and no +//! endpoint, `.socket/`, and subscribes to every endpoint, `.socket/**`, so the +//! session track names are what keep third parties out: a forged dial never connects, and no //! listing gives a session's name away. use std::time::Duration; use moq_net::broadcast::TracksEvent; -use moq_net::socket::{Endpoint, Stream}; +use moq_net::socket::{Endpoint, TcpStream, TrackStream}; use crate::memory; @@ -93,7 +93,7 @@ impl Fixture { /// its own endpoint, subscribe to every endpoint. async fn endpoint(&self, id: &str, version: &str) -> (Endpoint, moq_net::origin::Producer, moq_net::Session) { let (origin, session) = self - .client(&self.token(&format!(".tcp/{id}"), ".tcp/**"), version) + .client(&self.token(&format!(".socket/{id}"), ".socket/**"), version) .await; // The session is rooted at the prefix, so the endpoint's prefix is the root. let endpoint = Endpoint::new(&origin, "", id).expect("endpoint"); @@ -101,11 +101,26 @@ impl Fixture { } } -/// Dial `acceptor` from `dialer` while `acceptor` accepts, returning both ends. -async fn connect(dialer: &Endpoint, acceptor: &Endpoint) -> (Stream, Stream) { - let dial = dialer.dial(acceptor.id(), None).expect("dial"); +/// Dial `acceptor` from `dialer` in a `tcp` session while `acceptor` accepts, returning +/// both ends. +async fn connect_tcp(dialer: &Endpoint, acceptor: &Endpoint) -> (TcpStream, TcpStream) { + let dial = dialer.dial_tcp(acceptor.id()).expect("dial"); let accept = async { - let incoming = acceptor.accept().await?; + let incoming = acceptor.accept_tcp().await?; + assert_eq!(incoming.peer(), dialer.id()); + incoming.accept().await + }; + let (dialed, accepted) = kio::clock::timeout(TIMEOUT, async { futures::join!(dial, accept) }) + .await + .expect("connect timeout"); + (dialed.expect("dial"), accepted.expect("accept")) +} + +/// Dial `acceptor` from `dialer` in a `track` session; see [`connect_tcp`]. +async fn connect_track(dialer: &Endpoint, acceptor: &Endpoint) -> (TrackStream, TrackStream) { + let dial = dialer.dial_track(acceptor.id(), None).expect("dial"); + let accept = async { + let incoming = acceptor.accept_track().await?; assert_eq!(incoming.peer(), dialer.id()); incoming.accept(None).await }; @@ -115,14 +130,25 @@ async fn connect(dialer: &Endpoint, acceptor: &Endpoint) -> (Stream, Stream) { (dialed.expect("dial"), accepted.expect("accept")) } -/// Write `payload` as one message: one group of `track`. +/// Read the next write from a `tcp` session. +async fn read(reader: &mut moq_net::group::Consumer) -> Vec { + kio::clock::timeout(TIMEOUT, reader.read_frame()) + .await + .expect("read timeout") + .expect("frame") + .expect("a frame") + .payload + .to_vec() +} + +/// Write `payload` as one message of a `track` session: one group. fn send(track: &moq_net::track::Producer, payload: &'static [u8]) { let mut group = track.append_group().expect("group"); group.write_frame(None, payload).expect("frame"); group.finish().expect("finish"); } -/// Read the next message. +/// Read the next message of a `track` session. async fn recv(reader: &mut moq_net::track::Ordered) -> Vec { kio::clock::timeout(TIMEOUT, async { let mut group = reader.next_group().await.expect("group").expect("a group"); @@ -133,19 +159,29 @@ async fn recv(reader: &mut moq_net::track::Ordered) -> Vec { .expect("read timeout") } -/// The prefix grants admit a dial and its accept, and the session carries data both ways, -/// over moq-lite-07 and across a moq-transport hop. +/// The prefix grants admit a dial and its accept in either protocol, and the session +/// carries data, over moq-lite-07 and across a moq-transport hop. #[tokio::test] async fn prefix_grants_admit_a_session() { for (dialer, acceptor) in [(LITE, LITE), (LITE, IETF), (IETF, LITE)] { let fixture = Fixture::new().await; let (alice, _alice_origin, _alice_session) = fixture.endpoint("alice", dialer).await; let (bob, _bob_origin, _bob_session) = fixture.endpoint("bob", acceptor).await; - let (mut dialed, mut accepted) = connect(&alice, &bob).await; + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; + dialed.producer.write_frame(None, &b"ping"[..]).expect("write"); + assert_eq!(read(&mut accepted.consumer).await, b"ping", "{dialer} to {acceptor}"); + accepted.producer.write_frame(None, &b"pong"[..]).expect("write"); + assert_eq!(read(&mut dialed.consumer).await, b"pong", "{dialer} to {acceptor}"); + dialed.producer.finish().expect("finish"); + accepted.producer.finish().expect("finish"); + + let (dialed, accepted) = connect_track(&alice, &bob).await; send(&dialed.producer, b"ping"); - assert_eq!(recv(&mut accepted.consumer).await, b"ping", "{dialer} to {acceptor}"); - send(&accepted.producer, b"pong"); - assert_eq!(recv(&mut dialed.consumer).await, b"pong", "{dialer} to {acceptor}"); + assert_eq!( + recv(&mut accepted.consumer.ordered()).await, + b"ping", + "{dialer} to {acceptor}" + ); } } @@ -160,23 +196,23 @@ async fn a_forged_dial_never_connects() { let (_mallory, mallory_origin, _mallory_session) = fixture.endpoint("mallory", LITE).await; // Alice keeps accepting, as a server would, which is how she resets the forged accept. let alice_accepts = async { - let _ = alice.accept().await; + let _ = alice.accept_tcp().await; panic!("alice heard of a dial"); }; let forge = async { let broadcast = mallory_origin .consume() - .request(".tcp/bob", None) + .request(".socket/bob", None) .await .expect("bob's endpoint"); let forged = broadcast - .track("alice/0123456789abcdef0123456789abcdef") + .track("tcp/alice/0123456789abcdef0123456789abcdef") .expect("track") .subscribe(None); - let incoming = bob.accept().await.expect("accept"); + let incoming = bob.accept_tcp().await.expect("accept"); assert_eq!(incoming.peer(), "alice"); - let (accepted, forged) = futures::join!(incoming.accept(None), forged); + let (accepted, forged) = futures::join!(incoming.accept(), forged); assert!(accepted.is_err(), "bob connected to a forged dial"); assert!(forged.is_err(), "mallory read bob's half"); }; @@ -197,13 +233,16 @@ async fn a_reader_of_everything_lists_no_session() { let fixture = Fixture::new().await; let (alice, _alice_origin, _alice_session) = fixture.endpoint("alice", LITE).await; let (bob, _bob_origin, _bob_session) = fixture.endpoint("bob", LITE).await; - let (dialed, mut accepted) = connect(&alice, &bob).await; - send(&dialed.producer, b"secret"); - assert_eq!(recv(&mut accepted.consumer).await, b"secret"); + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; + dialed.producer.write_frame(None, &b"secret"[..]).expect("write"); + assert_eq!(read(&mut accepted.consumer).await, b"secret"); + let (track_dialed, track_accepted) = connect_track(&alice, &bob).await; + send(&track_dialed.producer, b"secret"); + assert_eq!(recv(&mut track_accepted.consumer.ordered()).await, b"secret"); - let (eve_origin, _eve_session) = fixture.client(&fixture.token(".tcp/eve", "**"), LITE).await; + let (eve_origin, _eve_session) = fixture.client(&fixture.token(".socket/eve", "**"), LITE).await; for id in ["alice", "bob"] { - let broadcast = kio::clock::timeout(TIMEOUT, eve_origin.consume().request(format!(".tcp/{id}"), None)) + let broadcast = kio::clock::timeout(TIMEOUT, eve_origin.consume().request(format!(".socket/{id}"), None)) .await .expect("request timeout") .expect("the endpoint"); From 90262b5c0aba18f353604e46a72c4f3f770b549d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 23:57:38 -0800 Subject: [PATCH 18/25] fix(net): pace tcp sessions without stalling, and fix the socket tests Detect a second tcp group through the subscriber's largest position once group 0 arrived, since polling poll_recv_group on every wake stalled the group it handed out. Watch this side's group through its public Demand. Co-Authored-By: Claude Opus 5.5 --- quest/m1/socket/README.md | 5 ++- rs/moq-net/src/model/socket.rs | 45 ++++++++++++++---------- rs/moq-net/tests/integration/socket.rs | 48 ++++++++++++++++++-------- 3 files changed, 65 insertions(+), 33 deletions(-) diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index 9d2763fa010..39912ff645e 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -77,7 +77,10 @@ Implementation choices in that reader, so it can end that group itself: on a second group from the peer, on the endpoint closing, or when the peer resets. It also finishes or aborts this side's track the way the app ended its group, and resets the - session when the app drops its reader early. + session when the app drops its reader early. Once group 0 arrives, it + watches for a second group through the subscriber's largest position rather + than `poll_recv_group`: polling `poll_recv_group` on every wake while + reading the group it handed out stalled the group's delivery (see the PR). - `TrackStream::consumer` is the handshake's own `track::Subscriber`, not a bare `track::Consumer`: it is already subscribed reliably and in order, so no message falls between the accept and the app's first read, and the app diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 72df1b89c22..095e1128f22 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -212,13 +212,14 @@ impl sealed::Open for TcpStream { } = parts; let (producer, ours) = match track.create_group(group::Info { sequence: 0 }) { Ok(group) => { - let handle = group.handle(); - (group, Some(handle)) + let demand = group.demand(); + (group, Some(demand)) } // The track already ended, such as by the endpoint closing: hand out a half // that reports it. Err(err) => { - let group = group::Producer::new(group::Info { sequence: 0 }, track::Info::default(), Default::default()); + let group = + group::Producer::new(group::Info { sequence: 0 }, track::Info::default(), Default::default()); let _ = group.handle().abort(err); (group, None) } @@ -943,8 +944,8 @@ struct Pump { shared: Arc, /// This side's half, held so the application can write only its one group. track: Option, - /// This side's group, watched for how the application ends it. - ours: Option, + /// This side's group, watched for how the application ends it without keeping it open. + ours: Option, /// The peer's half, watched for its one group, and for a second. subscriber: Option, /// The peer's group, once it arrived. @@ -1003,27 +1004,35 @@ impl Pump { // The peer's half holds one group: a second is a protocol error. if let Some(subscriber) = &mut self.subscriber { - loop { + if self.remote.is_none() { match subscriber.poll_recv_group(waiter) { - Poll::Pending => break, + Poll::Pending => {} Poll::Ready(Ok(Some(group))) => { - if self.remote.is_some() || group.sequence != 0 { + if group.sequence != 0 { tracing::debug!(sequence = group.sequence, "a second group on a tcp session"); return Err(Error::ProtocolViolation); } self.remote = Some(group); } - Poll::Ready(Ok(None)) => { - self.subscriber = None; - break; - } - Poll::Ready(Err(err)) => { - // Ended short of its group: the peer's reset. - if self.remote.is_none() { - return Err(err); + Poll::Ready(Ok(None)) => self.subscriber = None, + // Ended short of its group: the peer's reset. + Poll::Ready(Err(err)) => return Err(err), + } + } + if self.remote.is_some() + && let Some(subscriber) = &mut self.subscriber + { + while let Poll::Ready(largest) = subscriber.poll_largest(waiter) { + match largest { + Some(largest) if largest.group > 0 => { + tracing::debug!(sequence = largest.group, "a second group on a tcp session"); + return Err(Error::ProtocolViolation); + } + Some(_) => {} + None => { + self.subscriber = None; + break; } - self.subscriber = None; - break; } } } diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index a4b7ab5f73b..396992cf4a7 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -68,7 +68,6 @@ impl Fixture { } } - const MB: usize = 1 << 20; /// Dial `acceptor` from `dialer` in a `tcp` session while `acceptor` accepts, returning @@ -156,9 +155,17 @@ async fn tcp_sessions_carry_writes_both_ways() { // FIN, each way. dialed.producer.finish().unwrap(); - assert_eq!(read(&mut accepted.consumer).await.unwrap(), None, "{dialer} to {acceptor}"); + assert_eq!( + read(&mut accepted.consumer).await.unwrap(), + None, + "{dialer} to {acceptor}" + ); accepted.producer.finish().unwrap(); - assert_eq!(read(&mut dialed.consumer).await.unwrap(), None, "{dialer} to {acceptor}"); + assert_eq!( + read(&mut dialed.consumer).await.unwrap(), + None, + "{dialer} to {acceptor}" + ); } } @@ -174,7 +181,13 @@ async fn a_long_tcp_session_stays_in_order() { let write = async { for i in 0..WRITES { let payload = Bytes::copy_from_slice(&i.to_be_bytes()); - dialed.producer.ready(&frame::Info { size: 4, timestamp: None }).await; + dialed + .producer + .ready(&frame::Info { + size: 4, + timestamp: None, + }) + .await; dialed.producer.write_frame(None, payload).unwrap(); } dialed.producer.finish().unwrap(); @@ -200,7 +213,10 @@ async fn a_reset_tcp_half_fails_the_read() { producer.write_frame(None, &b"partial"[..]).unwrap(); assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), b"partial"); producer.abort(Error::Cancel).unwrap(); - assert!(read(&mut accepted.consumer).await.is_err(), "{version}: a reset read as FIN"); + assert!( + read(&mut accepted.consumer).await.is_err(), + "{version}: a reset read as FIN" + ); } } @@ -208,8 +224,8 @@ async fn a_reset_tcp_half_fails_the_read() { /// live tail loses nothing: the writer waits instead of running ahead. #[kio::test] async fn a_slow_tcp_reader_paces_the_writer() { - // Well past the 32 MiB a group copy holds. - const WRITES: usize = 64; + // Well past the 32 MiB a group copy holds, and the window every hop may buffer. + const WRITES: usize = 192; let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; @@ -367,19 +383,22 @@ async fn a_lost_track_message_does_not_block_later_ones() { let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; let (dialed, accepted) = connect_track(&alice, &bob).await; - let lossy = accepted.consumer.subscription().with_delay( - track::Delay::default().with_max(Duration::from_millis(500)), - ); + let lossy = accepted + .consumer + .subscription() + .with_delay(track::Delay::default().with_max(Duration::from_millis(500))); accepted.consumer.control().update(lossy).unwrap(); let mut reader = accepted.consumer.ordered(); - // Message 0 is never written. - for (sequence, payload) in [(1, &b"one"[..]), (2, &b"two"[..])] { + let write = |sequence: u64, payload: &'static [u8]| { let mut group = dialed.producer.create_group(group::Info { sequence }).unwrap(); group.write_frame(None, payload).unwrap(); group.finish().unwrap(); - } + }; + // Message 0 is never written: message 1 waits out the budget, then is read. + write(1, b"one"); assert_eq!(recv(&mut reader).await.unwrap().unwrap(), b"one"); + write(2, b"two"); assert_eq!(recv(&mut reader).await.unwrap().unwrap(), b"two"); } @@ -607,7 +626,8 @@ async fn closing_an_endpoint_ends_its_sessions() { assert!(matches!(bob.accept_tcp().await, Err(Error::Closed))); assert!(matches!(bob.accept_track().await, Err(Error::Closed))); assert!(matches!(bob.dial_tcp("alice"), Err(Error::Closed))); - tcp_dialed.producer.finish().unwrap(); + // Alice saw bob's half reset, so she reset hers: the session is gone both ways. + assert!(tcp_dialed.producer.finish().is_err()); } /// Session tracks are the secret that keeps third parties out, so no listing names them: From 28e52a0f9bde669197a434993c8d06da647c3c1d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 00:03:23 -0800 Subject: [PATCH 19/25] test(relay): run the socket auth test on tokio after the simulator move The in-memory harness now runs on kio's simulator, but this test talks to the reference auth server over real HTTP, so it dials over its own Tokio-driven in-memory link. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/tests/integration/socket_auth.rs | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/rs/moq-relay/tests/integration/socket_auth.rs b/rs/moq-relay/tests/integration/socket_auth.rs index a30005bf982..9624cecdcea 100644 --- a/rs/moq-relay/tests/integration/socket_auth.rs +++ b/rs/moq-relay/tests/integration/socket_auth.rs @@ -10,8 +10,8 @@ use std::time::Duration; use moq_net::broadcast::TracksEvent; use moq_net::socket::{Endpoint, TcpStream, TrackStream}; - -use crate::memory; +use moq_net::transport::memory; +use moq_relay::{Accepted, Connection}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -57,7 +57,7 @@ impl Fixture { .expect("auth init"); Self { key, - cluster: memory::cluster(), + cluster: crate::spawn_cluster(Default::default()), auth, _keys: keys, _server: server, @@ -83,12 +83,41 @@ impl Fixture { .with_subscriber(origin.clone()) .with_url(url); let version = version.parse().expect("version"); - let session = memory::connect(client, version, &self.cluster, &self.auth) - .await - .expect("connect"); + let session = self.connect(client, version).await.expect("connect"); (origin, session) } + /// Dial the relay over a fresh in-memory link, as `client` speaking `version`. + /// + /// On Tokio rather than kio's simulator, which `crate::memory` uses, since the + /// reference auth server is a real HTTP server. + async fn connect( + &self, + client: moq_net::Client, + version: moq_net::Version, + ) -> Result { + let mut config = memory::Config::default(); + config.protocol = Some(version.alpn().to_string()); + let (client_link, relay_link) = memory::pair(config); + + let (cluster, auth) = (self.cluster.clone(), self.auth.clone()); + tokio::spawn(async move { + let server = moq_net::Server::new().with_versions(version.into()); + let Ok(handshake) = server.accept_request(moq_net::time::Instant::now(), relay_link).await else { + return; + }; + let request = Accepted::new(handshake, moq_auth::Transport::Quic); + Connection::new(request, cluster, auth).run().await; + }); + + let (session, driver) = client + .with_versions(version.into()) + .connect(moq_net::time::Instant::now(), client_link) + .await?; + tokio::spawn(moq_net::time::run(driver)); + Ok(session) + } + /// The endpoint `id`'s client, holding the grants `doc/concept/socket.md` lists: publish /// its own endpoint, subscribe to every endpoint. async fn endpoint(&self, id: &str, version: &str) -> (Endpoint, moq_net::origin::Producer, moq_net::Session) { From d5e854bdc038d4d76cf61ad16e84034eb58d4d4f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 00:10:05 -0800 Subject: [PATCH 20/25] fix(net): refuse a second tcp group after the first finished Keep watching the peer's half for a second group for as long as either half is open, starting in the same poll that receives group 0. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/socket.rs | 84 ++++++++++++++------------ rs/moq-net/tests/integration/socket.rs | 74 ++++++++++++++++++----- 2 files changed, 106 insertions(+), 52 deletions(-) diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 095e1128f22..f9d9e860af3 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -233,6 +233,7 @@ impl sealed::Open for TcpStream { track: Some(track), ours, subscriber: Some(subscriber), + received: false, remote: None, local_handle: local.handle(), local: Some(local), @@ -948,7 +949,9 @@ struct Pump { ours: Option, /// The peer's half, watched for its one group, and for a second. subscriber: Option, - /// The peer's group, once it arrived. + /// Whether the peer's group arrived. + received: bool, + /// The peer's group, from its arrival until it ends. remote: Option, /// The application's copy of the peer's group, until it ends. local: Option, @@ -994,53 +997,58 @@ impl Pump { } } - if self.local.is_none() { - return Ok(()); - } - // Nobody reads what the peer writes. - if self.local_handle.poll_unwanted(waiter).is_ready() { - return Err(Error::Cancel); - } - - // The peer's half holds one group: a second is a protocol error. - if let Some(subscriber) = &mut self.subscriber { - if self.remote.is_none() { - match subscriber.poll_recv_group(waiter) { - Poll::Pending => {} - Poll::Ready(Ok(Some(group))) => { - if group.sequence != 0 { - tracing::debug!(sequence = group.sequence, "a second group on a tcp session"); - return Err(Error::ProtocolViolation); - } - self.remote = Some(group); + // The peer's half holds one group: a second is a protocol error, even after the + // first finished, for as long as either half is open. + if !self.received + && let Some(subscriber) = &mut self.subscriber + { + match subscriber.poll_recv_group(waiter) { + Poll::Pending => {} + Poll::Ready(Ok(Some(group))) => { + if group.sequence != 0 { + tracing::debug!(sequence = group.sequence, "a second group on a tcp session"); + return Err(Error::ProtocolViolation); } - Poll::Ready(Ok(None)) => self.subscriber = None, - // Ended short of its group: the peer's reset. - Poll::Ready(Err(err)) => return Err(err), + self.received = true; + self.remote = Some(group); } + Poll::Ready(Ok(None)) => self.subscriber = None, + // Ended short of its group: the peer's reset. + Poll::Ready(Err(err)) => return Err(err), } - if self.remote.is_some() - && let Some(subscriber) = &mut self.subscriber - { - while let Poll::Ready(largest) = subscriber.poll_largest(waiter) { - match largest { - Some(largest) if largest.group > 0 => { - tracing::debug!(sequence = largest.group, "a second group on a tcp session"); - return Err(Error::ProtocolViolation); - } - Some(_) => {} - None => { - self.subscriber = None; - break; - } + } + if self.received + && let Some(subscriber) = &mut self.subscriber + { + // Not `poll_recv_group`: polled on every wake while reading the group it handed + // out, it stalled that group's delivery. + while let Poll::Ready(largest) = subscriber.poll_largest(waiter) { + match largest { + Some(largest) if largest.group > 0 => { + tracing::debug!(sequence = largest.group, "a second group on a tcp session"); + return Err(Error::ProtocolViolation); + } + Some(_) => {} + None => { + self.subscriber = None; + break; } } } } + if self.local.is_none() { + return Ok(()); + } + // Nobody reads what the peer writes. + if self.local_handle.poll_unwanted(waiter).is_ready() { + return Err(Error::Cancel); + } + let Some(remote) = &mut self.remote else { // Finished without a group: an empty half. - if self.subscriber.is_none() + if !self.received + && self.subscriber.is_none() && let Some(mut local) = self.local.take() { let _ = local.finish(); diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 396992cf4a7..ada58d4d58f 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -267,13 +267,19 @@ async fn a_slow_tcp_reader_paces_the_writer() { .expect("session timeout"); } -/// A peer that writes a second group in a `tcp` session breaks the protocol: the session -/// is reset, both halves. -#[kio::test] -async fn a_second_tcp_group_resets_the_session() { - let mut fixture = Fixture::new(); - let bob = fixture.endpoint(LITE, "bob").await; - // Carol dials by hand, as a nonconforming implementation would. +/// A `tcp` session carol dialed by hand, as a nonconforming implementation would. +struct ByHand { + /// Bob's end. + accepted: TcpStream, + /// Carol's half, which she writes groups to as she likes. + half: track::Producer, + _dialed: track::Subscriber, + _endpoint: origin::Publication, + _carol: origin::Producer, +} + +/// Dial bob by hand from carol in a `tcp` session. +async fn dial_by_hand(fixture: &mut Fixture, bob: &Endpoint) -> ByHand { let carol = fixture.client(LITE).await; let endpoint = carol.create_broadcast("room/.socket/carol").unwrap(); endpoint.announce(Default::default()).unwrap(); @@ -297,14 +303,34 @@ async fn a_second_tcp_group_resets_the_session() { let (accepted, half, dialed) = kio::clock::timeout(TIMEOUT, async { futures::join!(accept, answer, dial) }) .await .expect("connect timeout"); - let mut accepted = accepted.expect("accept"); - let _dialed = dialed.expect("bob's half"); - - for sequence in [0, 1] { - let mut group = half.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"data"[..]).unwrap(); - group.finish().unwrap(); + ByHand { + accepted: accepted.expect("accept"), + half, + _dialed: dialed.expect("bob's half"), + _endpoint: endpoint, + _carol: carol, } +} + +/// Write `sequence` as a finished group of one write. +fn write_group(half: &track::Producer, sequence: u64) { + let mut group = half.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, &b"data"[..]).unwrap(); + group.finish().unwrap(); +} + +/// A peer that writes a second group in a `tcp` session breaks the protocol: the session +/// is reset, both halves. +#[kio::test] +async fn a_second_tcp_group_resets_the_session() { + let mut fixture = Fixture::new(); + let bob = fixture.endpoint(LITE, "bob").await; + // Bound whole, so carol's endpoint and read stay open to the end. + let mut session = dial_by_hand(&mut fixture, &bob).await; + let (accepted, half) = (&mut session.accepted, &session.half); + + write_group(half, 0); + write_group(half, 1); // Bob reads at most the first group's write, then the reset. let mut reads = 0; loop { @@ -320,6 +346,26 @@ async fn a_second_tcp_group_resets_the_session() { assert!(accepted.producer.write_frame(None, &b"late"[..]).is_err()); } +/// A second group after the first one finished, and was read to its end, still resets +/// the half that remains open. +#[kio::test] +async fn a_second_tcp_group_after_fin_resets_the_session() { + let mut fixture = Fixture::new(); + let bob = fixture.endpoint(LITE, "bob").await; + // Bound whole, so carol's endpoint and read stay open to the end. + let mut session = dial_by_hand(&mut fixture, &bob).await; + let (accepted, half) = (&mut session.accepted, &session.half); + + write_group(half, 0); + assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), b"data"); + assert_eq!(read(&mut accepted.consumer).await.unwrap(), None); + accepted.producer.write_frame(None, &b"still open"[..]).unwrap(); + + write_group(half, 1); + kio::clock::sleep(SETTLE).await; + assert!(accepted.producer.write_frame(None, &b"late"[..]).is_err()); +} + /// Messages flow both ways, in order, over every pairing of moq-lite-07 and /// moq-transport. #[kio::test] From 3fd88d8611e565aa166df05795fe030a72d14c31 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 00:49:57 -0800 Subject: [PATCH 21/25] fix(net): wake socket dials outside every lock, and pass on a peer reset after FIN - An endpoint polls its request handler outside its own lock, behind a lock of its own, and `broadcast::Requests` claims a request after releasing the broadcast's lock: claiming wakes the dialer's subscription, and an inline executor that closes the endpoint from that wake deadlocked on either lock. - A `tcp` session's task asks how the peer's track ended once its group arrived, so a reset after FIN resets the half that remains open; a clean finish still leaves it open. - Docs: a dial replayed from a session the claimed dialer really made passes the return-routability check; `track` messages nobody reads within the cache's idle window can be reclaimed; `accept_*` yields an `Incoming`. Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 29 +++-- doc/lib/rs/moq-net.md | 2 +- rs/moq-net/src/model/broadcast.rs | 4 + rs/moq-net/src/model/socket.rs | 160 ++++++++++++++++++++----- rs/moq-net/tests/integration/socket.rs | 28 +++++ 5 files changed, 183 insertions(+), 40 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 16feb3cb73d..55c50f728e6 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -8,7 +8,8 @@ description: Dial and accept point-to-point sessions over MoQ, as reliable byte A socket connects two endpoints through relays, the way a TCP connection connects two hosts. Each endpoint announces one hidden broadcast, dials other endpoints, and accepts their dials. Each connection is a session that only the -two ends can read, in one of two protocols: `tcp`, a reliable, ordered byte +two ends can read while its id stays secret (see [Hidden names](#hidden-names)), +in one of two protocols: `tcp`, a reliable, ordered byte pipe each way, or `track`, a track each way whose groups are messages. Use a socket for per-client state that only the two ends need, such as a viewer's commands and the replies to them, while whatever every client needs stays in an @@ -78,7 +79,8 @@ or others. and resets any other. That makes the exchange a return-routability check: a dial forged in `A`'s name, by anyone who can subscribe to `B`, reaches an `A` that never dialed it, so `A` resets `B`'s subscription and `B` resets the - forger's. + forger's. A dial replayed from a session `A` really dialed passes; see + [Hidden names](#hidden-names). 3. Once `A` answers, `B` answers `A`'s subscription, and the session is open for both. `B` writes nothing before `A` answers. @@ -92,8 +94,9 @@ track nobody serves. A dial that gives up resets `B`'s subscription in turn. ## Protocols Each side writes its own half and reads the peer's. A reader subscribes -reliably and in order: oldest group first, with no delay budget, so it never -skips anything. The protocol decides what a half holds. +reliably and in order: oldest group first, with no delay budget, so it waits +for a late group rather than skipping it. The protocol decides what a half +holds. ### `tcp` @@ -128,7 +131,10 @@ such as a proxied TCP connection, a file transfer, or a request and its reply. Each half is a whole track, and a message is one group. A slow message never holds up the next, and each group can carry its own priority. Messages are -ordered by group sequence. +ordered by group sequence. A reader that keeps up gets every message, but each +hop's cache reclaims a message nobody read within its idle window (30 s by +default), or under memory pressure, except the newest, so a reader that stalls +longer can miss messages. The reader's subscription starts reliable and in order, like `tcp`. An application may make it lossy for messages that go stale, such as positions or @@ -156,8 +162,17 @@ as a game's status, stays an ordinary broadcast. The session id is the secret that keeps a session to its two ends: the token grants below let any endpoint subscribe to any other, so whoever learns a -session's track name can read it, or answer it before its peer. A session -track therefore never appears where a third party can read it: +session's track name can read it, or answer it before its peer. + +That includes the acceptor. The name says who dialed, but not whom, so the +return-routability check proves only that `A` dialed `S`, not that it dialed +`B`. An endpoint `A` dialed knows `P/A/S`, and can dial a third endpoint `C` +with it: `C`'s accept finds `A` serving that half, so `C` accepts a session +that claims to be from `A`, reads what `A` writes to the replaying endpoint, +and sends its replies to that endpoint. Until ids are keys, the peer id of an +accepted session says only that the peer dialed `S`, to somebody. + +A session track therefore never appears where a third party can read it: - It is never announced. - It is never listed: an endpoint serves its session tracks only on request, diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index b635412e44e..ca7c8ca0cdb 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept: `dial_tcp(peer)` / `accept_tcp()` yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` / `accept_track()` yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed. Session tracks are never announced or listed. See [Sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index ca5e3b502d9..4e9076452b5 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -873,6 +873,10 @@ impl Requests { // Cache the served track so concurrent lookups coalesce onto it. If a live track already // holds the name (a publish raced the request), `insert` keeps it rather than shadowing it. let _ = state.cache_track(pending.weak()); + // Claimed after the lock is released, since claiming wakes the track's subscribers + // and an inline executor can re-enter the broadcast from that wake. A close landing + // in between rejects the request, as it would one still queued. + drop(state); // Attribute the served track to this broadcast's ingress scope (no-op untagged). Poll::Ready(Ok(pending.claim().with_stats(self.stats.clone()))) } diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index f9d9e860af3..6d267156785 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -18,7 +18,9 @@ //! the track `/A/S`. Endpoint `B` surfaces that request as an [`Incoming`] dial //! from `A` in its protocol's queue. To accept it, `B` subscribes to `/.socket/` //! for the same track name, and `A` answers only a session it dialed, so the exchange -//! checks that `A` really asked: a dial forged in `A`'s name is reset. Each side then +//! checks that `A` really asked: a dial forged in `A`'s name is reset. It does not check +//! that `A` asked `B`: an endpoint `A` dialed knows `/A/S`, and can replay it to +//! a third endpoint, which then accepts a session claiming to be from `A`. Each side then //! writes its own half and reads the peer's. //! //! See `doc/concept/socket.md` for the convention other languages follow. @@ -252,7 +254,10 @@ impl sealed::Open for TcpStream { /// A `track` session: each half is a whole track, and each message a group. /// /// [`Self::consumer`] starts subscribed reliably and in order: oldest first, with no -/// delay budget, so every group arrives. Update its subscription, such as with a finite +/// delay budget, so a late group is waited for rather than skipped. Each hop's cache +/// still reclaims a group nobody read within its idle window (30 s by default), or under +/// memory pressure, except the newest, so a reader that stalls longer can miss groups. +/// Update its subscription, such as with a finite /// [`Delay`] or [`Order::Descending`], to make it lossy or newest first, and read it in /// group order with [`track::Subscriber::ordered`]. Groups are written with /// [`track::Producer::append_group`], each with its own priority on the wire. @@ -311,7 +316,8 @@ struct Queue { struct State { /// The dials peers send, from the first accept on; `None` until then, so the endpoint /// answers a request for a session it does not hold with [`Error::NotFound`] at once. - requests: Option, + /// Behind its own lock, so it is polled outside this one. + requests: Option>, tcp: Queue, track: Queue, /// This side's half of every session handed out, aborted when the endpoint closes. @@ -343,35 +349,62 @@ impl Shared { /// under this endpoint's id names a session it never dialed, or one it gave up, such /// as an accept of a dial forged in this endpoint's name: it is reset with /// [`Error::NotFound`], which fails that accept. A request in a protocol nobody accepts - /// is reset with [`Error::Unsupported`]. Resets and wakes go out after the lock is - /// released. + /// is reset with [`Error::Unsupported`]. + /// + /// The handler is polled outside the lock, since taking a request wakes the dialer's + /// subscription, and resets and wakes go out after the lock is released too. fn poll_dial(&self, protocol: Protocol, waiter: &kio::Waiter) -> Poll> { + let requests = { + let mut state = self.state.lock(); + state.queue(protocol).listening = true; + match state.end { + None => Some( + state + .requests + .get_or_insert_with(|| kio::Lock::new(self.publication.requests())) + .clone(), + ), + Some(_) => None, + } + }; + let mut taken = Vec::new(); + let mut ended = None; + if let Some(requests) = &requests { + let mut requests = requests.lock(); + loop { + match requests.poll_requested_track(waiter) { + Poll::Pending => break, + Poll::Ready(Ok(request)) => taken.push(request), + Poll::Ready(Err(err)) => { + ended = Some(err); + break; + } + } + } + } + let mut resets = Vec::new(); let mut wakes = Vec::new(); let mut state = self.state.lock(); - state.queue(protocol).listening = true; - while state.end.is_none() { - let requests = state.requests.get_or_insert_with(|| self.publication.requests()); - match requests.poll_requested_track(waiter) { - Poll::Pending => break, - Poll::Ready(Ok(request)) => { - // `None` for a name naming no dial: outside the scheme, or this - // endpoint's own. - let target = match parse(request.name()) { - Some(name) if name.dialer != &*self.id => Some(Protocol::parse(name.protocol)), - _ => None, - }; - match target { - Some(Some(protocol)) if state.queue(protocol).listening => { - let queue = state.queue(protocol); - queue.dials.push_back(request); - wakes.push(queue.waiting.take()); - } - Some(_) => resets.push((request, Error::Unsupported)), - None => resets.push((request, Error::NotFound)), - } + if let Some(err) = ended { + state.end.get_or_insert(err); + } + for request in taken { + // `None` for a name naming no dial: outside the scheme, or this endpoint's own. + let target = match parse(request.name()) { + Some(name) if name.dialer != &*self.id => Some(Protocol::parse(name.protocol)), + _ => None, + }; + match target { + // Taken while the endpoint closed: reset like the dials it queued. + _ if state.end.is_some() => resets.push((request, Error::Cancel)), + Some(Some(protocol)) if state.queue(protocol).listening => { + let queue = state.queue(protocol); + queue.dials.push_back(request); + wakes.push(queue.waiting.take()); } - Poll::Ready(Err(err)) => state.end = Some(err), + Some(_) => resets.push((request, Error::Unsupported)), + None => resets.push((request, Error::NotFound)), } } let next = match &state.end { @@ -618,7 +651,8 @@ pub struct Incoming { impl Incoming { /// The id the dialer claims. An accept checks it: the dial connects only if that - /// endpoint really dialed this session. + /// endpoint really dialed this session, though not necessarily to this endpoint, since + /// an endpoint it dialed can replay the session to another. pub fn peer(&self) -> &str { &self.peer } @@ -1029,10 +1063,16 @@ impl Pump { return Err(Error::ProtocolViolation); } Some(_) => {} - None => { - self.subscriber = None; - break; - } + // The track ended, finished or reset: `poll_largest` reports both + // alike, so ask which, and pass on the peer's reset. + None => match subscriber.poll_finished(waiter) { + Poll::Ready(Ok(_)) => { + self.subscriber = None; + break; + } + Poll::Ready(Err(err)) => return Err(err), + Poll::Pending => break, + }, } } } @@ -1141,6 +1181,62 @@ mod tests { assert!(endpoint_path(&prefix, "a\\b").is_err()); } + /// Taking a dial wakes the dialer's subscription, and an inline executor can re-enter + /// the endpoint from that wake, such as to close it, so the endpoint polls its handler + /// outside its lock. + #[test] + fn taking_a_dial_wakes_outside_the_lock() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::task::Wake; + + struct Close { + shared: Arc, + woken: AtomicBool, + } + impl Wake for Close { + fn wake(self: Arc) { + self.shared.close(); + self.woken.store(true, Ordering::SeqCst); + } + } + + let (done, finished) = std::sync::mpsc::channel(); + let thread = std::thread::spawn(move || { + let (origin, _driver) = origin::Producer::new(origin::Config::new(crate::Hop::new(1).unwrap())); + let endpoint = Endpoint::new(&origin, "room", "bob").unwrap(); + let noop = kio::Waiter::noop(); + // Listening, so the dial queues for the handler. + assert!(endpoint.poll_accept_tcp(&noop).is_pending()); + + let broadcast = endpoint.shared.publication.broadcast().consume(); + let dial = broadcast + .track("tcp/alice/0123456789abcdef0123456789abcdef") + .unwrap() + .subscribe(None) + .into_inner(); + let close = Arc::new(Close { + shared: endpoint.shared.clone(), + woken: AtomicBool::new(false), + }); + let waiter = kio::Waiter::new(std::task::Waker::from(close.clone())); + assert!(dial.poll_ok(&waiter).is_pending()); + + // The wake closes the endpoint while this takes the dial, which it resets. + assert!(matches!( + endpoint.poll_accept_tcp(&noop), + Poll::Ready(Err(Error::Closed)) + )); + assert!(close.woken.load(Ordering::SeqCst)); + assert!(matches!(dial.poll_ok(&noop), Poll::Ready(Err(_)))); + done.send(()).unwrap(); + }); + let timeout = std::time::Duration::from_secs(10); + if let Err(std::sync::mpsc::RecvTimeoutError::Timeout) = finished.recv_timeout(timeout) { + panic!("deadlocked re-entering the endpoint"); + } + thread.join().unwrap(); + } + #[test] fn session_ids_are_128_bits() { let (a, b) = (session_id(), session_id()); diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index ada58d4d58f..91808f61807 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -366,6 +366,34 @@ async fn a_second_tcp_group_after_fin_resets_the_session() { assert!(accepted.producer.write_frame(None, &b"late"[..]).is_err()); } +/// The peer resetting its half after its group finished, and was read to its end, resets +/// the half that remains open, as a TCP RST after FIN does. Finishing it instead leaves +/// that half open, as a half-close does. +#[kio::test] +async fn a_peer_reset_after_fin_resets_the_session() { + for reset in [true, false] { + let mut fixture = Fixture::new(); + let bob = fixture.endpoint(LITE, "bob").await; + // Bound whole, so carol's endpoint and read stay open to the end. + let session = dial_by_hand(&mut fixture, &bob).await; + let ByHand { mut accepted, half, .. } = session; + + write_group(&half, 0); + assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), b"data"); + assert_eq!(read(&mut accepted.consumer).await.unwrap(), None); + accepted.producer.write_frame(None, &b"still open"[..]).unwrap(); + + if reset { + half.abort(Error::Cancel).unwrap(); + } else { + half.finish().unwrap(); + } + kio::clock::sleep(SETTLE).await; + let late = accepted.producer.write_frame(None, &b"late"[..]); + assert_eq!(late.is_err(), reset, "reset: {reset}"); + } +} + /// Messages flow both ways, in order, over every pairing of moq-lite-07 and /// moq-transport. #[kio::test] From 398333904abffdbc3e7aaf7b9b889eff2dece2f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 00:58:02 -0800 Subject: [PATCH 22/25] fix(net): poll a socket endpoint's handler with no lock held Round 3 of the local review: polling the handler under a mutex of its own still held a lock across the claim's wake, so an inline executor that accepted again from that wake deadlocked. A poller now takes the handler out of the endpoint's state and puts it back, and a poller that finds it taken waits to be woken when it returns. Polling a clone of `broadcast::Requests` instead would register a handler, and wake the broadcast's waiters, on every poll. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/socket.rs | 86 +++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 6d267156785..76dc596c27a 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -312,12 +312,25 @@ struct Queue { waiting: kio::WaiterList, } +/// An endpoint's handler for the requests to its broadcast: the dials peers send. +/// +/// A poller takes it out to poll it with no lock held, since taking a request wakes the +/// dialer's subscription, and an inline executor can re-enter the endpoint from that wake. +enum Handler { + /// Not made yet, until the first accept, so the endpoint answers a request for a + /// session it does not hold with [`Error::NotFound`] at once. + None, + /// Waiting for a poller. + Idle(super::broadcast::Requests), + /// Taken by a poller, which puts it back and wakes the pollers that found it taken, + /// so each polls it, and is woken by it, in turn. + Taken(kio::WaiterList), +} + /// What the endpoint's pollers share. struct State { - /// The dials peers send, from the first accept on; `None` until then, so the endpoint - /// answers a request for a session it does not hold with [`Error::NotFound`] at once. - /// Behind its own lock, so it is polled outside this one. - requests: Option>, + /// Where the dials peers send arrive. + handler: Handler, tcp: Queue, track: Queue, /// This side's half of every session handed out, aborted when the endpoint closes. @@ -351,26 +364,28 @@ impl Shared { /// [`Error::NotFound`], which fails that accept. A request in a protocol nobody accepts /// is reset with [`Error::Unsupported`]. /// - /// The handler is polled outside the lock, since taking a request wakes the dialer's - /// subscription, and resets and wakes go out after the lock is released too. + /// The handler is polled with no lock held (see [`Handler`]), and resets and wakes go + /// out after the lock is released. fn poll_dial(&self, protocol: Protocol, waiter: &kio::Waiter) -> Poll> { - let requests = { + let mut requests = { let mut state = self.state.lock(); state.queue(protocol).listening = true; - match state.end { - None => Some( - state - .requests - .get_or_insert_with(|| kio::Lock::new(self.publication.requests())) - .clone(), - ), - Some(_) => None, + if state.end.is_some() { + None + } else if let Handler::Taken(waiting) = &mut state.handler { + waiting.register(waiter); + None + } else { + match std::mem::replace(&mut state.handler, Handler::Taken(kio::WaiterList::new())) { + Handler::Idle(requests) => Some(requests), + Handler::None => Some(self.publication.requests()), + Handler::Taken(_) => unreachable!("checked above"), + } } }; let mut taken = Vec::new(); let mut ended = None; - if let Some(requests) = &requests { - let mut requests = requests.lock(); + if let Some(requests) = &mut requests { loop { match requests.poll_requested_track(waiter) { Poll::Pending => break, @@ -385,10 +400,25 @@ impl Shared { let mut resets = Vec::new(); let mut wakes = Vec::new(); + let mut retired = None; let mut state = self.state.lock(); if let Some(err) = ended { state.end.get_or_insert(err); } + // Put the handler back, unless the endpoint ended meanwhile: then it is dropped + // after the lock is released. + if let Some(handler) = requests.take() { + let next = match state.end { + None => Handler::Idle(handler), + Some(_) => { + retired = Some(handler); + Handler::None + } + }; + if let Handler::Taken(mut waiting) = std::mem::replace(&mut state.handler, next) { + wakes.push(waiting.take()); + } + } for request in taken { // `None` for a name naming no dial: outside the scheme, or this endpoint's own. let target = match parse(request.name()) { @@ -421,6 +451,7 @@ impl Shared { } }; drop(state); + drop(retired); for mut wake in wakes { wake.wake(); } @@ -447,9 +478,17 @@ impl Shared { let mut guard = self.state.lock(); let state = &mut *guard; state.end.get_or_insert(Error::Closed); - let requests = state.requests.take(); - let mut queued = Vec::new(); let mut wakes = Vec::new(); + // A poller holding the handler drops it once it sees the end. + let requests = match std::mem::replace(&mut state.handler, Handler::None) { + Handler::Idle(requests) => Some(requests), + Handler::Taken(mut waiting) => { + wakes.push(waiting.take()); + None + } + Handler::None => None, + }; + let mut queued = Vec::new(); for queue in [&mut state.tcp, &mut state.track] { queued.extend(queue.dials.drain(..)); wakes.push(queue.waiting.take()); @@ -514,7 +553,7 @@ impl Endpoint { tasks: origin.tasks(), publication, state: kio::Lock::new(State { - requests: None, + handler: Handler::None, tcp: Queue::default(), track: Queue::default(), streams: Vec::new(), @@ -1182,8 +1221,8 @@ mod tests { } /// Taking a dial wakes the dialer's subscription, and an inline executor can re-enter - /// the endpoint from that wake, such as to close it, so the endpoint polls its handler - /// outside its lock. + /// the endpoint from that wake, such as to accept again or to close it, so the endpoint + /// polls its handler with no lock held. #[test] fn taking_a_dial_wakes_outside_the_lock() { use std::sync::atomic::{AtomicBool, Ordering}; @@ -1195,6 +1234,9 @@ mod tests { } impl Wake for Close { fn wake(self: Arc) { + // The handler is out being polled, so this waits for it. + let accept = self.shared.poll_dial(Protocol::Track, &kio::Waiter::noop()); + assert!(accept.is_pending()); self.shared.close(); self.woken.store(true, Ordering::SeqCst); } From babc0aa08275d0c58737a3e9d591e2abfaf68678 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 06:34:34 -0800 Subject: [PATCH 23/25] feat(net): bind socket sessions to both ends, and wait on the dialer's endpoint Session tracks are now ///, so an endpoint a dialer reached cannot replay the session to a third endpoint. An accept follows the claimed dialer's endpoint for as long as the dial stays open, since a relay can deliver a dial before the dialer's announcement. Also: track session halves default to no max age, the endpoint closes its broadcast before dropping a retired handler, and the docs and quests say sockets need moq-lite-07 or moq-transport on every hop. Co-Authored-By: Claude Opus 5.5 --- doc/concept/socket.md | 69 ++++---- doc/lib/rs/moq-net.md | 2 +- quest/m1/socket/README.md | 52 ++++-- quest/m1/socket/js.md | 2 +- quest/m1/socket/old-hop-dial.md | 43 +++-- quest/m2/stream/README.md | 2 +- rs/moq-net/src/model/socket.rs | 155 ++++++++++-------- rs/moq-net/tests/integration/socket.rs | 119 ++++++++++++-- rs/moq-relay/tests/integration/socket_auth.rs | 2 +- 9 files changed, 294 insertions(+), 152 deletions(-) diff --git a/doc/concept/socket.md b/doc/concept/socket.md index 55c50f728e6..d096572d04f 100644 --- a/doc/concept/socket.md +++ b/doc/concept/socket.md @@ -17,8 +17,10 @@ ordinary broadcast. A socket is a convention over announcements, broadcasts, and track subscriptions, not a wire extension. It uses no instances and no track -listings, so it works over every moq-lite version and moq-transport, and relays -carry it unchanged. This page is the convention, enough to implement it in any +listings, and relays carry it unchanged. It needs moq-lite-07 or +moq-transport on every hop: an endpoint's broadcast is +[hidden](/concept/moq-lite#hidden-broadcasts), and an older moq-lite hop +never carries hidden broadcasts, so a dial across one never finds its peer. This page is the convention, enough to implement it in any language. [`moq_net::socket`](https://docs.rs/moq-net/latest/moq_net/socket/) implements it in Rust. @@ -42,45 +44,49 @@ session tracks, served on request, whatever their protocol. ## Sessions -A session between a dialer `A` and an acceptor `B` is named by its protocol `P` -and a session id `S` that `A` picks: random and unguessable, at least 128 bits, -such as 32 hex digits. Both halves of the session are tracks with the same -name, `//`, one in each endpoint's broadcast: +A session between a dialer `A` and an acceptor `B` is named by its protocol `P`, +both ends, and a session id `S` that `A` picks: random and unguessable, at +least 128 bits, such as 32 hex digits. Both halves of the session are tracks +with the same name, `///`, one in each +endpoint's broadcast: | Track | In broadcast | Written by | Read by | | --- | --- | --- | --- | -| `P/A/S` | `/.socket/A` | `A` | `B` | -| `P/A/S` | `/.socket/B` | `B` | `A` | +| `P/A/B/S` | `/.socket/A` | `A` | `B` | +| `P/A/B/S` | `/.socket/B` | `B` | `A` | The protocol is `tcp` or `track`; see [Protocols](#protocols). An endpoint resets a request whose protocol it doesn't speak, or isn't accepting, as unsupported. The dialer's id follows the protocol in both broadcasts, so an endpoint tells a -dial from an answer by the name alone: a request under another endpoint's id is -that endpoint dialing, and a request under its own id answers one of its own -dials. Since `S` is random per dial, every session is distinct, and one -endpoint can dial and accept any number of sessions at once, with the same peer -or others. +dial from an answer by the name alone: a request with another endpoint as the +dialer and itself as the acceptor is that endpoint dialing, and a request with +itself as the dialer answers one of its own dials. An endpoint resets any other +request, such as one naming another acceptor, as not found. Since `S` is random +per dial, every session is distinct, and one endpoint can dial and accept any +number of sessions at once, with the same peer or others. ### Dial -1. `A` picks `S` and holds its own half, `P/A/S` in its broadcast, unanswered. -2. `A` subscribes to `/.socket/B` for the track `P/A/S`, waiting for +1. `A` picks `S` and holds its own half, `P/A/B/S` in its broadcast, + unanswered. +2. `A` subscribes to `/.socket/B` for the track `P/A/B/S`, waiting for `B`'s endpoint to be announced if it is not yet. 3. `B` receives that subscription as a request for a track it does not have: an incoming dial in protocol `P`, claiming to be from `A`, with session `S`. ### Accept -1. `B` subscribes to `/.socket/A` for the track `P/A/S`, and holds - `A`'s subscription unanswered. -2. `A` answers that subscription only if it dialed `S` and is still waiting, - and resets any other. That makes the exchange a return-routability check: - a dial forged in `A`'s name, by anyone who can subscribe to `B`, reaches an - `A` that never dialed it, so `A` resets `B`'s subscription and `B` resets the - forger's. A dial replayed from a session `A` really dialed passes; see - [Hidden names](#hidden-names). +1. `B` subscribes to `/.socket/A` for the track `P/A/B/S`, and holds + `A`'s subscription unanswered. A relay can deliver the dial before `A`'s + announcement, so `B` waits for `A`'s endpoint to be announced for as long as + `A`'s subscription stays open, and gives up once it closes. +2. `A` answers that subscription only if it dialed `S` to `B` and is still + waiting, and resets any other. That makes the exchange a return-routability + check: a dial forged in `A`'s name, by anyone who can subscribe to `B`, + reaches an `A` that never dialed it, so `A` resets `B`'s subscription and `B` + resets the forger's. 3. Once `A` answers, `B` answers `A`'s subscription, and the session is open for both. `B` writes nothing before `A` answers. @@ -131,7 +137,9 @@ such as a proxied TCP connection, a file transfer, or a request and its reply. Each half is a whole track, and a message is one group. A slow message never holds up the next, and each group can carry its own priority. Messages are -ordered by group sequence. A reader that keeps up gets every message, but each +ordered by group sequence. A half has no max age unless its writer sets one, so +a newer message never makes an older one stale. A reader that keeps up gets +every message, but each hop's cache reclaims a message nobody read within its idle window (30 s by default), or under memory pressure, except the newest, so a reader that stalls longer can miss messages. @@ -164,13 +172,10 @@ The session id is the secret that keeps a session to its two ends: the token grants below let any endpoint subscribe to any other, so whoever learns a session's track name can read it, or answer it before its peer. -That includes the acceptor. The name says who dialed, but not whom, so the -return-routability check proves only that `A` dialed `S`, not that it dialed -`B`. An endpoint `A` dialed knows `P/A/S`, and can dial a third endpoint `C` -with it: `C`'s accept finds `A` serving that half, so `C` accepts a session -that claims to be from `A`, reads what `A` writes to the replaying endpoint, -and sends its replies to that endpoint. Until ids are keys, the peer id of an -accepted session says only that the peer dialed `S`, to somebody. +The acceptor learns the name too, but cannot replay it to a third endpoint +`C`: the name binds both ends. `C` resets `P/A/B/S` as a session for another +acceptor, and `A` never dialed `P/A/C/S`, so it resets `C`'s accept of that. +The peer id of an accepted session is the endpoint that dialed it to this one. A session track therefore never appears where a third party can read it: @@ -214,5 +219,5 @@ planned mesh layer makes ids public keys. Later protocols are other prefixes on the same endpoint, with no new announcement. A multiplexed `quic` mode, with many streams per session like -QUIC, would name its sessions `quic//` and bring back several +QUIC, would name its sessions `quic///` and bring back several groups or streams per session. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index c8a8cd24532..fe0dbd9c88c 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,7 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over or returns after an end, like JS `follow(path)`. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed. Session tracks are never announced or listed. See [Sockets](/concept/socket). +- **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track; the broadcast stays announced and keeps serving them until you close it or drop its last producer. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index 283126fa908..f405bb60dbd 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -21,25 +21,41 @@ lease and 1 s watchdog); signing and encryption, which the m2 [`doc/concept/socket.md`](/doc/concept/socket.md) is the convention JS and Python follow, and `rs/moq-net/src/model/socket.rs` implements it. +Decided 2026-11-04, second round (maintainer, settling the Waiting items of +[moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677)): + +- Name sessions `///`, binding a session to both + ends. An endpoint resets a dial naming another acceptor, and the dialer + serves only the name it dialed, so an endpoint `A` dialed can't replay the + session to a third endpoint posing as `A`. Done now, before the JS and Python + ports copy the scheme, rather than waiting for the m2 mesh keys. Not chosen: + keep `/` and document the replay. +- An accept waits for the claimed dialer's endpoint for as long as the dial + stays open, with no timer: a relay can deliver a dial before the dialer's + announcement, such as when two endpoints dial each other at start-up. A + forged dial claiming an endpoint nobody announces holds an accept only while + the forger holds its subscription. Not chosen: fail at once and let the + dialer redial; a bounded timer. + Decided 2026-11-04 (maintainer, /quest-complete), on [moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677), settling its Waiting items and replacing the `.tcp/` layout and the one-message-per-group -pipe below: +pipe below; the second round above adds the acceptor to the session name: -- Keep `/` session naming. A dial and an answer can't be confused, +- Name the dialer in the session. A dial and an answer can't be confused, the claimed dialer is named for the return-routability check, and `S` (random per dial) makes every session unique, so one endpoint can dial and accept any number of sessions. Not chosen: reader-named halves. - One announcement per endpoint, used only for routing: - `/.socket/`. The session track name carries the protocol, the - dialer and the session: `//`. A dial is SUBSCRIBE - `/.socket/`, track `tcp//`; the answer is the same - name in the dialer's broadcast. A requested track whose protocol the + `/.socket/`. The session track name carries the protocol, both + ends and the session: `///`. A dial is + SUBSCRIBE `/.socket/`, track `tcp///`; the + answer is the same name in the dialer's broadcast. A requested track whose protocol the endpoint doesn't speak, or isn't accepting, is reset as unsupported. Grants: publish `.socket/`, subscribe `.socket/**`. The module stays `socket`. - Two protocols ship now over the same handshake, each with its own dial and accept, and an incoming dial lands in its protocol's queue: - - `tcp//`: a session half is a single group, not a track of + - `tcp///`: a session half is a single group, not a track of messages. A group is one QUIC stream on every hop, so it's reliable, strictly ordered and flow controlled, like a TCP direction. Frames are writes, finishing the group is FIN, and resetting it is RST. There are no @@ -49,11 +65,13 @@ pipe below: `socket::TcpStream { producer: group::Producer, consumer: group::Consumer }` plus `peer()` and `session()`. Not chosen: `track::Ordered` with one message per group. - - `track//`: the same dial, accept, reject and return-routability + - `track///`: the same dial, accept, reject and return-routability check, but the app gets the whole session tracks, with groups as messages and its own priority, budget and order, and no single-group rule. The reader starts reliable and ascending (no delay budget) until the app - updates it. Rust: `Endpoint::dial_track` / `accept_track`, yielding + updates it. A half has no max age unless the app sets one, so a newer + message never makes an older one stale. Rust: `Endpoint::dial_track` / + `accept_track`, yielding `socket::TrackStream { producer: track::Producer, consumer: track::Subscriber }`. - Long-lived `tcp` sessions rely on unbounded groups (since [moq-dev/moq#7411](https://github.com/moq-dev/moq/pull/7411) a group has no @@ -90,8 +108,8 @@ Implementation choices in so a dial to it is reset with not found at once. A dial reserves its own half, which the peer's answer finds without a handler, so a dial-only endpoint never needs polling. A dial waits for the peer's endpoint to be - announced (drop it to give up); an accept fails at once when the claimed - dialer's endpoint is not announced. + announced (drop it to give up), and an accept waits for the claimed + dialer's endpoint until the dialer drops its subscription. - Accepted dials queue without limit while the app accepts, each holding the dialer's subscription open. - Session track names are the secret, so they never appear in a lite-07 track @@ -122,9 +140,13 @@ standing where the 2026-11-04 decisions above don't replace it: holds several, and a restart is a new session. - Grants are prefix only, rooted at the prefix. A relay test signs real `moq-auth` tokens. -- It works over every moq-lite version and moq-transport, since the handshake - uses no instances or listings; lite-07 instances only affect relay route - choice. Tests cross moq-transport hops. +- The handshake uses no instances or listings; lite-07 instances only affect + relay route choice. Tests cross moq-transport hops. Corrected 2026-11-04 in + [moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677): it needs + moq-lite-07 or moq-transport on every hop, since a moq-lite hop before 07 + has no hidden opt-in and never carries an endpoint's `.socket/` + broadcast; [older hops](/quest/m1/socket/old-hop-dial.md) make such a dial + fail instead of waiting. - Fixed channels become sessions or messages on one: boy's `command` and per-viewer latency, CARLA's `control`, ack, and e-stop. Shared state such as boy's `status` stays an ordinary broadcast. @@ -152,7 +174,7 @@ replacement arc), still standing: `doc/concept/socket.md`, not in the moq-lite draft: it is an application convention over moq-lite. -Later, not built now: a multiplexed `quic` protocol, `quic//` on +Later, not built now: a multiplexed `quic` protocol, `quic///` on the same endpoint with no new announcement, which brings back several groups or streams per session; and signed or encrypted sessions with ids as public keys, in the m2 [mesh](/quest/m2/mesh/README.md). diff --git a/quest/m1/socket/js.md b/quest/m1/socket/js.md index 8d3294863f5..e9a70cc0596 100644 --- a/quest/m1/socket/js.md +++ b/quest/m1/socket/js.md @@ -18,7 +18,7 @@ group `producer` and `consumer` (one group each way: frames are writes, finish is FIN, abort is RST, and a second group resets the session); and a `TrackStream` with a track `producer` and a reliably subscribed `consumer`. Both carry `peer` and `session`, and no message methods of their own. Session -tracks are `//` in both endpoints, served only on +tracks are `///` in both endpoints, served only on request so no listing names them; an unknown or unaccepted protocol is reset as unsupported. JS reads without fastest-reader pacing until [JS fastest-reader pacing](/quest/m2/js-fastest-reader.md), so a JS `tcp` diff --git a/quest/m1/socket/old-hop-dial.md b/quest/m1/socket/old-hop-dial.md index df58d3d1866..646a5884758 100644 --- a/quest/m1/socket/old-hop-dial.md +++ b/quest/m1/socket/old-hop-dial.md @@ -2,9 +2,12 @@ ## Goal -A socket dial through a hop older than lite-07 fails with `Unsupported` -instead of waiting forever. Today the older hop re-mints the answer's -instance, so the dialer never sees an answer it recognizes. +A socket dial or accept through a hop older than moq-lite-07 fails with +`Unsupported` instead of waiting forever. An endpoint's `.socket/` +broadcast is hidden, and a moq-lite hop before 07 has no hidden opt-in on its +announce requests, so it never carries the broadcast: a dial across one never +finds its peer's endpoint, and an acceptor reached by a dialer behind one waits +for the dialer's endpoint for as long as the dial stays open. ## Plan @@ -15,20 +18,26 @@ Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-a - m2 From [moq-dev/moq#7677](https://github.com/moq-dev/moq/pull/7677), which -lands Rust sockets and documents the hang (accept already fails -`Unsupported` over an older hop). - -- One option from #7677: the dialer also watches `path/.accept/` and - fails `Unsupported` on an answer whose instance `is_minted()`. Weigh the - cost of a second announce interest per pending dial against alternatives, - and record the choice. +landed Rust sockets on one hidden endpoint broadcast. Its first plan here, an +answer instance check, belonged to the earlier `.accept/` design, which +used instances; the endpoint design uses none. + +Facts from #7677's head, over the shared relay harness +(`rs/moq-net/tests/integration/socket.rs`): a moq-lite-06 client's request for +`room/.socket/bob` fails `Unroutable` where `room/plain/bob` resolves, and a +`tcp` session times out for lite-06 to lite-06, lite-07 to lite-06, and +lite-03 to lite-03. With a lite-06 dialer and a lite-07 acceptor, the acceptor +hears the dial, and its accept then waits until the dialer gives up. + +- Find where an endpoint can learn that a hop on the path can't carry hidden + broadcasts, such as the session's negotiated version, or a relay answering + a hidden request it can't route with a distinct error, and fail the dial or + accept `Unsupported` there. Weigh one that sees only the first hop against + one that covers relay-to-relay hops, and record the choice. - Python inherits it through moq-ffi. If [JS sockets](/quest/m1/socket/js.md) - has landed, its dial gets the same check. -- Test through a pre-lite-07 hop: the dial fails promptly with + has landed, its dial and accept get the same check. +- Test through a pre-lite-07 hop: the dial and the accept fail promptly with `Unsupported`. -Public API: none (an existing error). Wire: none. - -## Required - -- [Rust sockets](/quest/m1/socket/rust.md) - the dial this bounds +Public API: none (an existing error). Wire: none, unless a relay's answer +needs one. diff --git a/quest/m2/stream/README.md b/quest/m2/stream/README.md index e9b8ef11ba1..62c725bcd44 100644 --- a/quest/m2/stream/README.md +++ b/quest/m2/stream/README.md @@ -44,7 +44,7 @@ Amended 2026-11-01 with the maintainer (socket planning interview), `tcp` protocol became one group each way: connection setup is the [socket](/quest/m1/socket/README.md) epic's `Endpoint`, `dial_tcp`, and `accept_tcp`, which settle the layout (`/.socket/`, a -`tcp//` track each way). Each TCP connection is one `tcp` +`tcp///` track each way). Each TCP connection is one `tcp` session, whose halves are a single group each way for the connection's life: frames are writes, finishing is FIN, resetting is RST, and the writer awaits `group::Producer::ready` for backpressure. Each session gets its own priority, diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 76dc596c27a..31f948058da 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -3,7 +3,7 @@ //! An [`Endpoint`] announces one hidden broadcast, `/.socket/`, used only to //! route dials to it. From it, an endpoint dials other endpoints and accepts their dials. //! Each connection is a session: two tracks, one in each endpoint's broadcast, both named -//! `//`. Sessions are never announced or listed. +//! `///`. Sessions are never announced or listed. //! //! A session speaks one of two protocols: //! @@ -15,13 +15,13 @@ //! [`Endpoint::accept_track`], and read and write a [`TrackStream`]. //! //! A dialer `A` picks a random session id `S` and subscribes to `/.socket/` for -//! the track `/A/S`. Endpoint `B` surfaces that request as an [`Incoming`] dial +//! the track `/A/B/S`. Endpoint `B` surfaces that request as an [`Incoming`] dial //! from `A` in its protocol's queue. To accept it, `B` subscribes to `/.socket/` //! for the same track name, and `A` answers only a session it dialed, so the exchange -//! checks that `A` really asked: a dial forged in `A`'s name is reset. It does not check -//! that `A` asked `B`: an endpoint `A` dialed knows `/A/S`, and can replay it to -//! a third endpoint, which then accepts a session claiming to be from `A`. Each side then -//! writes its own half and reads the peer's. +//! checks that `A` really asked `B`: a dial forged in `A`'s name is reset. Naming both ends +//! binds a session to them, so an endpoint `A` dialed cannot replay the session to a third +//! endpoint `C`: `C` resets a name for another acceptor, and `A` never dialed +//! `/A/C/S`. Each side then writes its own half and reads the peer's. //! //! See `doc/concept/socket.md` for the convention other languages follow. @@ -60,10 +60,10 @@ fn session_id() -> Arc { format!("{:032x}", rand::rng().random::()).into() } -/// The track name of a session: `//`, the same in both -/// endpoints. -fn track_name(protocol: &str, dialer: &str, session: &str) -> String { - format!("{protocol}/{dialer}/{session}") +/// The track name of a session: `///`, the same in +/// both endpoints. +fn track_name(protocol: &str, dialer: &str, acceptor: &str, session: &str) -> String { + format!("{protocol}/{dialer}/{acceptor}/{session}") } /// A session's track name, split. @@ -71,23 +71,34 @@ fn track_name(protocol: &str, dialer: &str, session: &str) -> String { struct Name<'a> { protocol: &'a str, dialer: &'a str, + acceptor: &'a str, session: &'a str, } /// Split a session's track name, or `None` for a name outside the scheme. fn parse(name: &str) -> Option> { - let (protocol, rest) = name.split_once('/')?; - let (dialer, session) = rest.split_once('/')?; - for segment in [protocol, dialer, session] { + let mut segments = name.split('/'); + let [protocol, dialer, acceptor, session] = std::array::from_fn(|_| segments.next().unwrap_or_default()); + if segments.next().is_some() { + return None; + } + for segment in [protocol, dialer, acceptor, session] { check_id(segment).ok()?; } Some(Name { protocol, dialer, + acceptor, session, }) } +/// A `track` session half's default: every message stays until read, rather than only +/// the newest as for an authored track. +fn messages() -> track::Info { + track::Info::default().with_max_age(None) +} + /// What a session's reader asks for: every group, oldest first, however late it is. fn reliable() -> Subscription { Subscription::default() @@ -359,10 +370,11 @@ impl Shared { /// more, and fail with the endpoint's end once it stopped. /// /// A dial of this endpoint's reserves its own track, so a request reaching the handler - /// under this endpoint's id names a session it never dialed, or one it gave up, such - /// as an accept of a dial forged in this endpoint's name: it is reset with - /// [`Error::NotFound`], which fails that accept. A request in a protocol nobody accepts - /// is reset with [`Error::Unsupported`]. + /// with this endpoint as the dialer names a session it never dialed, or one it gave up, + /// such as an accept of a dial forged or replayed in this endpoint's name: it is reset + /// with [`Error::NotFound`], which fails that accept. So is a request naming another + /// acceptor, such as a session replayed from the endpoint it was dialed to. A request in + /// a protocol nobody accepts is reset with [`Error::Unsupported`]. /// /// The handler is polled with no lock held (see [`Handler`]), and resets and wakes go /// out after the lock is released. @@ -420,9 +432,12 @@ impl Shared { } } for request in taken { - // `None` for a name naming no dial: outside the scheme, or this endpoint's own. + // `None` for a name naming no dial to this endpoint: outside the scheme, this + // endpoint's own, or for another acceptor. let target = match parse(request.name()) { - Some(name) if name.dialer != &*self.id => Some(Protocol::parse(name.protocol)), + Some(name) if name.acceptor == &*self.id && name.dialer != &*self.id => { + Some(Protocol::parse(name.protocol)) + } _ => None, }; match target { @@ -451,7 +466,12 @@ impl Shared { } }; drop(state); - drop(retired); + if let Some(retired) = retired { + // Closed first, as `close` may not have yet: dropping the last handler rejects + // the dials still queued with `Dropped`, rather than the close's `Unroutable`. + self.publication.close(); + drop(retired); + } for mut wake in wakes { wake.wake(); } @@ -585,13 +605,14 @@ impl Endpoint { /// Dial the endpoint `peer` in a new `track` session; see [`Self::dial_tcp`]. /// - /// `info` describes this side's half. + /// `info` describes this side's half; `None` keeps every message until read, with no + /// max age. pub fn dial_track( &self, peer: &str, info: impl Into>, ) -> Result>, Error> { - self.dial(peer, info.into()) + self.dial(peer, Some(info.into().unwrap_or_else(messages))) } fn dial(&self, peer: &str, info: Option) -> Result>, Error> { @@ -604,12 +625,12 @@ impl Endpoint { return Err(err.clone()); } let session = session_id(); - let name = track_name(S::PROTOCOL.as_str(), &shared.id, &session); + let name = track_name(S::PROTOCOL.as_str(), &shared.id, peer, &session); // This side's half, waiting for the peer to ask for it, which only the peer can: // nobody else knows the session. let answer = shared.publication.reserve_unlisted_track(name.clone())?; // Followed rather than requested, so a peer not announced yet is waited for. - let resolving = Resolving::Follow(Box::new(shared.origin.follow(path, None)?)); + let resolving = Box::new(shared.origin.follow(path, None)?); Ok(kio::Pending::new(Dialing { shared: shared.clone(), peer: peer.into(), @@ -690,8 +711,7 @@ pub struct Incoming { impl Incoming { /// The id the dialer claims. An accept checks it: the dial connects only if that - /// endpoint really dialed this session, though not necessarily to this endpoint, since - /// an endpoint it dialed can replay the session to another. + /// endpoint really dialed this session to this endpoint. pub fn peer(&self) -> &str { &self.peer } @@ -712,22 +732,22 @@ impl Incoming { fn accept_with(mut self, info: Option) -> kio::Pending> { let shared = self.shared.clone(); let (peer, session) = (self.peer.clone(), self.session.clone()); - let request = self.request.take(); - // Requested, so a dial claiming an endpoint nobody announces fails at once: the - // dialer announced its endpoint before it dialed. - let resolving = endpoint_path(&shared.prefix, &peer) - .map(|path| Resolving::Request(Box::new(shared.origin.request(path, None).into_inner()))); - let step = match resolving { - Ok(resolving) => Step::Resolving(resolving), + let request = self.request.take().expect("taken only by accept, reject or drop"); + // Followed rather than requested: the dialer announced its endpoint before it + // dialed, but a relay can deliver the dial first. A dial claiming an endpoint + // nobody announces waits for as long as its dialer holds it open. + let step = match endpoint_path(&shared.prefix, &peer).and_then(|path| shared.origin.follow(path, None)) { + Ok(following) => Step::Resolving(Box::new(following)), Err(err) => Step::Failed(err), }; kio::Pending::new(Accepting { - shared, - name: track_name(S::PROTOCOL.as_str(), &peer, &session), + shared: shared.clone(), + name: track_name(S::PROTOCOL.as_str(), &peer, &shared.id, &session), peer, session, info, - request, + demand: request.demand(), + request: Some(request), step, stream: PhantomData, }) @@ -746,9 +766,10 @@ impl Incoming { impl Incoming { /// Accept the dial, resolving once the dialer answers, which proves it dialed. /// - /// `info` describes this side's half. Nothing is written before the dialer answers. + /// `info` describes this side's half, as for [`Endpoint::dial_track`]. Nothing is + /// written before the dialer answers. pub fn accept(self, info: impl Into>) -> kio::Pending> { - self.accept_with(info.into()) + self.accept_with(Some(info.into().unwrap_or_else(messages))) } } @@ -760,27 +781,10 @@ impl Drop for Incoming { } } -/// Resolving the peer's endpoint broadcast. -enum Resolving { - /// Failing once the endpoint is not announced. - Request(Box), - /// Waiting for the endpoint to be announced. - Follow(Box), -} - -impl Resolving { - fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - match self { - Self::Request(request) => request.poll_ok(waiter), - Self::Follow(follow) => follow.poll_available(waiter), - } - } -} - /// How far a dial or an accept got in reading the peer's half. enum Step { - /// Resolving the peer's endpoint broadcast. - Resolving(Resolving), + /// Waiting for the peer's endpoint broadcast to be announced. + Resolving(Box), /// Subscribed to the peer's half, waiting for the peer to serve it. Subscribing(track::Subscribing), /// The peer serves its half. @@ -796,8 +800,8 @@ impl Step { fn poll(&mut self, waiter: &kio::Waiter, track: &str) -> Poll> { loop { match self { - Self::Resolving(resolving) => { - let broadcast = ready!(resolving.poll(waiter))?; + Self::Resolving(following) => { + let broadcast = ready!(following.poll_available(waiter))?; let subscribing = broadcast.track(track)?.subscribe(reliable()).into_inner(); *self = Self::Subscribing(subscribing); } @@ -960,6 +964,9 @@ pub struct Accepting { info: Option, /// The dialer's request for this side's half, served once the dialer answers. request: Option, + /// Whether the dialer still subscribes to this side's half: the dial stays open while + /// it does. + demand: track::Demand, step: Step, stream: PhantomData S>, } @@ -967,8 +974,10 @@ pub struct Accepting { impl Accepting { /// Poll for the dialer to answer, which proves it dialed. /// - /// Fails with the dialer's reset when it never dialed this session, such as a dial - /// forged in its name, and resets the dialer's request with the same error. + /// Waits for the dialer's endpoint to be announced for as long as the dial stays open, + /// and fails with [`Error::Cancel`] once the dialer gives up. Fails with the dialer's + /// reset when it never dialed this session, such as a dial forged in its name, and + /// resets the dialer's request with the same error. pub fn poll_ok(&mut self, waiter: &kio::Waiter) -> Poll> { let result = self.poll_inner(waiter); if let Poll::Ready(Err(err)) = &result { @@ -982,6 +991,9 @@ impl Accepting { fn poll_inner(&mut self, waiter: &kio::Waiter) -> Poll> { self.shared.poll_end(waiter)?; + if self.demand.poll_state(waiter) != track::DemandState::Active { + return Poll::Ready(Err(Error::Cancel)); + } ready!(self.step.poll(waiter, &self.name))?; // Accepted outside the lock, since it wakes the dialer. @@ -1183,13 +1195,14 @@ mod tests { #[test] fn track_names_round_trip() { - let name = track_name("tcp", "alice", "0123abcd"); - assert_eq!(name, "tcp/alice/0123abcd"); + let name = track_name("tcp", "alice", "bob", "0123abcd"); + assert_eq!(name, "tcp/alice/bob/0123abcd"); assert_eq!( parse(&name), Some(Name { protocol: "tcp", dialer: "alice", + acceptor: "bob", session: "0123abcd" }) ); @@ -1200,12 +1213,14 @@ mod tests { for name in [ "command", "alice/s", - "tcp/alice/", - "tcp//s", - "/alice/s", - "tcp/alice/s/0", - "tcp/a\\b/s", - "tcp/alice/s\\", + "tcp/alice/s", + "tcp/alice/bob/", + "tcp//bob/s", + "tcp/alice//s", + "/alice/bob/s", + "tcp/alice/bob/s/0", + "tcp/a\\b/bob/s", + "tcp/alice/bob/s\\", ] { assert_eq!(parse(name), None, "{name}"); } @@ -1252,7 +1267,7 @@ mod tests { let broadcast = endpoint.shared.publication.broadcast().consume(); let dial = broadcast - .track("tcp/alice/0123456789abcdef0123456789abcdef") + .track("tcp/alice/bob/0123456789abcdef0123456789abcdef") .unwrap() .subscribe(None) .into_inner(); diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 91808f61807..cb2bcfb62f1 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -284,7 +284,7 @@ async fn dial_by_hand(fixture: &mut Fixture, bob: &Endpoint) -> ByHand { let endpoint = carol.create_broadcast("room/.socket/carol").unwrap(); endpoint.announce(Default::default()).unwrap(); let mut requests = endpoint.requests(); - let name = "tcp/carol/0123456789abcdef0123456789abcdef"; + let name = "tcp/carol/bob/0123456789abcdef0123456789abcdef"; let broadcast = kio::clock::timeout(TIMEOUT, carol.consume().request("room/.socket/bob", None)) .await .expect("request timeout") @@ -537,7 +537,7 @@ async fn a_protocol_nobody_accepts_is_reset() { .expect("request timeout") .expect("bob's endpoint"); let unknown = broadcast - .track("quic/mallory/0123456789abcdef0123456789abcdef") + .track("quic/mallory/bob/0123456789abcdef0123456789abcdef") .unwrap() .subscribe(None); let unknown = async { kio::clock::timeout(TIMEOUT, unknown).await }; @@ -569,9 +569,51 @@ async fn a_dial_waits_for_its_peer() { dialed.producer.finish().unwrap(); } -/// An accept of a dial claiming an endpoint nobody announces fails. +/// An accept waits for the dialer's endpoint while the dial stays open, since a relay can +/// deliver a dial before the dialer's announcement: carol dials bob before she announces +/// her endpoint, and bob's accept connects once she does. #[kio::test] -async fn a_dial_from_nobody_fails() { +async fn an_accept_waits_for_its_dialer() { + let mut fixture = Fixture::new(); + let bob = fixture.endpoint(LITE, "bob").await; + let carol = fixture.client(LITE).await; + let name = "tcp/carol/bob/0123456789abcdef0123456789abcdef"; + let broadcast = kio::clock::timeout(TIMEOUT, carol.consume().request("room/.socket/bob", None)) + .await + .expect("request timeout") + .expect("bob's endpoint"); + let dial = broadcast.track(name).unwrap().subscribe(None); + + let incoming = kio::clock::timeout(TIMEOUT, bob.accept_tcp()) + .await + .expect("accept timeout") + .unwrap(); + assert_eq!(incoming.peer(), "carol"); + let mut accept = incoming.accept(); + assert!( + kio::clock::timeout(SETTLE, &mut accept).await.is_err(), + "bob's accept ended before carol announced" + ); + + let endpoint = carol.create_broadcast("room/.socket/carol").unwrap(); + let mut requests = endpoint.requests(); + endpoint.announce(Default::default()).unwrap(); + let answer = async { + let request = kio::wait(|waiter| requests.poll_requested_track(waiter)).await.unwrap(); + assert_eq!(request.name(), name); + request.accept(None) + }; + let (accepted, _half, dialed) = kio::clock::timeout(TIMEOUT, async { futures::join!(accept, answer, dial) }) + .await + .expect("connect timeout"); + assert_eq!(accepted.expect("accept").peer(), "carol"); + dialed.expect("bob's half"); +} + +/// An accept of a dial claiming an endpoint nobody announces waits only while the dial +/// stays open: the forger dropping it fails the accept. +#[kio::test] +async fn an_accept_ends_with_its_dial() { let mut fixture = Fixture::new(); let bob = fixture.endpoint(LITE, "bob").await; let mallory = fixture.client(LITE).await; @@ -580,7 +622,7 @@ async fn a_dial_from_nobody_fails() { .expect("request timeout") .expect("bob's endpoint"); let forged = broadcast - .track("tcp/carol/0123456789abcdef0123456789abcdef") + .track("tcp/carol/bob/0123456789abcdef0123456789abcdef") .unwrap() .subscribe(None); let incoming = kio::clock::timeout(TIMEOUT, bob.accept_tcp()) @@ -588,16 +630,65 @@ async fn a_dial_from_nobody_fails() { .expect("accept timeout") .unwrap(); assert_eq!(incoming.peer(), "carol"); - let accepted = kio::clock::timeout(TIMEOUT, incoming.accept()) - .await - .expect("accept timeout"); - assert!(accepted.is_err(), "bob connected to nobody"); + let mut accept = incoming.accept(); assert!( - kio::clock::timeout(TIMEOUT, forged) - .await - .expect("forged timeout") - .is_err() + kio::clock::timeout(SETTLE, &mut accept).await.is_err(), + "bob's accept ended while the dial was open" ); + + drop(forged); + let accepted = kio::clock::timeout(TIMEOUT, accept).await.expect("accept timeout"); + assert!(accepted.is_err(), "bob connected to nobody"); +} + +/// A session is bound to both its ends: an endpoint alice dialed knows the session's id, +/// but replaying it to carol connects neither under bob's name nor under carol's. +#[kio::test] +async fn a_replayed_session_never_connects() { + let mut fixture = Fixture::new(); + let alice = fixture.endpoint(LITE, "alice").await; + let bob = fixture.endpoint(LITE, "bob").await; + let carol = fixture.endpoint(LITE, "carol").await; + let (mut dialed, mut accepted) = connect_tcp(&alice, &bob).await; + let session = accepted.session().to_string(); + + // Replayed from bob's side, which knows the session. + let mallory = fixture.client(LITE).await; + let broadcast = kio::clock::timeout(TIMEOUT, mallory.consume().request("room/.socket/carol", None)) + .await + .expect("request timeout") + .expect("carol's endpoint"); + + // Under the name alice dialed: carol is not its acceptor, so she resets it unheard. + let replayed = broadcast + .track(&format!("tcp/alice/bob/{session}")) + .unwrap() + .subscribe(None); + // Carol accepts first, so the replay reaches her listening. + let accept = async { kio::clock::timeout(SETTLE, carol.accept_tcp()).await }; + let (accepted_by_carol, replayed) = futures::join!(accept, async { kio::clock::timeout(TIMEOUT, replayed).await }); + assert!(replayed.expect("replay timeout").is_err(), "carol served a replayed session"); + assert!(accepted_by_carol.is_err(), "a session for bob surfaced at carol"); + + // Renamed for carol: alice never dialed it, so she resets carol's accept. + let renamed = broadcast + .track(&format!("tcp/alice/carol/{session}")) + .unwrap() + .subscribe(None); + let incoming = kio::clock::timeout(TIMEOUT, carol.accept_tcp()) + .await + .expect("accept timeout") + .unwrap(); + assert_eq!(incoming.peer(), "alice"); + let (accepted_by_carol, renamed) = kio::clock::timeout(TIMEOUT, async { futures::join!(incoming.accept(), renamed) }) + .await + .expect("connect timeout"); + assert!(accepted_by_carol.is_err(), "carol connected to a replayed session"); + assert!(renamed.is_err(), "the replayer read carol's half"); + + // The real session is untouched. + dialed.producer.write_frame(None, &b"still here"[..]).unwrap(); + assert_eq!(read(&mut accepted.consumer).await.unwrap().unwrap(), b"still here"); } /// A third party asking bob for a session in alice's name surfaces at bob as a dial from @@ -616,7 +707,7 @@ async fn a_forged_dial_never_connects() { .expect("request timeout") .expect("bob's endpoint"); let forged = broadcast - .track(&format!("{protocol}/alice/0123456789abcdef0123456789abcdef")) + .track(&format!("{protocol}/alice/bob/0123456789abcdef0123456789abcdef")) .unwrap() .subscribe(None); diff --git a/rs/moq-relay/tests/integration/socket_auth.rs b/rs/moq-relay/tests/integration/socket_auth.rs index 9624cecdcea..6b394d201c8 100644 --- a/rs/moq-relay/tests/integration/socket_auth.rs +++ b/rs/moq-relay/tests/integration/socket_auth.rs @@ -236,7 +236,7 @@ async fn a_forged_dial_never_connects() { .await .expect("bob's endpoint"); let forged = broadcast - .track("tcp/alice/0123456789abcdef0123456789abcdef") + .track("tcp/alice/bob/0123456789abcdef0123456789abcdef") .expect("track") .subscribe(None); let incoming = bob.accept_tcp().await.expect("accept"); From f861bf8ee0ba49c6c529f69dcffd49db530216d5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 06:37:19 -0800 Subject: [PATCH 24/25] style: cargo fmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/socket.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index cb2bcfb62f1..24ee7d76e00 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -667,7 +667,10 @@ async fn a_replayed_session_never_connects() { // Carol accepts first, so the replay reaches her listening. let accept = async { kio::clock::timeout(SETTLE, carol.accept_tcp()).await }; let (accepted_by_carol, replayed) = futures::join!(accept, async { kio::clock::timeout(TIMEOUT, replayed).await }); - assert!(replayed.expect("replay timeout").is_err(), "carol served a replayed session"); + assert!( + replayed.expect("replay timeout").is_err(), + "carol served a replayed session" + ); assert!(accepted_by_carol.is_err(), "a session for bob surfaced at carol"); // Renamed for carol: alice never dialed it, so she resets carol's accept. @@ -680,9 +683,10 @@ async fn a_replayed_session_never_connects() { .expect("accept timeout") .unwrap(); assert_eq!(incoming.peer(), "alice"); - let (accepted_by_carol, renamed) = kio::clock::timeout(TIMEOUT, async { futures::join!(incoming.accept(), renamed) }) - .await - .expect("connect timeout"); + let (accepted_by_carol, renamed) = + kio::clock::timeout(TIMEOUT, async { futures::join!(incoming.accept(), renamed) }) + .await + .expect("connect timeout"); assert!(accepted_by_carol.is_err(), "carol connected to a replayed session"); assert!(renamed.is_err(), "the replayer read carol's half"); From 0161d5bbcdb970f6f68cef60f0755ad4bcd86814 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 06:50:02 -0800 Subject: [PATCH 25/25] fix(net): drop queued socket dials whose dialer gave up Co-Authored-By: Claude Opus 5.5 --- quest/m1/socket/README.md | 6 ++-- rs/moq-net/src/model/socket.rs | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/quest/m1/socket/README.md b/quest/m1/socket/README.md index f405bb60dbd..64d5b994365 100644 --- a/quest/m1/socket/README.md +++ b/quest/m1/socket/README.md @@ -110,8 +110,10 @@ Implementation choices in endpoint never needs polling. A dial waits for the peer's endpoint to be announced (drop it to give up), and an accept waits for the claimed dialer's endpoint until the dialer drops its subscription. -- Accepted dials queue without limit while the app accepts, each holding the - dialer's subscription open. +- Accepted dials queue while the app accepts, each holding the dialer's + subscription open. A queued dial whose dialer gave up leaves its queue on the + endpoint's next poll, which each new dial triggers, so a queue holds only + open dials, even in a protocol the app stopped accepting. - Session track names are the secret, so they never appear in a lite-07 track listing or anywhere else a third party reads. moq-net lists only tracks a producer creates or a handler's `Lister` answers, never one served on diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index 31f948058da..801d1d0d42a 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -431,6 +431,19 @@ impl Shared { wakes.push(waiting.take()); } } + // Dials whose dialer gave up leave both queues on every poll, which each new dial + // triggers, so a queue holds no more than the dials still open, even in a protocol + // the application stopped accepting. + for protocol in [Protocol::Tcp, Protocol::Track] { + let queue = state.queue(protocol); + if queue.dials.iter().any(|request| !request.demand().is_wanted()) { + let (open, abandoned): (VecDeque<_>, VecDeque<_>) = std::mem::take(&mut queue.dials) + .into_iter() + .partition(|request| request.demand().is_wanted()); + queue.dials = open; + resets.extend(abandoned.into_iter().map(|request| (request, Error::Cancel))); + } + } for request in taken { // `None` for a name naming no dial to this endpoint: outside the scheme, this // endpoint's own, or for another acceptor. @@ -660,9 +673,8 @@ impl Endpoint { /// Poll for the next `tcp` dial from a peer; fails with [`Error::Closed`] once the /// endpoint closed. /// - /// The first call starts accepting `tcp` dials. They queue until taken, without limit, - /// each holding the dialer's request open, so keep accepting, and accept or reject - /// each one. + /// The first call starts accepting `tcp` dials. They queue until taken, or until their + /// dialer gives up, so keep accepting, and accept or reject each one. pub fn poll_accept_tcp(&self, waiter: &kio::Waiter) -> Poll, Error>> { self.poll_accept(waiter) } @@ -1294,6 +1306,40 @@ mod tests { thread.join().unwrap(); } + /// A dial queued in a protocol the application stopped accepting leaves the queue once + /// its dialer gives up, so dialing and giving up again never piles up requests. + #[test] + fn abandoned_dials_leave_the_queue() { + let (origin, _driver) = origin::Producer::new(origin::Config::new(crate::Hop::new(1).unwrap())); + let endpoint = Endpoint::new(&origin, "room", "bob").unwrap(); + let noop = kio::Waiter::noop(); + // Accepting both protocols, then only `tcp`. + assert!(endpoint.poll_accept_track(&noop).is_pending()); + assert!(endpoint.poll_accept_tcp(&noop).is_pending()); + + let broadcast = endpoint.shared.publication.broadcast().consume(); + let queued = || endpoint.shared.state.lock().track.dials.len(); + let dial = |session: u32| { + let dial = broadcast + .track(&format!("track/alice/bob/{session:032x}")) + .unwrap() + .subscribe(None) + .into_inner(); + assert!(dial.poll_ok(&noop).is_pending()); + dial + }; + for session in 0..3 { + let dial = dial(session); + assert!(endpoint.poll_accept_tcp(&noop).is_pending()); + assert_eq!(queued(), 1, "dial {session}"); + drop(dial); + } + let _open = dial(3); + let _also = dial(4); + assert!(endpoint.poll_accept_tcp(&noop).is_pending()); + assert_eq!(queued(), 2); + } + #[test] fn session_ids_are_128_bits() { let (a, b) = (session_id(), session_id());