diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 1659c6c2bba..1581b292378 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -96,6 +96,5 @@ revisions on the same host and worker count, and profiles with - [An idle export poll is cheap](/quest/m2/perf/export-idle-poll.md) - an export poll nothing woke skips the track fill and catalog work it repeats today - [Mesh session cost](/quest/m2/perf/mesh-session-cost.md) - moq-net's per-session announce, route sync, and scope work drops, swept over sessions and announces - [Cluster route memory](/quest/m2/perf/cluster-route-memory.md) - a full-mesh cluster's ~3.7 KB per route per broadcast shrinks, the in-memory hop is profiled, and `bench-profile lock` tolerates lost records -- [Lite request allocations](/quest/m2/perf/lite-request-allocs.md) - `Path::concat` allocates once and the lite request handler keeps one track-name copy - [Lock-free single-delivery writes](/quest/m2/perf/single-delivery-write.md) - a frame with one delivery writes without a lock, recovering #6922's remaining FETCH cost - [Small hot-path waste](/quest/m2/perf/hot-path-waste.md) - one track-name copy per lite subscription, no full-history `#drift` walk, and one header buffer per JS object diff --git a/quest/m2/perf/lite-request-allocs.md b/quest/m2/perf/lite-request-allocs.md deleted file mode 100644 index 387fb2da476..00000000000 --- a/quest/m2/perf/lite-request-allocs.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Fewer allocations on the lite request path - -## Goal - -`Path::concat`, and so `Path::join`, allocates once for a result over 22 -bytes instead of twice, and the lite request handler keeps one copy of the -track name instead of two. Allocation-counting tests show the drop. - -## Plan - -Found by [moq-dev/moq#6952](https://github.com/moq-dev/moq/pull/6952): - -- `Path::concat` builds a `String`, then copies it into the `Arc`. - Build it on the stack the way `from_tuple` does. -- The lite request serve copies the track name into the owned message and - again into its log context in `RequestServe::track`. - -Extend `rs/moq-net-tests/tests/announce_allocs.rs`. - -Public API: none. diff --git a/rs/moq-net-tests/tests/announce_allocs.rs b/rs/moq-net-tests/tests/announce_allocs.rs index 0f3be705960..77c0e0a80cb 100644 --- a/rs/moq-net-tests/tests/announce_allocs.rs +++ b/rs/moq-net-tests/tests/announce_allocs.rs @@ -1,7 +1,8 @@ //! Allocation counting for the announce hot paths: decoding the paths a peer -//! announces, and revisiting a cursor whose winning route is unchanged. Also the -//! messages around them: a decoded field allocates only once kept, and encoding -//! copies nothing. +//! announces, joining them, and revisiting a cursor whose winning route is +//! unchanged. Also the messages around them: a decoded field allocates only once +//! kept, encoding copies nothing, and a served request keeps one copy of its track +//! name. //! //! Every route change at a prefix revisits each cursor presenting it, and on a //! dense mesh most of those visits find the cursor's winner unchanged: a worse @@ -13,9 +14,20 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::cell::Cell; use moq_net::{ - Hop, Hops, Path, Pattern, Patterns, + Client, Hop, Hops, Path, Pattern, Patterns, Server, Version, fuzz::{decode_path, decode_subscribe, encode_path, encode_subscribe}, - origin, + kio, origin, + time::{Driver, Instant}, + track, + transport::memory, +}; + +#[path = "../../moq-net/tests/integration/support/mod.rs"] +mod support; + +use support::{ + harness::{now, spawn, subscribed}, + mock::create_mock_session_pair, }; thread_local! { @@ -25,6 +37,13 @@ thread_local! { /// `const` initialised, so reading it from inside the allocator cannot allocate /// and recurse. static ALLOCS: Cell = const { Cell::new(0) }; + + /// A byte size to watch, and the fresh allocations of exactly that size *this* + /// thread made. Size 0 watches nothing, since nothing allocates zero bytes. + static SIZED: Cell<(usize, usize)> = const { Cell::new((0, 0)) }; + + /// The size [`SIZED`] watches while a [`Watched`] driver is polled. + static WATCH: Cell = const { Cell::new(0) }; } struct Counting; @@ -36,6 +55,12 @@ unsafe impl GlobalAlloc for Counting { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { // `try_with`, because TLS is unavailable while a thread is being torn down. let _ = ALLOCS.try_with(|allocs| allocs.set(allocs.get() + 1)); + let _ = SIZED.try_with(|sized| { + let (size, count) = sized.get(); + if size == layout.size() { + sized.set((size, count + 1)); + } + }); unsafe { System.alloc(layout) } } @@ -143,6 +168,19 @@ fn a_decoded_path_allocates_only_its_buffer() { } } +/// Joining two paths copies them straight into the one buffer the result keeps, or +/// inline when short, rather than through a `String` first. +#[test] +fn a_joined_path_allocates_only_its_buffer() { + for path in [SHORT, LONG] { + let (head, tail) = path.split_once('/').unwrap(); + let (head, tail) = (Path::new(head).unwrap(), Path::new(tail).unwrap()); + assert_eq!(head.join(&tail).unwrap().as_str(), path); + let joined = allocs(|| head.join(&tail).unwrap()); + assert_eq!(joined, path_buffers(path), "allocations joining {path:?}"); + } +} + /// Encoding a path writes its fields straight from its text, whatever the wire. #[test] fn an_encoded_path_allocates_nothing() { @@ -178,3 +216,79 @@ fn a_decoded_subscribe_borrows_its_track_name() { } } } + +/// A session driver that counts the allocations sized [`WATCH`] only while it is +/// polled, so one end of a session is counted apart from the other on the same thread. +struct Watched(D); + +impl Driver for Watched { + fn poll(&mut self, now: Instant, waiter: &kio::Waiter) -> moq_net::Result> { + SIZED.with(|sized| sized.set((WATCH.with(Cell::get), sized.get().1))); + let next = self.0.poll(now, waiter); + SIZED.with(|sized| sized.set((0, sized.get().1))); + next + } +} + +/// A served request keeps one copy of its track name: the one its owned message +/// made, which moves into the log context as the request starts, rather than a +/// second one for the log context alone. +/// +/// Counts the publisher's allocations sized exactly like the name across a whole +/// subscription, so the name is an unusual length: a `String` copy of it is sized +/// that way, while the model's shared `Arc` copies carry a header. +#[kio::test] +async fn a_lite_request_keeps_one_copy_of_its_track_name() { + const TRACK: &str = "video/hd/a-track-name-of-an-odd-length"; + + // Moq-lite 05 and later ask for the track's info on a TRACK stream of its own + // before the SUBSCRIBE, so the publisher serves two requests to 03's one. + for (version, requests) in [("moq-lite-03", 1), ("moq-lite-07-wip", 2)] { + let version: Version = version.parse().unwrap(); + let (publisher, driver) = origin::Producer::new(origin::Config::new(Hop::new(1).unwrap())); + spawn(driver); + let broadcast = publisher.publish("room", Default::default()).unwrap(); + let track = broadcast.create_track(TRACK, track::Info::default()).unwrap(); + let (viewer, driver) = origin::Producer::new(origin::Config::new(Hop::new(2).unwrap())); + spawn(driver); + + let mut transport = memory::Config::default(); + transport.protocol = Some(version.alpn().to_string()); + let (client, server) = create_mock_session_pair(transport); + let connect = Client::new() + .with_versions(version.into()) + .with_subscriber(viewer.clone()); + let accept = Server::new() + .with_versions(version.into()) + .with_publisher(publisher.consume()); + // Both at once: each handshake waits on the other's driver. + let (client, server) = futures::join!( + async { + let (session, driver) = connect.connect(now(), client).await.unwrap(); + spawn(driver); + session + }, + async { + let (session, driver) = accept.accept(now(), server).await.unwrap(); + spawn(Watched(driver)); + session + } + ); + + let remote = viewer + .consume() + .follow("room", None) + .unwrap() + .available() + .await + .unwrap(); + WATCH.with(|watch| watch.set(TRACK.len())); + SIZED.with(|sized| sized.set((0, 0))); + let subscriber = remote.track(TRACK).unwrap().subscribe(None).await.unwrap(); + subscribed(&track).await; + WATCH.with(|watch| watch.set(0)); + let copies = SIZED.with(|sized| sized.replace((0, 0)).1); + assert_eq!(copies, requests, "{version}: the publisher's copies of the track name"); + drop((subscriber, client, server)); + } +} diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index fb17d2439dd..2d241bcd852 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -6,6 +6,7 @@ use std::sync::atomic::AtomicU64; use crate::{SessionError, announce, frame, group, origin, track}; use rangemap::{RangeMap, RangeSet}; use std::{ + borrow::Cow, collections::HashMap, ops::{Bound, ControlFlow, Range}, sync::{ @@ -1554,11 +1555,21 @@ trait Request: Sized { /// The broadcast and track the request names. fn target(msg: &Self::Message) -> (&crate::Path, &str); + /// Move the track name out of the message as the request starts, so the log context + /// keeps the one copy. + fn take_track(msg: &mut Self::Message) -> Cow<'static, str>; + /// The publisher instance the request names, if any. fn epoch(msg: &Self::Message) -> Option<&crate::Epoch>; - /// Start answering, once the broadcast resolves. - fn start(shared: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result; + /// Start answering, once the broadcast resolves. `track` is the name [`Request::take_track`] + /// moved out of `msg`, which no longer holds it. + fn start( + shared: &Shared, + msg: Self::Message, + track: &str, + broadcast: crate::broadcast::Consumer, + ) -> Result; /// Apply an update from the requester. fn update(&mut self, update: Self::Update); @@ -1608,7 +1619,9 @@ struct RequestServe { // Log context, filled in after the decode. Relative to the origin's root, so a path // too long under it is refused by the lookup, on this stream alone. broadcast: crate::Path, - track: String, + // Moved out of the message as the request starts; until then, read from the message. + // See `RequestServe::track`. + track: Cow<'static, str>, // A reply that always has another group ready still lets the session's other tasks run. budget: kio::coop::Budget, // The requester's update failed to decode: the peer's fault, unlike a failed reply. @@ -1657,6 +1670,14 @@ impl RequestServe { } } + /// The track name, for logs: one copy, held by the message until the request starts. + fn track(&self) -> &str { + match &self.state { + RequestState::Resolve { msg, .. } => R::target(msg).1, + _ => &self.track, + } + } + /// The reply no longer owes the peer anything a draining close should wait for. fn settle(&mut self) { if std::mem::take(&mut self.owing) { @@ -1667,7 +1688,7 @@ impl RequestServe { fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { match ready!(self.poll_serve(waiter)) { Ok(()) => { - tracing::debug!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track, "{} complete", R::KIND); + tracing::debug!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track(), "{} complete", R::KIND); Poll::Ready(Ok(())) } // The peer's own malformed bytes are a control violation, even after request setup. @@ -1675,13 +1696,13 @@ impl RequestServe { Err(err) => { match &err { err if is_cancel(err) => { - tracing::debug!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track, "{} cancelled", R::KIND) + tracing::debug!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track(), "{} cancelled", R::KIND) } Error::Unauthorized | Error::Stream(crate::StreamError::Unauthorized) => { - tracing::info!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track, "{} unauthorized", R::KIND) + tracing::info!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track(), "{} unauthorized", R::KIND) } err => { - tracing::warn!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track, %err, "{} error", R::KIND) + tracing::warn!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track(), %err, "{} error", R::KIND) } } self.stream.take().expect("stream present").writer.abort(&err); @@ -1712,8 +1733,13 @@ impl RequestServe { return Poll::Ready(Err(Error::Cancel)); } // Drop the reply mid-flight: anything half-sent is cancelled rather than - // completed for nobody. - self.state = RequestState::Finish { finished: false }; + // completed for nobody. A request that never started moves its name into the log + // context, as `start` would. + if let RequestState::Resolve { mut msg, .. } = + std::mem::replace(&mut self.state, RequestState::Finish { finished: false }) + { + self.track = R::take_track(&mut msg); + } } let stream = self.stream.as_mut().expect("stream present"); @@ -1723,11 +1749,10 @@ impl RequestServe { let msg = ready!(stream.reader.poll_decode_with(waiter, |r| R::decode(r, version)))?; let (broadcast, track) = R::target(&msg); self.broadcast = broadcast.clone(); - self.track = track.to_string(); if R::SUBSCRIPTION { self._subscription = Some(self.shared.subscriptions.hold()); } - tracing::debug!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %self.track, request = ?msg, "{} started", R::KIND); + tracing::debug!(broadcast = %crate::path::Rooted(self.shared.origin.root(), &self.broadcast), track = %track, request = ?msg, "{} started", R::KIND); // Checked before anything is resolved, so a broadcast the session does not // allow is never served, not just cut off later. let mut gate = self.shared.gate(R::target(&msg).0); @@ -1750,12 +1775,13 @@ impl RequestServe { } let broadcast = ready!(requesting.as_ref().expect("requesting").poll_ok(waiter))?; // Any placeholder but `Decode`, so a failed start is answered on the stream. - let RequestState::Resolve { msg, .. } = + let RequestState::Resolve { mut msg, .. } = std::mem::replace(&mut self.state, RequestState::Finish { finished: false }) else { unreachable!() }; - let mut request = R::start(&self.shared, msg, broadcast)?; + self.track = R::take_track(&mut msg); + let mut request = R::start(&self.shared, msg, &self.track, broadcast)?; if let Some(update) = self.update.take() { request.update(update); } @@ -1887,12 +1913,16 @@ impl Request for TrackInfoServe { (&msg.broadcast, &msg.track) } + fn take_track(msg: &mut Self::Message) -> Cow<'static, str> { + std::mem::take(&mut msg.track) + } + fn epoch(msg: &Self::Message) -> Option<&crate::Epoch> { msg.epoch.as_ref() } - fn start(_: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { - let querying = broadcast.track(&msg.track)?.info().into_inner(); + fn start(_: &Shared, _: Self::Message, track: &str, broadcast: crate::broadcast::Consumer) -> Result { + let querying = broadcast.track(track)?.info().into_inner(); Ok(Self { querying }) } @@ -1947,11 +1977,15 @@ impl Request for TracksServe { (&msg.broadcast, "") } + fn take_track(_: &mut Self::Message) -> Cow<'static, str> { + Cow::Borrowed("") + } + fn epoch(msg: &Self::Message) -> Option<&crate::Epoch> { msg.epoch.as_ref() } - fn start(shared: &Shared, _: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { + fn start(shared: &Shared, _: Self::Message, _: &str, broadcast: crate::broadcast::Consumer) -> Result { Ok(Self { tracks: broadcast.tracks_as(shared.lister)?, }) @@ -2044,12 +2078,21 @@ impl Request for SubscribeServe { (&msg.broadcast, &msg.track) } + fn take_track(msg: &mut Self::Message) -> Cow<'static, str> { + std::mem::take(&mut msg.track) + } + fn epoch(msg: &Self::Message) -> Option<&crate::Epoch> { msg.epoch.as_ref() } - fn start(shared: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { - let track = broadcast.track(&msg.track)?; + fn start( + shared: &Shared, + msg: Self::Message, + track: &str, + broadcast: crate::broadcast::Consumer, + ) -> Result { + let track = broadcast.track(track)?; let latest = track.latest_sequence(); if Bounds::has_time(&msg.ranges) { let querying = track.info().into_inner(); @@ -2292,13 +2335,22 @@ impl Request for FetchServe { (&msg.broadcast, &msg.track) } + fn take_track(msg: &mut Self::Message) -> Cow<'static, str> { + std::mem::take(&mut msg.track) + } + fn epoch(msg: &Self::Message) -> Option<&crate::Epoch> { msg.epoch.as_ref() } - fn start(_: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { + fn start( + _: &Shared, + msg: Self::Message, + track: &str, + broadcast: crate::broadcast::Consumer, + ) -> Result { let fetching = broadcast - .track(&msg.track)? + .track(track)? .fetch_range( msg.group, track::FetchOptions { @@ -8954,13 +9006,14 @@ mod tests { lite::Fetch { epoch: None, broadcast: crate::Path::new("room").unwrap(), - track: "video".into(), + track: Default::default(), priority: 0, max_delay: None, group: 0, start_frame: 2, end_frame: Some(3), }, + "video", broadcast.consume(), ) .unwrap(); diff --git a/rs/moq-net/src/path/mod.rs b/rs/moq-net/src/path/mod.rs index 2da24fa99d5..a26e1dc2f1b 100644 --- a/rs/moq-net/src/path/mod.rs +++ b/rs/moq-net/src/path/mod.rs @@ -303,8 +303,8 @@ impl Path { /// The path as a string, with no leading or trailing slash. pub fn as_str(&self) -> &str { match &self.0 { - // SAFETY: `Repr::copy` and `concat` fill `buf[..len]` only from whole `&str`s and an - // ASCII `/`, so those bytes are valid UTF-8. + // SAFETY: `Repr::copy_bounded` and `concat` fill `buf[..len]` only from whole `&str`s + // and an ASCII `/`, so those bytes are valid UTF-8. Repr::Inline { len, buf } => unsafe { std::str::from_utf8_unchecked(&buf[..*len as usize]) }, Repr::Static(s) => s, Repr::Shared { buf, start } => &buf[*start as usize..], @@ -371,13 +371,14 @@ impl Path { buf[a.len() + 1..len].copy_from_slice(b.as_bytes()); Ok(Path(Repr::Inline { len: len as u8, buf })) } else { - // Sized up front: `format!` guesses low and reallocates. - let mut buf = String::with_capacity(len); + // Joined on the stack, which `arrayvec` leaves uninitialized, so the shared buffer + // is the only allocation: a `String` would be a second, copied again into it. + let mut buf = arrayvec::ArrayString::<{ Self::MAX_BYTES }>::new(); buf.push_str(a); buf.push('/'); buf.push_str(b); Ok(Path(Repr::Shared { - buf: buf.into(), + buf: buf.as_str().into(), start: 0, })) }