From 3f3bc73d0457c83ec7eb637392ec724a4eecff0f Mon Sep 17 00:00:00 2001 From: Pouriya Jahanbakhsh Date: Sun, 20 Sep 2026 15:09:28 +0330 Subject: [PATCH] perf(qmux): batch writer frames, flushing the transport once The session writer flushed after every frame, so each small frame cost one syscall and one TCP segment. It now feeds the frames already queued and flushes once. A fed FIN's SendStream is signalled Finished only after the flush succeeds, so closed() still never reports success for a FIN the transport failed to write. Co-Authored-By: Claude Opus 5.5 --- rs/qmux/src/sched.rs | 7 ++ rs/qmux/src/session.rs | 240 +++++++++++++++++++++++++++++++++++---- rs/qmux/src/transport.rs | 45 +++++++- 3 files changed, 269 insertions(+), 23 deletions(-) diff --git a/rs/qmux/src/sched.rs b/rs/qmux/src/sched.rs index b121aedf..d3662b28 100644 --- a/rs/qmux/src/sched.rs +++ b/rs/qmux/src/sched.rs @@ -178,6 +178,13 @@ impl PriorityQueue { } } + /// Pop the next frame if one is queued right now, without waiting. Used by the + /// writer to drain a batch before flushing the transport once. + pub fn try_pop(&self) -> Option { + let mut inner = self.inner.lock().unwrap(); + self.pop_locked(&mut inner) + } + fn pop_locked(&self, inner: &mut Inner) -> Option { // Highest band first. let (&band, queue) = inner.bands.iter_mut().next_back()?; diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 5b7d708f..bde9a9b6 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -290,6 +290,15 @@ struct SessionState { rtt: Arc, } +/// Upper bounds on one writer batch (see [`WriterState::transmit_batch`]). The +/// byte cap stays under tungstenite's 128 KiB write buffer so a batch leaves in +/// a single write; the frame cap bounds a batch of tiny control frames. +const BATCH_MAX_BYTES: usize = 64 * 1024; +const BATCH_MAX_FRAMES: usize = 256; +/// A batch this small is likely a frame header whose payload is still being +/// queued, so the writer yields once before flushing it. +const BATCH_YIELD_BELOW_BYTES: usize = 128; + /// Pick the next outbound frame in strict priority order: control (lossless, /// e.g. RESET/STOP/CLOSE/window updates) first, then datagrams (low-latency but /// droppable), then bulk stream data scheduled by [`PriorityQueue`]. Returns @@ -375,9 +384,9 @@ struct WriterState { streams: Arc>, record_limit: Arc, - // Set while a `send` is in flight so the timer can tell a wedged-on- - // backpressure connection (peer alive, its recv window full) apart from a - // genuinely dead one, and not idle-close the former. See `transmit`. + // Set while a `feed` or `flush` is in flight so the timer can tell a + // wedged-on-backpressure connection (peer alive, its recv window full) apart + // from a genuinely dead one, and not idle-close the former. See `transmit`. writer_backpressured: Arc, closed: watch::Sender>, @@ -390,6 +399,9 @@ struct WriterState { // Stamped when a QX_PING request reaches the wire. The timer allocates the // sequence, but only the writer knows when it actually left. rtt: Arc, + + // Signals for FINs fed since the last flush, sent `Finished` once it lands. + finished: Vec>, } /// Outcome of a teardown-aware write (see [`WriterState::transmit_or_teardown`]). @@ -450,11 +462,16 @@ impl WriterState { // mid-frame), so the transport is at a frame boundary: best-effort // flush of any queued control frames (e.g. a ConnectionClose) // before we stop. + let mut flushable = true; while let Ok(frame) = self.control.try_recv() { if self.transmit(frame).await.is_err() { + flushable = false; break; } } + if flushable { + let _ = self.flush().await; + } break; } } @@ -482,7 +499,7 @@ impl WriterState { ) -> Transmitted { tokio::select! { biased; - result = self.transmit(frame) => match result { + result = self.transmit_batch(frame) => match result { Ok(()) => Transmitted::Ok, Err(err) => Transmitted::Failed(err), }, @@ -494,10 +511,76 @@ impl WriterState { } } + /// Feed `first` plus every frame that is already queued behind it, then flush + /// the transport once. A fan-out server writes several tiny frames per media + /// frame (e.g. moq-lite's timestamp and size varints, then the payload); one + /// write per frame costs a syscall, a TCP segment and an ACK each. + /// + /// Nothing waits for more data, so batching adds no latency, with one + /// exception: if a batch of stream data is still only a few bytes and the + /// queues are empty, yield once so a producer mid-way through a frame can queue + /// the rest. Control frames never wait: a CONNECTION_CLOSE must reach the wire + /// before teardown wins the race in `transmit_or_teardown`. The batch is + /// bounded so one busy session can't hog its worker thread. + async fn transmit_batch(&mut self, first: Frame) -> Result<(), Error> { + let may_yield = matches!(&first, Frame::Stream(stream) if !stream.fin); + let mut bytes = self.transmit(first).await?; + let mut frames = 1; + let mut yielded = !may_yield; + while bytes < BATCH_MAX_BYTES && frames < BATCH_MAX_FRAMES { + let frame = match self.try_next_outbound() { + Some(frame) => frame, + None if !yielded && bytes < BATCH_YIELD_BELOW_BYTES => { + yielded = true; + tokio::task::yield_now().await; + continue; + } + None => break, + }; + bytes += self.transmit(frame).await?; + frames += 1; + } + self.flush().await + } + + /// Non-blocking [`next_outbound`], in the same priority order. + fn try_next_outbound(&mut self) -> Option { + if let Ok(frame) = self.control.try_recv() { + return Some(frame); + } + if let Ok(payload) = self.datagrams.try_recv() { + return Some(Frame::Datagram(payload.into())); + } + self.outbound.try_pop() + } + + /// Flush everything fed so far, then report the fed FINs as finished and + /// publish send progress. + async fn flush(&mut self) -> Result<(), Error> { + // Taken up front so a failed or abandoned flush drops them, which + // `closed()` reports as a connection error. + let finished = std::mem::take(&mut self.finished); + // A flush stuck on a full peer window proves the peer is alive; see the + // matching flag in `transmit`. + self.writer_backpressured.store(true, Ordering::Release); + let result = self.writer.flush().await; + self.writer_backpressured.store(false, Ordering::Release); + result?; + for finished in finished { + finished.send(SendSignal::Finished).ok(); + } + // Publish send progress for the timer's keep-alive and idle scheduling. + let now = tokio::time::Instant::now(); + self.last_send_at + .store(millis_since(self.base, now), Ordering::Release); + Ok(()) + } + /// Retire the stream a terminal frame closes, encode the frame (validating its - /// size for QMux01), and write it. The `streams` lock is only held for the - /// synchronous retirement, never across the `send` await. - async fn transmit(&mut self, mut frame: Frame) -> Result<(), Error> { + /// size for QMux01), and feed it to the transport. The caller must + /// [`flush`](Self::flush) afterwards. Returns the encoded size. The `streams` + /// lock is only held for the synchronous retirement, never across the await. + async fn transmit(&mut self, mut frame: Frame) -> Result { let transmitted_stream = match &frame { Frame::Stream(stream) if !stream.fin => Some((stream.id, stream.data.len() as u64)), _ => None, @@ -509,8 +592,8 @@ impl WriterState { _ => None, }; - // Held until the FIN is on the transport: a failed or abandoned write drops - // it instead, which `closed()` reports as a connection error. + // Held until the FIN is flushed: a failed or abandoned write drops it + // instead, which `closed()` reports as a connection error. let mut finished = None; match &mut frame { Frame::ResetStream(reset) => { @@ -546,30 +629,30 @@ impl WriterState { } } // Flag the in-flight write so the timer won't idle-close a connection - // that's merely backpressured: a `send` stuck here proves the peer is + // that's merely backpressured: a write stuck here proves the peer is // still there (its receive window is just full). Cleared as soon as the // write lands. Only the session idle timeout consults this — a WebSocket - // transport's own keep-alive deadline is independent. + // transport's own keep-alive deadline is independent. A `feed` only + // blocks once the transport's buffer is full. + let len = bytes.len(); self.writer_backpressured.store(true, Ordering::Release); - let result = self.writer.send(bytes).await; + let result = self.writer.feed(bytes).await; self.writer_backpressured.store(false, Ordering::Release); result?; - if let Some(finished) = finished { - finished.send(SendSignal::Finished).ok(); - } + // Signalled by `flush`, once the FIN is actually on the transport. + self.finished.extend(finished); + // Counted once fed: the batch is flushed before any later frame, so a + // RESET_STREAM's final size still matches what the peer receives. if let Some((id, len)) = transmitted_stream { if let Some(send) = self.streams.lock().unwrap().send.get_mut(&id) { send.sent_offset += len; } } - let now = tokio::time::Instant::now(); + // Stamped when fed; the flush follows within the same batch. if let Some(sequence) = transmitted_ping { - self.rtt.sent(sequence, now); + self.rtt.sent(sequence, tokio::time::Instant::now()); } - // Publish send progress for the timer's keep-alive and idle scheduling. - self.last_send_at - .store(millis_since(self.base, now), Ordering::Release); - Ok(()) + Ok(len) } } @@ -653,6 +736,40 @@ mod writer_final_size_tests { } } + /// Records each fed message and how many had been fed at every flush. With + /// `fail_flush`, every flush errors after the feeds succeeded. + #[derive(Default)] + struct BatchWriter { + fed: Arc>>, + flushes: Arc>>, + fail_flush: bool, + } + + impl Writer for BatchWriter { + async fn send(&mut self, data: Bytes) -> Result<(), Error> { + self.feed(data).await?; + self.flush().await + } + + async fn feed(&mut self, data: Bytes) -> Result<(), Error> { + self.fed.lock().unwrap().push(data); + Ok(()) + } + + async fn flush(&mut self) -> Result<(), Error> { + if self.fail_flush { + return Err(Error::Closed); + } + let fed = self.fed.lock().unwrap().len(); + self.flushes.lock().unwrap().push(fed); + Ok(()) + } + + async fn close(&mut self) -> Result<(), Error> { + Ok(()) + } + } + fn writer_state(writer: W, streams: Arc>) -> WriterState { let (_control_tx, control) = mpsc::unbounded_channel(); let (_datagram_tx, datagrams) = mpsc::channel(1); @@ -669,6 +786,7 @@ mod writer_final_size_tests { base: tokio::time::Instant::now(), last_send_at: Arc::new(AtomicU64::new(0)), rtt: Arc::new(Rtt::default()), + finished: Vec::new(), } } @@ -689,7 +807,7 @@ mod writer_final_size_tests { let mut writer = writer_state(writer, streams); let result = writer - .transmit( + .transmit_batch( Stream { id, offset: 0, @@ -721,6 +839,83 @@ mod writer_final_size_tests { ); } + #[tokio::test] + async fn failed_fin_flush_does_not_signal_finished() { + let writer = BatchWriter { + fail_flush: true, + ..Default::default() + }; + let fed = writer.fed.clone(); + let (result, signal) = transmit_fin(writer).await; + assert!(result.is_err()); + assert_eq!(fed.lock().unwrap().len(), 1); + assert!( + signal.is_none(), + "closed() must not succeed for an unflushed FIN" + ); + } + + #[tokio::test] + async fn queued_frames_are_fed_then_flushed_once() { + let streams = Arc::new(Mutex::new(Streams::default())); + let id = StreamId::new(0, StreamDir::Uni, false); + let (signal, _signal_rx) = mpsc::unbounded_channel(); + streams.lock().unwrap().send.insert( + id, + SendState { + inbound_signal: signal, + sent_offset: 0, + stream_credit: None, + }, + ); + + let outbound = PriorityQueue::new(16); + // A media frame as moq-lite writes it: timestamp, size, payload. + let chunks: [&'static [u8]; 3] = [b"\x01", b"\x05", b"hello"]; + let mut offset = 0; + for chunk in chunks { + let frame = Stream { + id, + offset, + data: Bytes::from_static(chunk), + fin: false, + }; + offset += chunk.len() as u64; + outbound + .reserve() + .await + .unwrap() + .send(0, id, frame.into()) + .unwrap(); + } + + let batch = BatchWriter::default(); + let (fed, flushes) = (batch.fed.clone(), batch.flushes.clone()); + let mut writer = writer_state(batch, streams.clone()); + writer.outbound = outbound; + + let first = writer.try_next_outbound().unwrap(); + writer.transmit_batch(first).await.unwrap(); + + assert_eq!(*flushes.lock().unwrap(), vec![3]); + let payloads: Vec = fed + .lock() + .unwrap() + .iter() + .map(|wire| { + match Frame::decode(wire.clone(), Version::QMux01) + .unwrap() + .unwrap() + { + Frame::Stream(stream) => stream.data, + other => panic!("expected STREAM, got {other:?}"), + } + }) + .collect(); + assert_eq!(payloads, chunks.map(Bytes::from_static)); + assert_eq!(streams.lock().unwrap().send[&id].sent_offset, 7); + } + #[tokio::test] async fn reset_returns_credit_reserved_for_dropped_frames() { let id = StreamId::new(0, StreamDir::Uni, false); @@ -1817,6 +2012,7 @@ impl Session { base, last_send_at: last_send_at.clone(), rtt: rtt.clone(), + finished: Vec::new(), }; tokio::spawn(async move { writer.run().await }); diff --git a/rs/qmux/src/transport.rs b/rs/qmux/src/transport.rs index 32ab5ff7..9bbc4d9a 100644 --- a/rs/qmux/src/transport.rs +++ b/rs/qmux/src/transport.rs @@ -42,6 +42,20 @@ pub trait Writer: Send + 'static { /// Send a single complete message. fn send(&mut self, data: Bytes) -> impl std::future::Future> + Send; + /// Queue a single complete message without flushing it. The session's writer + /// feeds every frame that is already queued and then calls [`Writer::flush`] + /// once, so a burst of small frames reaches the socket in one write instead of + /// one write (and one TCP segment) each. The default sends immediately. + fn feed(&mut self, data: Bytes) -> impl std::future::Future> + Send { + self.send(data) + } + + /// Write out everything queued by [`Writer::feed`]. The default has nothing to + /// flush because the default `feed` already sent. + fn flush(&mut self) -> impl std::future::Future> + Send { + std::future::ready(Ok(())) + } + /// Gracefully close the transport. fn close(&mut self) -> impl std::future::Future> + Send; @@ -185,15 +199,25 @@ mod stream_transport { impl Writer for StreamWriter { async fn send(&mut self, data: Bytes) -> Result<(), Error> { + self.feed(data).await?; + self.flush().await + } + + async fn feed(&mut self, data: Bytes) -> Result<(), Error> { // Record-framed drafts (QMux01+) travel inside size-prefixed records // on byte streams. (Records are implicit on WebSocket, where the - // message boundary delimits them.) + // message boundary delimits them.) Left in the BufWriter, which only + // writes to the socket when full; `flush` sends the rest. if self.version.uses_records() { let mut size_buf = BytesMut::with_capacity(8); VarInt::try_from(data.len())?.encode(&mut size_buf); self.writer.write_all(&size_buf).await?; } self.writer.write_all(&data).await?; + Ok(()) + } + + async fn flush(&mut self) -> Result<(), Error> { self.writer.flush().await?; Ok(()) } @@ -823,6 +847,25 @@ mod ws_transport { Ok(()) } + async fn feed(&mut self, data: Bytes) -> Result<(), Error> { + use futures::SinkExt; + // Still one WebSocket message per frame (the message boundary is the + // record boundary), but tungstenite only appends it to its write + // buffer. The socket write happens on `flush`, or early once the + // buffer passes `write_buffer_size` (128 KiB by default). + self.sink + .feed(Message::Binary(data)) + .await + .map_err(|_| Error::Closed)?; + Ok(()) + } + + async fn flush(&mut self) -> Result<(), Error> { + use futures::SinkExt; + self.sink.flush().await.map_err(|_| Error::Closed)?; + Ok(()) + } + async fn close(&mut self) -> Result<(), Error> { use futures::SinkExt; self.sink.close().await.map_err(|_| Error::Closed)?;