Repository navigation
fix(relay): enforce the handshake deadline on io_uring workers #5244
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
dec1d73
164fd8d
9f2ebf9
88b12f9
3f9aacd
ce7e418
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -70,6 +70,9 @@ struct Serve { | |
| versions: moq_net::Versions, | ||
| /// The address every worker socket shares, reported as a session's `local`. | ||
| local: SocketAddr, | ||
| /// `listen.timeout`, which each worker's endpoint starts at a connection's | ||
| /// first packet. | ||
| timeout: Option<std::time::Duration>, | ||
| } | ||
|
|
||
| /// A bound group of io_uring QUIC workers sharing one port. | ||
|
|
@@ -94,6 +97,7 @@ pub struct Workers { | |
| pin: bool, | ||
| alpns: Arc<Vec<String>>, | ||
| versions: moq_net::Versions, | ||
| timeout: Option<std::time::Duration>, | ||
|
|
||
| threads: Vec<(u16, std::thread::JoinHandle<()>)>, | ||
| stops: Vec<Arc<Stop>>, | ||
|
|
@@ -249,6 +253,7 @@ impl Workers { | |
| pin: config.pin, | ||
| alpns: Arc::new(alpns), | ||
| versions, | ||
| timeout: listen.resolved_timeout(), | ||
| threads: Vec::new(), | ||
| stops: Vec::new(), | ||
| failures, | ||
|
|
@@ -300,6 +305,7 @@ impl Workers { | |
| alpns: self.alpns.clone(), | ||
| versions: self.versions.clone(), | ||
| local: self.addr, | ||
| timeout: self.timeout, | ||
| }; | ||
|
|
||
| let cores = match self.pin { | ||
|
|
@@ -537,6 +543,7 @@ fn serve_worker(spawn: Spawn) -> bool { | |
| .context("failed to adopt the worker socket")?; | ||
| let mut endpoint_config = moq_uring::quic::endpoint::Config::default().with_server(server); | ||
| endpoint_config.activity = serve.shutdown.activity.clone(); | ||
| endpoint_config.handshake.timeout = serve.timeout; | ||
| let endpoint = | ||
| moq_uring::quic::Endpoint::new(socket, endpoint_config).context("failed to build the QUIC endpoint")?; | ||
| Ok((worker, endpoint)) | ||
|
|
@@ -583,6 +590,45 @@ fn serve_worker(spawn: Spawn) -> bool { | |
| true | ||
| } | ||
|
|
||
| /// What remains of a connection's handshake deadline, and how to tell the peer | ||
| /// it passed. | ||
| struct Deadline { | ||
| timer: moq_uring::Timer, | ||
| /// Closes with the timeout code once a session exists to carry it. Until | ||
| /// then a timed-out handshake is just dropped, and its guard closes in the | ||
| /// handshake's own error space, as HTTP/3 has no MoQ codes. | ||
| close: Option<moq_uring::quic::web::Session>, | ||
| } | ||
|
|
||
| impl Deadline { | ||
| fn new(handle: &moq_uring::Handle, conn: &moq_uring::quic::Connection) -> Self { | ||
| let mut timer = handle.timer(); | ||
| timer.set(conn.handshake().deadline); | ||
| Self { timer, close: None } | ||
| } | ||
|
|
||
| async fn bound<T, E: Into<anyhow::Error>>( | ||
| &mut self, | ||
| handshake: impl Future<Output = Result<T, E>>, | ||
| ) -> anyhow::Result<T> { | ||
| use web_transport_trait::poll::Session as _; | ||
| let timer = std::pin::pin!(self.timer.wait()); | ||
| let handshake = std::pin::pin!(handshake); | ||
| match futures::future::select(timer, handshake).await { | ||
| futures::future::Either::Left(_) => { | ||
| // Close before dropping the pending handshake, whose cancellation guard | ||
| // would otherwise replace the timeout with an abandoned-handshake error. | ||
| let err = moq_net::Error::Timeout; | ||
| if let Some(close) = &mut self.close { | ||
| close.close(moq_net::SessionError::Timeout.to_code(), &err.to_string()); | ||
| } | ||
| Err(err.into()) | ||
| } | ||
| futures::future::Either::Right((result, _)) => result.map_err(Into::into), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// Serve one accepted connection on its worker: handshake and session start | ||
| /// here; authentication and supervision on the shared runtime. | ||
| async fn serve_connection( | ||
|
|
@@ -593,6 +639,10 @@ async fn serve_connection( | |
| ) -> anyhow::Result<()> { | ||
| use web_transport_trait::poll::Session as _; | ||
|
|
||
| // The endpoint started this deadline at the connection's first packet and | ||
| // enforced it through the QUIC handshake; it covers SETUP OK too. | ||
| let mut deadline = Deadline::new(&handle, &conn); | ||
|
|
||
| // Read the link facts now: the handshake below consumes the connection. | ||
| let remote = conn.remote_addr(); | ||
| let sni = conn.server_name().map(str::to_owned); | ||
|
|
@@ -617,7 +667,8 @@ async fn serve_connection( | |
| let mut alpn = conn.protocol().map(str::to_owned); | ||
| let (transport, url) = match conn.protocol() { | ||
| Some("h3") => { | ||
| let request = moq_uring::quic::web::Request::accept(conn) | ||
| let request = deadline | ||
| .bound(moq_uring::quic::web::Request::accept(conn)) | ||
| .await | ||
| .context("WebTransport handshake failed")?; | ||
| let url = request.url().clone(); | ||
|
|
@@ -631,18 +682,19 @@ async fn serve_connection( | |
| response = response.with_protocol(protocol); | ||
| } | ||
| alpn = protocol.clone(); | ||
| let session = request | ||
| .respond(response) | ||
| let session = deadline | ||
| .bound(request.respond(response)) | ||
| .await | ||
| .context("WebTransport response failed")?; | ||
| (session, Some(url)) | ||
| } | ||
| _ => (moq_uring::quic::web::Session::raw(conn), None), | ||
| }; | ||
|
|
||
| let request = moq_net::Server::new() | ||
| .with_versions(serve.versions.clone()) | ||
| .accept_request_lite(std::time::Instant::now(), moq_uring::transport::Session::new(transport)) | ||
| deadline.close = Some(transport.clone()); | ||
| let server = moq_net::Server::new().with_versions(serve.versions.clone()); | ||
| let request = deadline | ||
| .bound(server.accept_request_lite(std::time::Instant::now(), moq_uring::transport::Session::new(transport))) | ||
| .await | ||
| .context("moq handshake failed")?; | ||
|
Comment on lines
+697
to
699
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a client finishes SETUP before the deadline but the configured auth server or embedded admission remains pending past it, this Useful? React with 👍 / 👎.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is for this PR. The tokio listener has the same shape: its deadline bounds the SETUP read and Whether (Written by Claude Opus 5.5) |
||
|
|
||
|
|
@@ -766,7 +818,7 @@ async fn serve_connection( | |
| request = request.with_subscriber(publisher); | ||
| } | ||
| let lease = lease.preparing(request.auth(), request.role())?; | ||
| let (session, driver) = request.ok().await?; | ||
| let (session, driver) = deadline.bound(request.ok()).await?; | ||
| let driver_handle = handle.clone(); | ||
| handle.spawn(async move { | ||
| match driver_handle.run(driver).await { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| //! The handshake an accepted connection goes through before it is served. | ||
| //! | ||
| //! An [`Endpoint`](super::Endpoint) bounds it by [`Config::timeout`] from the | ||
| //! connection's first packet, closing a QUIC handshake still running at the | ||
| //! deadline. The rest of the deadline is handed to whatever the application | ||
| //! layers on top (HTTP/3, MoQ SETUP) as [`Info::deadline`]. | ||
|
|
||
| use std::time::{Duration, Instant}; | ||
|
|
||
| /// How an endpoint bounds the handshakes it accepts. | ||
| #[derive(Clone, Debug, Default)] | ||
| #[non_exhaustive] | ||
| pub struct Config { | ||
| /// How long an incoming connection has from its first packet to finish its | ||
| /// whole handshake, or `None` (the default) for no bound but the idle | ||
| /// timeout. | ||
| pub timeout: Option<Duration>, | ||
| } | ||
|
|
||
| /// An established connection's handshake, from | ||
| /// [`Connection::handshake`](super::Connection::handshake). | ||
| #[derive(Clone, Copy, Debug)] | ||
| #[non_exhaustive] | ||
| pub struct Info { | ||
| /// When the handshake, including whatever the application layers on top of | ||
| /// QUIC, must be done. `None` on a dialed connection or without a | ||
| /// [`Config::timeout`]. | ||
| pub deadline: Option<Instant>, | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P2] Own the H3 close guard before racing an already-expired deadline
If TLS completes near the deadline, or the worker delays serving the accepted connection,
Deadline::boundcan first poll an already-expired timer.select(timer, handshake)then dropsRequest::accept(conn)without polling it, so the guard constructed inside that async function (quic/web.rs:139-143) never exists.deadline.closeis stillNone, and droppingConnectiondoes not close QUIC (quic/noq/connection.rs:265-270). The relay returns Timeout locally while the peer remains connected until the idle timeout, which keep-alives can extend. Establish H3-aware close ownership before entering this race, and test an accepted H3 connection whose deadline has elapsed before the first accept poll.Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed in 4315feb. I fixed it at the source rather than by reordering the
select.Request::acceptis now a plainfnreturning a future, and it creates the close guard before it returns. A handshake that is dropped before its first poll therefore still closes withH3_GENERAL_PROTOCOL_ERROR. The newan_unpolled_handshake_closes_the_connectiontest inrs/moq-uring/tests/web.rsdrops an unpolled accept and waits for the close. It fails when the guard is created at the first poll.(Written by Claude Opus 5.5)