From 958c525863ab6aada3fda359604423a472438f43 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 14:51:20 -0700 Subject: [PATCH 1/5] quest: start CLI FETCH clean close Co-Authored-By: GPT-6 From 559cdce6a4e28bde31851a61f0d728f2b5a7ade4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 15:09:09 -0700 Subject: [PATCH 2/5] fix(cli): close FETCH connection before exiting Co-Authored-By: GPT-6 --- quest/m1/moq-fetch-close.md | 2 ++ rs/moq-cli/src/fetch.rs | 15 +++++++++++---- test/interop/README.md | 2 +- test/interop/interop.sh | 31 +++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/quest/m1/moq-fetch-close.md b/quest/m1/moq-fetch-close.md index 2c8216e4588..ebc7443bf43 100644 --- a/quest/m1/moq-fetch-close.md +++ b/quest/m1/moq-fetch-close.md @@ -24,3 +24,5 @@ which it excludes because they may not close cleanly or log which connection closed. Public API: none. Wire: none. + +**Draft progress:** The connection is now held outside the timed operation and explicitly closed before the client endpoint closes, including on errors and deadline cancellation. A current-relay FETCH round is wired into the existing interop CI lane and passes the idle-out check. The new round also passes before the fix locally, so it does not yet prove deterministic sensitivity to the process-exit close-flush race. The quest remains open for review of that coverage limitation. diff --git a/rs/moq-cli/src/fetch.rs b/rs/moq-cli/src/fetch.rs index bddf5b1f6c2..32f9213f86f 100644 --- a/rs/moq-cli/src/fetch.rs +++ b/rs/moq-cli/src/fetch.rs @@ -74,19 +74,26 @@ async fn fetch( .with_subscriber(origin.clone()) .with_reconnect(false); - fetch_from( + let connection = client.connect(url); + let result = fetch_from( Request { args, path: &broadcast, deadline, }, async { - let connection = client.connect(url).established().await.context("failed to connect")?; + let connection = connection.clone().established().await.context("failed to connect")?; anyhow::Ok((origin, connection)) }, out, ) - .await + .await; + // Keep an independent handle outside the timed operation: cancellation of setup or + // a read must still send the session close before the runtime can stop. + let closed = connection.close().await; + client.close().await; + result?; + closed.context("failed to close fetch connection") } /// The operation's selection and one absolute deadline, including setup. @@ -104,7 +111,7 @@ async fn fetch_from( ) -> anyhow::Result<()> { let Request { args, path, deadline } = request; let result = timeout_at(deadline, async { - // Held until the last frame is read; dropping it closes the session. + // Held until the last frame is read; the caller closes the connection afterward. let (origin, _connection) = setup.await?; // Wait for a covering route rather than asking on the spot: the announcement diff --git a/test/interop/README.md b/test/interop/README.md index cc276caad37..d28fcd11659 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -23,7 +23,7 @@ The normal run also checks a finite raw track through the relay. Its publisher announces end 4 before writing four 256 KiB groups; the reader verifies every byte, exactly groups 0 through 3, and a clean end at 4. Rust-to-Rust always runs. Selecting native JS adds both Rust-to-JS and JS-to-Rust under that runtime, so `--all` covers -Node and Bun. `--tail` runs just these five lanes. The wire-compat lanes, which swap in released relays and clients, skip them. +Node and Bun. The current CLI also FETCHes group 0, verifies every byte, and closes under the relay idle-out check. `--tail` runs this FETCH round and the five finite-tail lanes. The wire-compat lanes, which swap in released relays and clients, skip them. The finite clients keep their session alive until the harness acknowledges the reader's complete clean end over stdin, so a lane tests delivery rather than diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 63370628dbb..e4e5130915d 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1048,6 +1048,35 @@ run_tail_pair() { rm "$fifo" } +# Exercise this checkout's CLI FETCH under the same relay close check as live readers. +run_fetch_round() { + local name="fetch-rust" fifo="$HARNESS_RUN/fetch-ack" broadcast="fetch-$$-$RANDOM" pid from + from=$(wc -l <"$HARNESS_RUN/relay.log") + from=$((from + 1)) + mkfifo "$fifo" + exec 9<>"$fifo" + harness_spawn "$name-pub" "$HARNESS_RUN/$name-pub.log" run_tail_publisher "$fifo" rust publish "$broadcast" "$(token_url --publish "$broadcast")" + pid="$HARNESS_PID" + if timeout -k 3 "$TIMEOUT" "$MOQ" --connect "$(token_url --subscribe "**/$broadcast")" \ + --broadcast "$broadcast" fetch tail --group 0 >"$HARNESS_RUN/$name.raw" && + python3 - "$HARNESS_RUN/$name.raw" <<'PYCODE' +import pathlib, sys +assert pathlib.Path(sys.argv[1]).read_bytes() == bytes(256 * 1024), "incorrect FETCH payload" +PYCODE + then + printf 'clean end\n' >&9 + harness_wait "$pid" || true + echo " PASS CLI FETCH (group 0; 262144 bytes)" + else + echo " FAIL CLI FETCH" + overall=1 + harness_reap "$pid" + fi + exec 9>&- + rm "$fifo" + check_idle_outs "$name" "$from" +} + # One media.ts invocation. It reports its own verdict (a negative control passes by failing on the # assertion it names), so the exit code is the whole answer. run_media() { @@ -1176,6 +1205,8 @@ elif [[ "$TAIL_ONLY" -eq 0 ]]; then fi if [[ "$NEGATIVE" -eq 0 && "$MEDIA" -eq 0 && "$IN_TREE" -eq 1 ]]; then + echo "=== CLI FETCH close ===" + run_fetch_round echo "=== finite track tails ===" run_tail_pair rust rust for runtime in js-native-node js-native-bun; do From ceae5c181f096557c0fdaf3073b0e508d1a3fa6b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 20:37:57 -0700 Subject: [PATCH 3/5] test(interop): require the FETCH publisher acknowledgement Reject a publisher failure after the FETCH payload instead of reporting a successful round without its acknowledgement. Co-Authored-By: GPT-6 Astra --- quest/m1/README.md | 1 - quest/m1/moq-fetch-close.md | 28 ---------------------------- test/interop/interop.sh | 8 +++++++- 3 files changed, 7 insertions(+), 30 deletions(-) delete mode 100644 quest/m1/moq-fetch-close.md diff --git a/quest/m1/README.md b/quest/m1/README.md index ecedacafb78..85e152f02a7 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -203,7 +203,6 @@ bench all require its crate. - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS - [A/V sync across a break](/quest/m1/watch-break-av-sync.md) - `@moq/watch` never plays pre-break audio out of sync with pre-break video - [moqsrc reconnect](/quest/m1/moqsrc-reconnect.md) - `moqsrc` redials after losing its relay and resumes on the same pads -- [moq fetch closes cleanly](/quest/m1/moq-fetch-close.md) - `moq fetch` closes its session before exiting instead of leaving the relay to time it out - [Terminal dial errors](/quest/m1/moq-tokio-terminal-errors.md) - moq-tokio's reconnect loop stops on errors that can never succeed - [Interop close log](/quest/m1/interop-close-log.md) - the interop idle-out check reads the relay close log at a pinned level diff --git a/quest/m1/moq-fetch-close.md b/quest/m1/moq-fetch-close.md deleted file mode 100644 index ebc7443bf43..00000000000 --- a/quest/m1/moq-fetch-close.md +++ /dev/null @@ -1,28 +0,0 @@ -# [XS] moq fetch closes its connection cleanly - -## Goal - -`moq fetch` (`rs/moq-cli/src/fetch.rs`) closes its session before exiting, -so the relay sees a clean close instead of timing the connection out. - -## Plan - -Found while merging #5174 (2026-10-10), which taught every interop client to -close cleanly and made the harness fail a round on an idled-out connection. -`moq fetch` still drops its connection without `Connection::close` or -`Client::close`; the js-native FETCH_DATA wire-compat cell reproduced the -relay timing it out. Only the wire-compat lanes reach it, and they skip the -idle-out check, so it doesn't fail CI today. - -Close the session the way #5174 did for the other clients, on every exit -after connecting, including an error or the `timeout_at` deadline. For the -regression, run a FETCH round with the current relay and the current -`moq fetch` under the idle-out check. No such round exists today (FETCH only -runs in the wire-compat lanes), so add one and wire it into CI. Don't enable -the check for the wire-compat cells: they mix in released relays and clients, -which it excludes because they may not close cleanly or log which connection -closed. - -Public API: none. Wire: none. - -**Draft progress:** The connection is now held outside the timed operation and explicitly closed before the client endpoint closes, including on errors and deadline cancellation. A current-relay FETCH round is wired into the existing interop CI lane and passes the idle-out check. The new round also passes before the fix locally, so it does not yet prove deterministic sensitivity to the process-exit close-flush race. The quest remains open for review of that coverage limitation. diff --git a/test/interop/interop.sh b/test/interop/interop.sh index e4e5130915d..bcfff80b116 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1066,7 +1066,13 @@ PYCODE then printf 'clean end\n' >&9 harness_wait "$pid" || true - echo " PASS CLI FETCH (group 0; 262144 bytes)" + if grep -qx 'tail acknowledged' "$HARNESS_RUN/$name-pub.log"; then + echo " PASS CLI FETCH (group 0; 262144 bytes)" + else + echo " FAIL CLI FETCH (publisher did not acknowledge)" + overall=1 + cat "$HARNESS_RUN/$name-pub.log" + fi else echo " FAIL CLI FETCH" overall=1 From bcca76a86de62cdd5850b0a002a4e341fecfde51 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 10:55:32 -0700 Subject: [PATCH 4/5] test(interop): close the tail fixture and scope idle checks to the round The Rust tail client dropped its session and client on exit, so the FETCH round's publisher idled out at the relay. Await Connection::close and Client::close before exiting, in both roles. relay_connections counted any connection that logged after FROM, so refused-round subscribers idling out during the FETCH round were blamed on it. The relay numbers connections in accept order; only count ids above the highest one logged before FROM. Label a connection the relay fetched from as the publisher too. Co-Authored-By: Claude Opus 5.5 --- test/interop/clients/rust/tail.rs | 10 ++++++++-- test/interop/interop.sh | 33 +++++++++++++++---------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/test/interop/clients/rust/tail.rs b/test/interop/clients/rust/tail.rs index 82f64f74669..32c20b5276f 100644 --- a/test/interop/clients/rust/tail.rs +++ b/test/interop/clients/rust/tail.rs @@ -26,7 +26,8 @@ async fn main() -> anyhow::Result<()> { let broadcast = origin.create_broadcast(&args[3])?; let mut track = broadcast.create_track("tail", None)?; broadcast.announce(Default::default())?; - let _session = client + let session = client + .clone() .with_publisher(&origin) .connect(url) .established() @@ -50,11 +51,13 @@ async fn main() -> anyhow::Result<()> { "reader did not acknowledge a clean end" ); broadcast.close(); + session.close().await?; eprintln!("tail acknowledged"); } "subscribe" => { let consumer = origin.consume(); - let _session = client + let session = client + .clone() .with_subscriber(origin) .connect(url) .established() @@ -86,9 +89,12 @@ async fn main() -> anyhow::Result<()> { "wrong groups: {seen:?}" ); ensure!(track.finished().await? == GROUPS, "wrong declared end"); + session.close().await?; eprintln!("tail clean end=4 groups=0,1,2,3 bytes=1048576"); } _ => anyhow::bail!("unknown role: {}", args[1]), } + // Dropping only queues the QUIC close; send it before the runtime stops. + client.close().await; Ok(()) } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index bcfff80b116..4c103414986 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -707,24 +707,23 @@ relay_log_since() { tail -n +"$1" "$HARNESS_RUN/relay.log" | sed 's/\x1b\[[0-9;]*m//g' } -# Print each relay connection that appears from log line FROM on, with how it closed: the +# Print each relay connection opened from log line FROM on, with how it closed: the # relay's close error, "ended" for a close without one, or "open" while it hasn't closed yet. # The patterns follow the relay's `conn{id=...}` span and its `connection closed` log -# (`rs/moq-relay/src/relay.rs`). Only connections opened in the window count, so a straggler -# from an earlier round closing here isn't blamed on this one. +# (`rs/moq-relay/src/relay.rs`). The relay numbers connections in accept order, so an id at +# or below the highest one logged before FROM was opened earlier: a straggler from an earlier +# round that logs or idles out here isn't blamed on this one. relay_connections() { - relay_log_since "$1" | awk ' - match($0, /conn\{id=[0-9]+ /) { - id = substr($0, RSTART + 8, RLENGTH - 9) - if (!(id in state)) { state[id] = "open"; order[++n] = id } - } - match($0, /connection closed id=[0-9]+/) { - id = substr($0, RSTART + 21, RLENGTH - 21) - if (id in state) { - rest = substr($0, RSTART + RLENGTH) - state[id] = sub(/^ err=/, "", rest) ? rest : "ended" - } - } + relay_log_since 1 | awk -v from="$1" ' + BEGIN { before = -1 } + { id = -1; closed = match($0, /connection closed id=[0-9]+/) } + closed { id = substr($0, RSTART + 21, RLENGTH - 21) + 0; rest = substr($0, RSTART + RLENGTH) } + !closed && match($0, /conn\{id=[0-9]+ /) { id = substr($0, RSTART + 8, RLENGTH - 9) + 0 } + id < 0 { next } + NR < from { if (id > before) before = id; next } + id <= before { next } + !(id in state) { state[id] = "open"; order[++n] = id } + closed { state[id] = sub(/^ err=/, "", rest) ? rest : "ended" } END { for (i = 1; i <= n; i++) print order[i], state[order[i]] } ' } @@ -744,9 +743,9 @@ check_idle_outs() { done while read -r id reason; do [[ "$reason" == "open" || "$reason" == *"timed out"* ]] || continue - # The relay subscribes upstream only on the publisher's connection. + # The relay subscribes or fetches upstream only on the publisher's connection. role=subscriber - relay_log_since "$from" | grep "conn{id=$id .*::subscriber: subscribe started" >/dev/null && role=publisher + relay_log_since "$from" | grep -E "conn\{id=$id .*::subscriber: (subscribe|fetch) started" >/dev/null && role=publisher echo " FAIL $pub round: relay connection $id ($role) idled out instead of closing ($reason)" relay_log_since "$from" | grep -E "conn\{id=$id |connection closed id=$id " | sed 's/^/ /' failed=1 From 8a14079bb435f4097351e0cedae311207b9bbc3c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 19:15:37 -0700 Subject: [PATCH 5/5] docs(interop): say --tail also runs the CLI FETCH close round Co-Authored-By: Claude Opus 5.5 --- test/interop/README.md | 2 +- test/justfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/interop/README.md b/test/interop/README.md index c3336e4d4d4..1dcbf80ce51 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -141,7 +141,7 @@ just test interop --all # Pick your own axes: just test interop --publishers rust,python --subscribers rust,c,js-native-bun -# Finite track tails only: Rust-to-Rust and Rust/Node/Bun in both directions. +# CLI FETCH close and finite track tails only: Rust-to-Rust and Rust/Node/Bun in both directions. just test interop --tail # Subscription termination: Rust/JS response bytes over in-memory transports. diff --git a/test/justfile b/test/justfile index a8d1aed0faa..dfc425602af 100644 --- a/test/justfile +++ b/test/justfile @@ -34,7 +34,7 @@ harness: # browser + C++ publish; everyone subscribes (incl. native node/bun JS, the # moq-c C client, and the GStreamer moqsrc plugin). Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. -# Finite track tails run too; `--tail` selects just Rust/Node/Bun tail checks. +# Finite track tails run too; `--tail` selects just the CLI FETCH close and Rust/Node/Bun tail checks. # Both runs first check js/net's varints against moq-net's encoder at every size boundary, # and on lite-06/07 messages (the filter also matches `lite_varint_interop`), then # moq-transport OBJECT_DATAGRAMs both ways on every draft, and Lite AUTH_ERROR