From d657c12f32a8c16c8d9eca521acc658f199a3041 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:45:51 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/test-flakes-3/cli-fetch-rust.md Quest-Claim: c6ded6e5eecd0723 From 13d969eda37a061ba62701291bb58d47571baf94 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 07:21:40 -0800 Subject: [PATCH 2/3] test(interop): the relay log keeps its warnings, and a failed CLI FETCH shows both sides The relay's RUST_LOG named only connection targets when none was inherited, which silenced every other relay warning, such as why it refused a request. Keep the relay's info level as the base, log its upstream subscribe and fetch starts (which the idle-out check already reads to name a publisher), run the Rust FETCH reader at debug, and print the relay's side of a failed FETCH round. The CLI FETCH rust -> rust flake did not reproduce on main in 34 full runs and about 1,400 FETCH rounds, so its quest is deleted pending the maintainer's decision. A follow-up quest stops sh/local.ts from loading the root .env. Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-3/cli-fetch-rust.md | 20 -------------------- quest/m1/local-env-file.md | 24 ++++++++++++++++++++++++ test/interop/interop.sh | 13 +++++++++---- 3 files changed, 33 insertions(+), 24 deletions(-) delete mode 100644 quest/m0/test-flakes-3/cli-fetch-rust.md create mode 100644 quest/m1/local-env-file.md diff --git a/quest/m0/test-flakes-3/cli-fetch-rust.md b/quest/m0/test-flakes-3/cli-fetch-rust.md deleted file mode 100644 index 0e7c09d9c79..00000000000 --- a/quest/m0/test-flakes-3/cli-fetch-rust.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Interop's CLI FETCH rust -> rust finds its group - -## Goal - -`just test interop --all`'s `CLI FETCH rust -> rust` case fetches group 1 of -`tail` on every run, instead of sometimes failing with "group 1 of tail not -found". - -## Plan - -Seen 2026-11-15 on the branch of -[moq-dev/moq#8836](https://github.com/moq-dev/moq/pull/8836), whose change -touches only lite-07 AUTH; the case runs on `moq-lite-06`, and the next full -run passed. The fetcher's session logged `session error err=no error` and the -publisher's `track::Producer dropped without finish() or abort()` just before -the fetch failed, so the publisher may end the track before the fetch is -served. Reproduce under load (`test/interop/interop.sh`), find whether the -fetch races the publisher's exit or the relay's cache, and fix that cause. - -Public API: none. Wire: none. diff --git a/quest/m1/local-env-file.md b/quest/m1/local-env-file.md new file mode 100644 index 00000000000..3c8bbb2c7ab --- /dev/null +++ b/quest/m1/local-env-file.md @@ -0,0 +1,24 @@ +# [XS] A local suite's environment does not depend on where it starts + +## Goal + +A suite run through `sh/local.sh` gets the same environment whichever +directory it starts from, so a local run reproduces CI's logging and timing. + +## Plan + +Found 2026-11-16 while reproducing +[the interop FETCH flake](https://github.com/moq-dev/moq/pull/8894): +`sh/local.sh` runs `sh/local.ts` under `bun`, which loads `.env` from the +working directory. The repository's `.env` sets `RUST_LOG="debug"` and +`RUST_BACKTRACE=1`, so a suite started from the repository root runs every +Rust process at debug, while `just test interop` (whose recipes run in `test/`) +and CI run at the default info. Debug logging slows every relay and client +enough to change which races a run can hit. + +Recommendation: run `sh/local.ts` with `bun --no-env-file`, as `quest` itself +runs, and delete the root `.env` if nothing else reads it (it dates from +[moq-dev/moq#928](https://github.com/moq-dev/moq/pull/928)). A developer who +wants debug output sets `RUST_LOG` in their shell. + +Public API: none. Wire: none. diff --git a/test/interop/interop.sh b/test/interop/interop.sh index fbd959d6539..bf643562b6a 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -984,10 +984,12 @@ else METRICS_URL="http://127.0.0.1:${HARNESS_PORT}/metrics" printf '\n[internal]\nlisten = "127.0.0.1:%s"\n\n[stats]\nenabled = true\n' "$HARNESS_PORT" >>"$HARNESS_RUN/relay.toml" fi -# Connection spans and clean closes must survive an inherited warn-only filter. The route key lane -# reads the relay's debug line for each verified proof. +# The relay's own level stays the base: a RUST_LOG naming only targets would silence every other +# warning, such as why the relay refused a request. Connection spans and clean closes must survive an +# inherited warn-only filter. The route key lane reads the relay's debug line for each verified proof, +# and the idle-out check its upstream subscribe and fetch starts. harness_spawn relay "$HARNESS_RUN/relay.log" env \ - "RUST_LOG=${RUST_LOG:+$RUST_LOG,}moq_relay::connection=info,moq_relay::relay=info,moq_net::lite::setup=debug" \ + "RUST_LOG=${RUST_LOG:-info},moq_relay::connection=info,moq_relay::relay=info,moq_net::lite::setup=debug,moq_net::lite::subscriber=debug" \ "$RELAY" "$HARNESS_RUN/relay.toml" if ! harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"; then echo "relay never became ready" >&2 @@ -2146,6 +2148,8 @@ run_fetch_round() { why="incorrect or missing FETCH payload" overall=1 cat "$LOGS/$name-pub.log" "$LOGS/$name-sub.log" + # The run's last lines may be a later round's, so print the relay's side of this one. + relay_log_since "$from" | sed 's/^/ relay: /' harness_reap "$pid" fi exec 9>&- @@ -2166,7 +2170,8 @@ run_fetch_subscriber() { grep -qx 'fetch clean group=1 frames=3 bytes=196608' "$LOGS/$name-sub.log" return fi - timeout -k 3 "$TIMEOUT" "$MOQ" --connect "$(token_url --subscribe "$(subscribe_grant "$broadcast")")" ${version:+--connect-version "$version"} \ + # At debug, a failed fetch says whether it went on the wire and how the relay answered it. + RUST_LOG="${RUST_LOG:-info},moq_net=debug" timeout -k 3 "$TIMEOUT" "$MOQ" --connect "$(token_url --subscribe "$(subscribe_grant "$broadcast")")" ${version:+--connect-version "$version"} \ --broadcast "$broadcast" fetch tail --group 1 --json >"$LOGS/$name.json" 2>"$LOGS/$name-sub.log" && python3 - "$LOGS/$name.json" <<'PYCODE' import base64, json, sys From 4e8fc12302ab06679d43e25e64b58b167276ef8d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:38:23 -0800 Subject: [PATCH 3/3] interop: keep an info base under a target-only RUST_LOG, and print a failed FETCH's relay log after its connections close Co-Authored-By: Claude Opus 5.5 --- test/interop/interop.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 28b5a8eab85..d98244c15d7 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -984,12 +984,13 @@ else METRICS_URL="http://127.0.0.1:${HARNESS_PORT}/metrics" printf '\n[internal]\nlisten = "127.0.0.1:%s"\n\n[stats]\nenabled = true\n' "$HARNESS_PORT" >>"$HARNESS_RUN/relay.toml" fi -# The relay's own level stays the base: a RUST_LOG naming only targets would silence every other -# warning, such as why the relay refused a request. Connection spans and clean closes must survive an +# info stays the base, ahead of the inherited RUST_LOG so its own level still wins: a filter naming +# only targets, such as quinn=debug, would otherwise silence every other warning, such as why the +# relay refused a request. Connection spans and clean closes must survive an # inherited warn-only filter. The route key lane reads the relay's debug line for each verified proof, # and the idle-out check its upstream subscribe and fetch starts. harness_spawn relay "$HARNESS_RUN/relay.log" env \ - "RUST_LOG=${RUST_LOG:-info},moq_relay::connection=info,moq_relay::relay=info,moq_net::lite::setup=debug,moq_net::lite::subscriber=debug" \ + "RUST_LOG=info,${RUST_LOG:+$RUST_LOG,}moq_relay::connection=info,moq_relay::relay=info,moq_net::lite::setup=debug,moq_net::lite::subscriber=debug" \ "$RELAY" "$HARNESS_RUN/relay.toml" if ! harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"; then echo "relay never became ready" >&2 @@ -1037,7 +1038,7 @@ subscribe_grant() { } # moq-cli logs each grant it receives over AUTH at debug, under `moq_net::auth`. -CLI_LOG="${RUST_LOG:-info},moq_net::auth=debug" +CLI_LOG="info,${RUST_LOG:+$RUST_LOG,}moq_net::auth=debug" # The lite version with AUTH, which no client offers by default. The clients the # harness reads a grant or a refusal from dial it. Compat dials its pinned version. @@ -2111,7 +2112,7 @@ run_fetch_round() { local pub="$1" sub="$2" mode="${3:-cached}" pid from version local name="fetch-$1-$2${3:+-$3}" fifo="$LOGS/fetch-ack" broadcast="fetch-$1-$$-$RANDOM" local label="CLI FETCH${3:+ $3}" - local started=$SECONDS why="" + local started=$SECONDS why="" fetch_failed=0 local cell=(case="$label" publisher="$pub" subscriber="$sub" log="$LOGS/$name-sub.log") # The relay fills its miss with an upstream FETCH, which a JS publisher refuses on # moq-transport by design: it serves FETCH only over moq-lite. @@ -2148,8 +2149,7 @@ run_fetch_round() { why="incorrect or missing FETCH payload" overall=1 cat "$LOGS/$name-pub.log" "$LOGS/$name-sub.log" - # The run's last lines may be a later round's, so print the relay's side of this one. - relay_log_since "$from" | sed 's/^/ relay: /' + fetch_failed=1 harness_reap "$pid" fi exec 9>&- @@ -2159,6 +2159,11 @@ run_fetch_round() { why="${why:-a relay connection negotiated another version}" overall=1 fi + # The run's last lines may be a later round's, so print the relay's side of this one. Printed + # after check_idle_outs waited for the round's connections, so it includes how each closed. + if ((fetch_failed)); then + relay_log_since "$from" | sed 's/^/ relay: /' + fi cell+=(seconds=$((SECONDS - started))) if [[ -z "$why" ]]; then record pass "${cell[@]}"; else record fail "${cell[@]}" detail="$why"; fi } @@ -2171,7 +2176,7 @@ run_fetch_subscriber() { return fi # At debug, a failed fetch says whether it went on the wire and how the relay answered it. - RUST_LOG="${RUST_LOG:-info},moq_net=debug" timeout -k 3 "$TIMEOUT" "$MOQ" --connect "$(token_url --subscribe "$(subscribe_grant "$broadcast")")" ${version:+--connect-version "$version"} \ + RUST_LOG="info,${RUST_LOG:+$RUST_LOG,}moq_net=debug" timeout -k 3 "$TIMEOUT" "$MOQ" --connect "$(token_url --subscribe "$(subscribe_grant "$broadcast")")" ${version:+--connect-version "$version"} \ --broadcast "$broadcast" fetch tail --group 1 --json >"$LOGS/$name.json" 2>"$LOGS/$name-sub.log" && python3 - "$LOGS/$name.json" <<'PYCODE' import base64, json, sys