diff --git a/CLAUDE.md b/CLAUDE.md index 73178ed4cf0..3fe77aa25bd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,7 @@ revision first and print base-to-current changes. Timing changes are information benchmark crashes, zero delivery, and invalid samples still fail the command. Relay CPU and RSS are included on Linux, where `moq-bench-host` can read `/proc`. -`just check`, `just test`, and `just fix` all diff the branch against its base and touch only the crates that changed plus everything depending on them, which is what keeps them fast when several worktrees are building at once. They skip a language entirely when the diff doesn't touch it. Reach for `just check-all` / `just test all` / `just fix-all` when you want the unscoped suite. See [Workflow](#workflow) for how the base is resolved. +`just check`, `just test`, and `just fix` all diff the branch against its base and touch only the crates that changed plus everything depending on them, which is what keeps them fast when several worktrees are building at once. They skip a language entirely when the diff doesn't touch it. Reach for `just check-all` / `just test all` / `just fix-all` when you want the unscoped suite. They also switch to it themselves once the changed-file list outgrows what a single argv/env string can hold (64 KiB, roughly 1500 paths), because the list is passed to each per-language recipe as one argument. See [Workflow](#workflow) for how the base is resolved. To force a base, `just check origin/dev` and `just fix origin/dev` take it positionally. `just test` can't: it's a module, so `just test origin/dev` looks for a *recipe* named `origin/dev`. Name the recipe to get past that: `just test default origin/dev`. diff --git a/justfile b/justfile index 3609cda9ebd..6815a8e327c 100644 --- a/justfile +++ b/justfile @@ -28,6 +28,10 @@ mod relay 'demo/relay' mod sub 'demo/sub' mod web 'demo/web' +# Byte budget for a changed-file list, sized so it survives being passed as a +# single argv/env string on every hop of the dispatch. See `_changed`. +changed_max := '65536' + # Run the demo by default. default: just demo @@ -47,10 +51,13 @@ install: cargo install --locked cargo-shear cargo-sort cargo-upgrades cargo-edit cargo-semver-checks release-plz # Reports the base it picked on stderr, so a surprising scope is traceable. +# +# LIMIT is the byte budget for the printed list, and exists as a parameter so +# `_changed-test` can force the oversized path without a synthetic 30k-file diff. -# Print the files this branch changed relative to BASE, one per line. +# Print the files this branch changed relative to BASE, one per line, or `ALL`. [private] -_changed $BASE: +_changed $BASE $LIMIT=changed_max: #!/usr/bin/env bash set -euo pipefail @@ -79,10 +86,126 @@ _changed $BASE: echo "base: $base" >&2 # Untracked files count too: a brand new crate or module is the whole change. - { + files=$({ git diff --name-only "$merge_base" git ls-files --others --exclude-standard - } | sort -u + } | sort -u) + + # Assigned rather than tested inline, so a failing `_changed-cap` aborts here + # under `set -e`. Inside `[[ ]]` its exit status is discarded, and the caller + # would then scope to a list that never got budgeted -- straight back into the + # E2BIG this exists to prevent. + cap=$(printf '%s' "$files" | just _changed-cap "$LIMIT") + if [[ "$cap" == ALL ]]; then + echo ALL + exit 0 + fi + + # Guarded because a bare printf of an empty list prints a newline, and the + # callers test the result for emptiness to decide whether anything changed. + if [[ -n "$files" ]]; then + printf '%s\n' "$files" + fi + +# Every hop of the dispatch takes the changed-file list as one argument, and +# just exports recipe parameters into the child's environment, so the whole list +# has to fit in a single execve string. Linux caps one string at MAX_ARG_STRLEN +# (32 pages, 131072 bytes) however large ARG_MAX is, so past that each hop dies +# with E2BIG, which just reports as exit code 126 and no mention of the diff. A +# diff that large selects most of the workspace anyway, so the callers widen to +# the unscoped suite, which passes no list at all. +# +# Split out from `_changed` so `_changed-test` can drive the decision with a +# synthetic list, rather than needing the working tree to hold a diff of a +# particular size. +# +# The list arrives on stdin, which is both the only channel that can carry an +# oversized one and the only way to measure it honestly: `${#var}` counts +# CHARACTERS under a UTF-8 locale while execve counts BYTES, so a path set of +# 3-byte characters would read as a third of its real size and sail past a +# budget it actually blows. + +# Print `ALL` when the changed-file list on stdin is too long for one argument. +[private] +_changed-cap $LIMIT=changed_max: + #!/usr/bin/env bash + set -euo pipefail + + [[ "$LIMIT" =~ ^[0-9]+$ ]] || { + echo "changed: not a byte count: $LIMIT" >&2 + exit 2 + } + + bytes=$(wc -c | tr -d '[:space:]') + if ((bytes > LIMIT)); then + echo "changed: $bytes bytes of paths exceeds the $LIMIT budget; selecting everything." >&2 + echo ALL + fi + +# Guards the thing that fails LOUDLY but unhelpfully: past the budget every +# `just` hop dies with "Argument list too long" and exit 126, naming neither the +# diff nor the recipe that could not receive it. Both halves matter -- a budget +# that never trips scopes nothing, and one above what execve accepts still dies. + +# Check that an oversized diff widens to `ALL`, and that the budget fits in argv. +[private] +_changed-test $LIMIT=changed_max: + #!/usr/bin/env bash + set -euo pipefail + + fail() { echo "changed: _changed-test: $1" >&2; exit 1; } + + # Synthetic sizes rather than whatever the working tree happens to hold: a + # clean checkout has no diff at all, and `check-all` runs there (cache.yml + # warms the cache from `main`), so a test keyed on the real list would take + # down the one job allowed to write the shared Rust cache. + [[ "$(printf 'aaaa' | just _changed-cap 3)" == ALL ]] || fail "over budget must print ALL" + [[ -z "$(printf 'aaa' | just _changed-cap 3)" ]] || fail "at budget must print nothing" + [[ -z "$(printf '' | just _changed-cap 3)" ]] || fail "an empty diff must print nothing" + + # The smallest nonempty list there is, against the only budget below it. + [[ "$(printf 'x' | just _changed-cap 0)" == ALL ]] || fail "a 1-byte list must exceed a 0 budget" + + # execve counts bytes, so the budget has to as well. Spelled as raw bytes + # rather than as characters: this is one CJK character, 3 bytes wide, which + # `${#var}` would count as 1 under a UTF-8 locale and let past a 2-byte + # budget it actually blows. + [[ "$(printf '\xe6\x97\xa5' | just _changed-cap 2)" == ALL ]] \ + || fail "the budget must count bytes, not characters" + + # A byte count is the whole input, so anything else is a caller bug, not a + # reason to silently scope to nothing. + ! printf '' | just _changed-cap not-a-number 2> /dev/null || fail "a bad budget must be rejected" + + # ...and that rejection has to reach the caller. A swallowed one would hand + # back an unbudgeted list, which is the failure this whole recipe prevents. + ! just _changed "" not-a-number 2> /dev/null || fail "a rejected budget must fail _changed" + + # Linux caps a single argv/env string at MAX_ARG_STRLEN (32 pages), whatever + # ARG_MAX says, and the list travels as one string. Asserted rather than + # probed because this repo's CI is the Linux host and a dev box may be laxer. + ((LIMIT <= 131072)) || fail "budget $LIMIT exceeds Linux MAX_ARG_STRLEN" + + # The budget still has to survive the hop it was sized for, which the checks + # above cannot show: they never pass a list that big to anything. + payload=$(head -c "$LIMIT" /dev/zero | tr '\0' x) + [[ "$(just _echo "$payload" | wc -c)" -eq $((LIMIT + 1)) ]] \ + || fail "a $LIMIT-byte list does not survive an argv hop" + + # End to end, but only when there is a diff to be oversized: see above. The + # budget is zero rather than one because the real list is whatever the + # checkout holds, and a single one-character root path is a one-byte list + # that a one-byte budget does not exceed. Zero is below every nonempty list. + if [[ -n "$(just _changed "" 100000000)" ]]; then + [[ "$(just _changed "" 0)" == ALL ]] || fail "a diff over budget must print ALL" + fi + + echo "changed: budget ok" + +# Print an argument back, to measure what survives a `just` invocation. +[private] +_echo $VALUE: + @printf '%s\n' "$VALUE" # Tools every scope guards with `command -v`, so an incomplete local toolchain # checks less instead of failing. That trade is wrong in CI, where a skip is @@ -149,6 +272,14 @@ check $BASE="": set -euo pipefail files=$(just _changed "$BASE") + + # `_changed` says ALL when the list outgrew what argv can carry. The unscoped + # suite is the path that passes no list at all, so it is the one that works. + if [[ "$files" == ALL ]]; then + just check-all + exit 0 + fi + just _tools "$files" # The dispatch below lives in these two files, and neither matches any @@ -267,6 +398,7 @@ _shell $ACTION: # Repository-wide lints, shared by `check` and `check-all`. [private] _check-common: + just _changed-test bun install --frozen-lockfile bun remark . --quiet --frail just _shell check @@ -285,6 +417,12 @@ fix $BASE="": files=$(just _changed "$BASE") + # Mirrors `check`: too long for argv means fix everything instead. + if [[ "$files" == ALL ]]; then + just fix-all + exit 0 + fi + if [[ -n "$files" ]]; then just js fix "$files" just rs fix-changed "$files" diff --git a/test/justfile b/test/justfile index 5af6a60a712..5b775991811 100644 --- a/test/justfile +++ b/test/justfile @@ -31,6 +31,13 @@ default $BASE="": set -euo pipefail files=$(just _changed "$BASE") + + # Mirrors the root `check`: too long for argv means test everything instead. + if [[ "$files" == ALL ]]; then + just all + exit 0 + fi + just _tools "$files" # Mirrors the root `check`: the dispatch below lives in these two files and