From 797e7afc987ca57510d19b5c6954595ab37e86bd Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 08:19:28 -0400 Subject: [PATCH 01/27] fix(ci): pin drills-nightly to nself-ci 1.2.11, same as nself-ci.yml (D-1090) The nightly failed at its first step because CI_VERSION, CI_RELEASE_URL and CI_RELEASE_SHA256 were empty placeholders. v1.2.11 is the pin nself-ci.yml already runs green; sha256 checked against the release asset digest. --- .github/workflows/drills-nightly.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/drills-nightly.yml b/.github/workflows/drills-nightly.yml index 1bc4e0759..05fdb9037 100644 --- a/.github/workflows/drills-nightly.yml +++ b/.github/workflows/drills-nightly.yml @@ -9,7 +9,8 @@ # stops firing shows as a missing run here, not as an `overdue` report. # The pin: CI_VERSION, CI_RELEASE_URL and CI_RELEASE_SHA256 take the `ci-release` handoff (tag plus sha256 of each # platform tarball) of P7-PLUG-69, the first plugins release that holds `--pipeline` and drill jobs (Epic P7-CI D34). -# Until that release exists the first step fails with a clear message; filling the three values is the whole change. +# Moving the pin means changing these three values here, in nself-ci.yml and in nself-ci-source.yml in one commit. If a +# value is ever empty again the first step fails with a clear message instead of running an unpinned binary. # See also scripts/drill/README.md and .github/wiki/Catalog.md (Freshness drill). name: drills-nightly @@ -28,9 +29,9 @@ concurrency: env: PIPELINE: drills # the pipeline name in .nself/ci.yaml - CI_VERSION: "" # the pinned release, for example 1.2.7 (shown in the summary) - CI_RELEASE_URL: "" # https URL of the release tarball for this runner's platform - CI_RELEASE_SHA256: "" # sha256 of that tarball, 64 hex digits + CI_VERSION: "1.2.11" # the pinned release (shown in the summary); same pin as nself-ci.yml + CI_RELEASE_URL: "https://github.com/nself-org/plugins/releases/download/v1.2.11/ci-1.2.11-linux-amd64.tar.gz" + CI_RELEASE_SHA256: "0ca65f839b007764f70ba9e8e719aee88f336efb74b932297b2b9097cbdd0016" NSELF_V15: "1" # the plugin binary does not follow the cli flip; kept until v1.6 jobs: From d3be313589e59f0f41d1f07fdc8ed9b93d0a0e54 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:02:42 -0400 Subject: [PATCH 02/27] fix(ci): make the drills pipeline and nself-ci/source runnable (D-1090, D-1093) - drills-nightly: Go, docker/podman and gVisor runsc set up as in ci-isolation-linux; nself-ci on PATH; read-only GH_TOKEN, checks/statuses read; run store kept and job logs uploaded - nself-ci-source: --trust owner for a head in this repository (a PR merge ref is untrusted otherwise, E625) - toolchain-pins: one compact JSON line (E640); ghcr.io images resolve like docker.io - truth-sweep: list public repositories anonymously when no token reaches the drill - security-suite/mutate.sh: copies see the sibling forgejo and github-runner plugins, so the baseline is green - boundary test: agent/update/source.go may import net/http (same release fetch as agent/install) - blob rig: warm-up with a frame that does not fail the route (TestE14BlobAnnouncementForms was racy) --- .github/workflows/drills-nightly.yml | 44 ++++++++++++++++++- .github/workflows/nself-ci-source.yml | 9 +++- .../ci/internal/invariants/testdata/mutate.sh | 5 +++ free/ci/test/security/boundary_test.go | 7 ++- .../security/redteam/e_attack_blob_test.go | 8 ++-- scripts/drill/security-suite/run | 3 ++ scripts/drill/toolchain-pins/run | 40 ++++++++++++----- scripts/drill/truth-sweep/run | 22 ++++++++-- 8 files changed, 116 insertions(+), 22 deletions(-) diff --git a/.github/workflows/drills-nightly.yml b/.github/workflows/drills-nightly.yml index 05fdb9037..f8ddc08c8 100644 --- a/.github/workflows/drills-nightly.yml +++ b/.github/workflows/drills-nightly.yml @@ -22,6 +22,8 @@ on: permissions: contents: read + checks: read # main-status-audit reads the check runs of main's commits + statuses: read # ... and their commit statuses concurrency: group: drills-nightly-${{ github.ref }} @@ -33,6 +35,9 @@ env: CI_RELEASE_URL: "https://github.com/nself-org/plugins/releases/download/v1.2.11/ci-1.2.11-linux-amd64.tar.gz" CI_RELEASE_SHA256: "0ca65f839b007764f70ba9e8e719aee88f336efb74b932297b2b9097cbdd0016" NSELF_V15: "1" # the plugin binary does not follow the cli flip; kept until v1.6 + GVISOR_RELEASE: "20261012.0" # same pin and checksum as ci-isolation-linux.yml + GVISOR_SHA512: "9606445eed5aedfcf05310e7ccef61d994089914c6622fc82624c24d2b18db1b2210f40e40a7be27b8b7772859de7d95c501b4eba66423053c960bf671377c5e" + TRUST04_TEST_IMAGE: "alpine:3.19" jobs: pipeline: @@ -55,6 +60,36 @@ jobs: with: fetch-depth: 0 + # The drills run Go (security-suite, truth-sweep, scenarios), docker and podman (isolation-levels) and + # gVisor's runsc registered with docker; the steps are the ones ci-isolation-linux.yml runs. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: free/ci/go.mod + cache-dependency-path: free/ci/go.sum + + - name: Container runtimes + run: | + set -euo pipefail + docker info --format '{{.ServerVersion}} rootless={{.SecurityOptions}}' + if ! command -v podman >/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y -qq podman + fi + podman info --format '{{.Version.Version}} rootless={{.Host.Security.Rootless}}' + docker pull "$TRUST04_TEST_IMAGE" + + - name: Install gVisor runsc + run: | + set -euo pipefail + base="https://storage.googleapis.com/gvisor/releases/release/${GVISOR_RELEASE}/x86_64" + curl -fsSL -o "$RUNNER_TEMP/gvisor.tar.bz2" "$base/gvisor.tar.bz2" + echo "${GVISOR_SHA512} $RUNNER_TEMP/gvisor.tar.bz2" | sha512sum -c - + sudo tar -xjf "$RUNNER_TEMP/gvisor.tar.bz2" -C /usr/local/bin + sudo runsc install + sudo systemctl restart docker + docker info --format '{{json .Runtimes}}' | tee "$RUNNER_TEMP/runtimes.json" + grep -q runsc "$RUNNER_TEMP/runtimes.json" + - name: Install the pinned nself-ci run: | set -euo pipefail @@ -70,11 +105,16 @@ jobs: fi chmod +x "$bin" echo "NSELF_CI_BIN=$bin" >> "$GITHUB_ENV" + # The drills look the binary up by name (main-status-audit), and the engine's job environment does not carry NSELF_CI_BIN. + echo "$(dirname "$bin")" >> "$GITHUB_PATH" - name: Install nft (scenario 1 cuts the network) run: command -v nft >/dev/null || (sudo -n apt-get update -qq && sudo -n apt-get install -y -qq nftables) - name: Run the pipeline + env: + GH_TOKEN: ${{ github.token }} # read-only; the drills that call gh (main-status-audit, truth-sweep) take it from here + NSELF_CI_HOME: ${{ runner.temp }}/ci-home # a known run store, so the job logs can be kept below run: | set -o pipefail mkdir -p "$RUNNER_TEMP/evidence" @@ -91,6 +131,8 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: evidence-${{ github.run_id }} - path: ${{ runner.temp }}/evidence/ + path: | + ${{ runner.temp }}/evidence/ + ${{ runner.temp }}/ci-home/logs/ if-no-files-found: warn retention-days: 30 diff --git a/.github/workflows/nself-ci-source.yml b/.github/workflows/nself-ci-source.yml index a9cc3e984..24f59e5c6 100644 --- a/.github/workflows/nself-ci-source.yml +++ b/.github/workflows/nself-ci-source.yml @@ -67,11 +67,18 @@ jobs: - name: Run the gate env: NSELF_CI_BASE_SHA: ${{ github.event.pull_request.base.sha }} + # A pull request checkout is a merge ref no branch contains, so the built binary's source + # trust (P7-TRUST-13) classes it untrusted and refuses to run (E625). A head in this very + # repository is the owner's own code on a disposable hosted runner: say so with the audited + # --trust owner. A fork head leaves this empty, stays untrusted and is refused, as intended. + NSELF_CI_TRUST: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'owner' || '' }} run: | set -o pipefail mkdir -p "$RUNNER_TEMP/evidence" status=0 - "$NSELF_CI_BIN" --no-status --json > "$RUNNER_TEMP/evidence/nself-ci-source.json" || status=$? + args=(--no-status --json) + if [ -n "$NSELF_CI_TRUST" ]; then args+=(--trust "$NSELF_CI_TRUST"); fi + "$NSELF_CI_BIN" "${args[@]}" > "$RUNNER_TEMP/evidence/nself-ci-source.json" || status=$? { echo "### nself-ci/source (built from $GITHUB_SHA)" echo "exit code: $status" diff --git a/free/ci/internal/invariants/testdata/mutate.sh b/free/ci/internal/invariants/testdata/mutate.sh index 76b972e96..bd020abf3 100644 --- a/free/ci/internal/invariants/testdata/mutate.sh +++ b/free/ci/internal/invariants/testdata/mutate.sh @@ -14,6 +14,11 @@ copy_module() { mkdir -p "$scratch/module" rsync -a --exclude vendor "$module/" "$scratch/module/" if [ -d "$module/vendor" ]; then ln -s "$module/vendor" "$scratch/module/vendor"; fi + # Tests of the security suite read the sibling plugins (../../../../forgejo, ../../../../github-runner from + # test/security/redteam). The copy lives in $scratch, so link those siblings next to it or the baseline is red. + for sibling in forgejo github-runner; do + if [ -d "$module/../$sibling" ] && [ ! -e "$scratch/$sibling" ]; then ln -s "$(cd "$module/../$sibling" && pwd)" "$scratch/$sibling"; fi + done } copy_module if (cd "$scratch/module" && CGO_ENABLED=0 go test -mod=vendor -count=1 -timeout 120s -v -run "$regex" "$@") >"$scratch/baseline.out" 2>&1; then diff --git a/free/ci/test/security/boundary_test.go b/free/ci/test/security/boundary_test.go index c07c54ec5..350b1584a 100644 --- a/free/ci/test/security/boundary_test.go +++ b/free/ci/test/security/boundary_test.go @@ -203,8 +203,11 @@ func TestNoTelemetryClientInTheProduct(t *testing.T) { checked++ slash := filepath.ToSlash(path) // internal/agent/install downloads the agent release the operator named (a one-way fetch of a binary - // by checksum); every other package of the agent runs jobs and speaks only the carrier protocol. - inAgent := (strings.Contains(slash, "/internal/agent/") || strings.Contains(slash, "/cmd/agent/")) && !strings.Contains(slash, "/internal/agent/install/") + // by checksum); internal/agent/update/source.go is the same fetch for an upgrade: it only wraps the + // http.Client that install.Release verifies with (P7-NODE-24) and sends nothing but that request. Every + // other package of the agent runs jobs and speaks only the carrier protocol. + inAgent := (strings.Contains(slash, "/internal/agent/") || strings.Contains(slash, "/cmd/agent/")) && + !strings.Contains(slash, "/internal/agent/install/") && !strings.HasSuffix(slash, "/internal/agent/update/source.go") for _, imp := range file.Imports { if inAgent && imp.Path.Value == `"net/http"` { t.Errorf("%s: the agent imports net/http", slash) diff --git a/free/ci/test/security/redteam/e_attack_blob_test.go b/free/ci/test/security/redteam/e_attack_blob_test.go index 9f558b159..874a2ba30 100644 --- a/free/ci/test/security/redteam/e_attack_blob_test.go +++ b/free/ci/test/security/redteam/e_attack_blob_test.go @@ -112,11 +112,11 @@ func newBlobRig(t *testing.T, r *eRig, n string) *eBlobRig { _ = b.d.Run(ctx, node, protocolLease(lease, att, ep), b.sink) }() t.Cleanup(func() { cancel(); <-b.done }) - // Run registers the route asynchronously: a harmless frame for an unknown stream proves it is there. + // Run registers the route asynchronously: a frame of a type the handler does not act on proves it is there + // (nil error), and unlike a refused announcement it does not fail the route, so the frames a test sends next + // still reach a live route instead of racing its release. for i := 0; i < 100; i++ { - _, err := b.d.Blobs().Frame(ctx, "n-1", b.msg(&protocol.Digest{Stream: "", SHA256: "", Size: 0}, "digest")) - var ref interface{ Refusal() bool } - if err != nil && errors.As(err, &ref) && ref.Refusal() && !strings.Contains(err.Error(), "no run is waiting") { + if _, err := b.d.Blobs().Frame(ctx, "n-1", b.msg(struct{}{}, "ping")); err == nil { return b } time.Sleep(20 * time.Millisecond) diff --git a/scripts/drill/security-suite/run b/scripts/drill/security-suite/run index d0f151217..1f0c5f94f 100755 --- a/scripts/drill/security-suite/run +++ b/scripts/drill/security-suite/run @@ -48,6 +48,9 @@ for ((g = 0; g < jobs; g++)); do rsync -a --exclude vendor --exclude test/security/testdata/regressions "$root/free/ci/" "$m/" ln -s "$root/free/ci/vendor" "$m/vendor" done +# Two redteam tests read the sibling plugins (../../../../forgejo, ../../../../github-runner from test/security/redteam): +# the copies sit in $tmp, so $tmp must hold those directories, or the baseline of every copy is red. +for sibling in forgejo github-runner; do ln -s "$root/free/$sibling" "$tmp/$sibling"; done i=0 for patch in "${patches[@]}"; do cp "$patch" "$tmp/m$((i % jobs))/test/security/testdata/regressions/" diff --git a/scripts/drill/toolchain-pins/run b/scripts/drill/toolchain-pins/run index dc1bfc9fa..b4b583eaf 100755 --- a/scripts/drill/toolchain-pins/run +++ b/scripts/drill/toolchain-pins/run @@ -58,24 +58,44 @@ newer() { # newer CURRENT LATEST: true when LATEST sorts after CURRENT [ -n "$2" ] && [ "$1" != "$2" ] && [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -1)" = "$2" ] } -# oci_ok REF DIGEST: the digest still resolves (anonymous Docker Hub pull token; docker.io images only). +# oci_where REF: sets oci_repo, oci_token_url and oci_base for the registry that REF names (docker.io, the default, +# or ghcr.io); anonymous pull tokens only. Any other registry host returns 1 and the check fails closed. +oci_where() { + local ref=$1 host=docker.io + case "$ref" in + docker.io/*) ref=${ref#docker.io/} ;; + ghcr.io/*) host=ghcr.io ref=${ref#ghcr.io/} ;; + *.*/*) return 1 ;; + esac + oci_repo=$ref + case "$host" in + ghcr.io) + oci_token_url="https://ghcr.io/token?service=ghcr.io&scope=repository:$oci_repo:pull" + oci_base=https://ghcr.io ;; + *) + oci_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$oci_repo:pull" + oci_base=https://registry-1.docker.io ;; + esac +} + +# oci_ok REF DIGEST: the digest still resolves (anonymous pull token; docker.io and ghcr.io images). oci_ok() { - local repo=${1#docker.io/} - local tok - tok=$(get "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$repo:pull" | jq -r .token) || return 1 + local tok oci_repo oci_token_url oci_base + oci_where "$1" || return 1 + tok=$(get "$oci_token_url" | jq -r .token) || return 1 curl "${curl_opts[@]}" --fail --head --output /dev/null -H "Authorization: Bearer $tok" \ -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ - "$(u "https://registry-1.docker.io/v2/$repo/manifests/$2")" + "$(u "$oci_base/v2/$oci_repo/manifests/$2")" } # oci_tag_digest REF TAG: the digest the tag points at now (empty when unreadable). oci_tag_digest() { - local repo=${1#docker.io/} - local tok - tok=$(get "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$repo:pull" 2>/dev/null | jq -r .token) || return 0 + local tok oci_repo oci_token_url oci_base + oci_where "$1" || return 0 + tok=$(get "$oci_token_url" 2>/dev/null | jq -r .token) || return 0 curl "${curl_opts[@]}" --fail --head --output /dev/null --dump-header - -H "Authorization: Bearer $tok" \ -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \ - "$(u "https://registry-1.docker.io/v2/$repo/manifests/$2")" 2>/dev/null | tr -d '\r' | + "$(u "$oci_base/v2/$oci_repo/manifests/$2")" 2>/dev/null | tr -d '\r' | awk -F': ' 'tolower($1) == "docker-content-digest" {print $2}' || true } @@ -150,7 +170,7 @@ case "${GITHUB_EVENT_NAME:-}" in schedule | push | pull_request) trigger=$GITHUB platform=$(uname -s | tr A-Z a-z)/$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/) cfg=$(sha "$versions" | cut -d' ' -f1) -jq -s --arg revision "$revision" --arg repo "$repo_id" --arg run "${GITHUB_RUN_ID:-toolchain-pins-$t0}" --arg trigger "$trigger" \ +jq -sc --arg revision "$revision" --arg repo "$repo_id" --arg run "${GITHUB_RUN_ID:-toolchain-pins-$t0}" --arg trigger "$trigger" \ --arg cfg "$cfg" --argjson dirty "$dirty" --arg platform "$platform" --arg host "$(hostname)" \ --arg provider "$([ -n "${GITHUB_RUN_ID:-}" ] && echo github || echo local)" \ --arg started "$started" --arg finished "$finished" --argjson ms "$((($(date +%s) - t0) * 1000))" \ diff --git a/scripts/drill/truth-sweep/run b/scripts/drill/truth-sweep/run index 6f1d69475..b8eb0e3c4 100755 --- a/scripts/drill/truth-sweep/run +++ b/scripts/drill/truth-sweep/run @@ -37,18 +37,32 @@ fi if [ -n "${TRUTH_REPOS_FILE:-}" ]; then repos=$(grep -v '^[[:space:]]*\(#\|$\)' "$TRUTH_REPOS_FILE" || true) else - command -v gh >/dev/null || { echo "truth-sweep: gh is required" >&2; exit 2; } + limit=200 if [ -n "${GH_TOKEN:-${GITHUB_TOKEN:-}}" ]; then + command -v gh >/dev/null || { echo "truth-sweep: gh is required" >&2; exit 2; } token=${GH_TOKEN:-$GITHUB_TOKEN} mkdir -p "$tmp/gh-config/gh" chmod 700 "$tmp/gh-config" "$tmp/gh-config/gh" printf 'github.com:\n oauth_token: %s\n git_protocol: https\n' "$token" >"$tmp/gh-config/gh/hosts.yml" chmod 600 "$tmp/gh-config/gh/hosts.yml" export XDG_CONFIG_HOME="$tmp/gh-config" + listed=$(gh repo list "$org" --visibility public --no-archived --limit "$limit" --json name,diskUsage \ + -q '.[] | select(.diskUsage > 0) | .name') + else + # No token reaches a drill (the exec funnel drops them) and every repository read here is public: + # list through the anonymous REST API, 100 per page, at most `limit` repositories. + command -v curl >/dev/null || { echo "truth-sweep: curl is required without a token" >&2; exit 2; } + listed="" + for page in 1 2; do + chunk=$(curl --fail --silent --show-error --location --proto '=https' \ + "https://api.github.com/orgs/$org/repos?type=public&per_page=100&page=$page" | + jq -r '.[] | select(.archived == false and .size > 0) | .name') || { + echo "truth-sweep: the anonymous repository list failed" >&2 + exit 2 + } + listed=$(printf '%s\n%s\n' "$listed" "$chunk" | grep . || true) + done fi - limit=200 - listed=$(gh repo list "$org" --visibility public --no-archived --limit "$limit" --json name,diskUsage \ - -q '.[] | select(.diskUsage > 0) | .name') if [ "$(printf '%s\n' "$listed" | grep -c .)" -ge "$limit" ]; then echo "truth-sweep: $limit repositories listed: the list may be cut off; raise the limit" >&2 exit 2 From ff3f92734300f32cde3956f79065359555d40631 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:20:36 -0400 Subject: [PATCH 03/27] test(ci): refresh regression patch 23 against the current deliver.go (it no longer applied) --- .../23-lease-checkout-revision-empty.patch | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/free/ci/test/security/testdata/regressions/23-lease-checkout-revision-empty.patch b/free/ci/test/security/testdata/regressions/23-lease-checkout-revision-empty.patch index 7c31d778e..0f5f2a41e 100644 --- a/free/ci/test/security/testdata/regressions/23-lease-checkout-revision-empty.patch +++ b/free/ci/test/security/testdata/regressions/23-lease-checkout-revision-empty.patch @@ -1,13 +1,13 @@ diff --git a/internal/dispatch/deliver.go b/internal/dispatch/deliver.go -index 05a9e265..2e67b17c 100644 +index 7fb81fba..d896a156 100644 --- a/internal/dispatch/deliver.go +++ b/internal/dispatch/deliver.go -@@ -54,7 +54,7 @@ func (d *Dispatcher) knownTransport(r sched.Runner) bool { - // deliver starts the carrier for a committed remote lease; the caller has +@@ -55,7 +55,7 @@ func (d *Dispatcher) knownTransport(r sched.Runner) bool { // already counted it in d.wg. - func (d *Dispatcher) deliver(q Queued, r sched.Runner, a Admitted) { -- delivery := Delivery{Node: r.Capability, Job: q.Job.Model, AttemptID: a.AttemptID, LeaseID: a.LeaseID, Token: a.Token, Epoch: a.Epoch, Revision: q.Job.Revision, ExpiresAt: d.cfg.Clock().Add(d.cfg.LeaseTTL), Hosted: r.Location == "hosted" || r.Ephemeral} -+ delivery := Delivery{Node: r.Capability, Job: q.Job.Model, AttemptID: a.AttemptID, LeaseID: a.LeaseID, Token: a.Token, Epoch: a.Epoch, Revision: "", ExpiresAt: d.cfg.Clock().Add(d.cfg.LeaseTTL), Hosted: r.Location == "hosted" || r.Ephemeral} + func (d *Dispatcher) deliver(q Queued, r sched.Runner, a Admitted, pol sched.Policy) { + hosted := sched.RunnerHosted(r) +- delivery := Delivery{Node: r.Capability, Job: raiseIsolation(q.Job.Model, requiredIsolation(q, r, pol)), AttemptID: a.AttemptID, LeaseID: a.LeaseID, Token: a.Token, Epoch: a.Epoch, Revision: q.Job.Revision, ExpiresAt: d.cfg.Clock().Add(d.cfg.LeaseTTL), Hosted: hosted} ++ delivery := Delivery{Node: r.Capability, Job: raiseIsolation(q.Job.Model, requiredIsolation(q, r, pol)), AttemptID: a.AttemptID, LeaseID: a.LeaseID, Token: a.Token, Epoch: a.Epoch, Revision: "", ExpiresAt: d.cfg.Clock().Add(d.cfg.LeaseTTL), Hosted: hosted} go func() { // admit registered the delivery with the WaitGroup defer d.wg.Done() if err := d.cfg.Transport.Deliver(d.life, delivery); err != nil && d.life.Err() == nil { From 6f5d9d3bad132efc4655842c1c9db192817a2693 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:21:30 -0400 Subject: [PATCH 04/27] fix(ci): --trust owner for a head in this repository in ci-scenarios and the isolation drill (E625 on PR merge refs) --- .github/workflows/ci-isolation-linux.yml | 3 +++ .github/workflows/ci-scenarios.yml | 8 ++++++-- scripts/drill/isolation-levels/run | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-isolation-linux.yml b/.github/workflows/ci-isolation-linux.yml index abf0b9c7b..0fcc99763 100644 --- a/.github/workflows/ci-isolation-linux.yml +++ b/.github/workflows/ci-isolation-linux.yml @@ -84,6 +84,9 @@ jobs: PKGS: ${{ inputs.packages }} TAGS: ${{ inputs.tags }} INSTALL_RUNSC: ${{ inputs.install_runsc }} + # A pull request checkout is a merge ref no branch contains: the built binary classes it untrusted (E625). + # A head in this repository is the owner's own code on a disposable hosted runner; a fork head leaves this empty. + NSELF_CI_TRUST: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'owner' || '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/ci-scenarios.yml b/.github/workflows/ci-scenarios.yml index bb5bc2295..7ebe80e71 100644 --- a/.github/workflows/ci-scenarios.yml +++ b/.github/workflows/ci-scenarios.yml @@ -39,6 +39,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-14] + env: + # A pull request checkout is a merge ref no branch contains: the built binary classes it untrusted (E625). + # A head in this repository is the owner's own code on a disposable hosted runner; a fork head leaves this empty. + NSELF_CI_TRUST: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'owner' || '' }} steps: - name: Check out the repository (the pull request head, not the merge commit) uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -80,7 +84,7 @@ jobs: run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/evidence" - .drill-bin/nself-ci --no-status --pipeline drills --job drill-scenario-12-external-consumer --json > "$RUNNER_TEMP/evidence/scenario-12.json" || true # exit 1 is the partial selection; the job check below decides + .drill-bin/nself-ci --no-status ${NSELF_CI_TRUST:+--trust "$NSELF_CI_TRUST"} --pipeline drills --job drill-scenario-12-external-consumer --json > "$RUNNER_TEMP/evidence/scenario-12.json" || true # exit 1 is the partial selection; the job check below decides # A single-job run is a partial selection, so its overall result is never pass: read the job's own check. jq -r '"scenario-12: " + ([.data.checks[] | select(.id == "drill-scenario-12-external-consumer") | .result][0] // "no result")' "$RUNNER_TEMP/evidence/scenario-12.json" | tee -a "$GITHUB_STEP_SUMMARY" jq -e '[.data.checks[] | select(.id == "drill-scenario-12-external-consumer") | .result] == ["pass"]' "$RUNNER_TEMP/evidence/scenario-12.json" >/dev/null @@ -90,7 +94,7 @@ jobs: run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/evidence" - .drill-bin/nself-ci --no-status --pipeline drills --job drill-scenario-01-local-only --json > "$RUNNER_TEMP/evidence/scenario-01.json" || true # exit 1 is the partial selection; the job check below decides + .drill-bin/nself-ci --no-status ${NSELF_CI_TRUST:+--trust "$NSELF_CI_TRUST"} --pipeline drills --job drill-scenario-01-local-only --json > "$RUNNER_TEMP/evidence/scenario-01.json" || true # exit 1 is the partial selection; the job check below decides # A single-job run is a partial selection, so its overall result is never pass: read the job's own check. jq -r '"scenario-01: " + ([.data.checks[] | select(.id == "drill-scenario-01-local-only") | .result][0] // "no result")' "$RUNNER_TEMP/evidence/scenario-01.json" | tee -a "$GITHUB_STEP_SUMMARY" jq -e '[.data.checks[] | select(.id == "drill-scenario-01-local-only") | .result] == ["pass"]' "$RUNNER_TEMP/evidence/scenario-01.json" >/dev/null diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index 75d936c48..883bb7fc9 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -8,7 +8,8 @@ # (go test -tags integration | tart); a prerequisite the host lacks makes that level's check `error` with # "reason: prerequisite.missing" in its excerpt, which never counts as a pass. A level that does not exist on this # host class gets no check at all. -# Inputs: NSELF_CI_BIN (default nself-ci on PATH) and NSELF_CI_HOME (the run store) for a direct run; ISOLATION_DRILL_LEVELS +# Inputs: NSELF_CI_BIN (default nself-ci on PATH), NSELF_CI_HOME (the run store) and NSELF_CI_TRUST (passed as --trust, for a +# pull request checkout on a CI runner) for a direct run; ISOLATION_DRILL_LEVELS # (comma list, narrows the levels); TRUST04_TEST_IMAGE (default alpine:3.19); for vm NSELF_TEST_TART_IMAGE, # NSELF_TEST_TART_SSH_KEY and optionally NSELF_TEST_LAN_ADDR, NSELF_TEST_EXCLUDED_ADDR. # Outputs: inside a drills job the document is the last stdout line and the exit code is 0 only when no check failed or @@ -25,7 +26,7 @@ if [ -z "${NSELF_CI_RUN_ID:-}" ]; then # not inside a job: record through nself- bin=${NSELF_CI_BIN:-nself-ci} command -v "$bin" >/dev/null 2>&1 || { say "$bin not found; set NSELF_CI_BIN to the nself-ci binary"; exit 2; } cd "$root" || exit 2 - NSELF_V15=1 NSELF_CI_REPO=$root exec "$bin" --no-status --pipeline drills --job "$job" + NSELF_V15=1 NSELF_CI_REPO=$root exec "$bin" --no-status ${NSELF_CI_TRUST:+--trust "$NSELF_CI_TRUST"} --pipeline drills --job "$job" fi for tool in jq go; do From f3a9fbbcd21ec2cbba34a27c8084ccd6fea2dbdd Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:22:08 -0400 Subject: [PATCH 05/27] test(ci): resolve the macOS tempdir symlink in the node prepare tests (E612) --- free/ci/internal/gates/node_prepare_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/free/ci/internal/gates/node_prepare_test.go b/free/ci/internal/gates/node_prepare_test.go index e374b4690..86d50ca42 100644 --- a/free/ci/internal/gates/node_prepare_test.go +++ b/free/ci/internal/gates/node_prepare_test.go @@ -37,7 +37,12 @@ func newNodeHost(t *testing.T) *nodeHost { if runtime.GOOS == "windows" { t.Skip("POSIX shells only") } - h := &nodeHost{t: t, root: t.TempDir(), bin: t.TempDir()} + // The executor refuses a workdir whose resolved path differs (E612), and macOS TMPDIR sits behind /var -> /private/var. + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + h := &nodeHost{t: t, root: root, bin: t.TempDir()} for _, tool := range []string{"sh", "ps", "tail", "head", "cat", "mkdir", "sed", "grep", "cut", "wc", "rm", "basename", "awk", "touch", "ln"} { p, err := exec.LookPath(tool) if err != nil { From d0d062990d5fedbe9bf862dbd12ad51ac28887a4 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:26:11 -0400 Subject: [PATCH 06/27] fix(ci): drills-nightly pins nself-ci 1.2.13 (policy protect --branch) and the newest Go 1.26; isolation drill excerpt names the failing assertion --- .github/workflows/ci-scenarios.yml | 3 ++- .github/workflows/drills-nightly.yml | 18 +++++++++++------- scripts/drill/isolation-levels/run | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-scenarios.yml b/.github/workflows/ci-scenarios.yml index 7ebe80e71..9e1176f85 100644 --- a/.github/workflows/ci-scenarios.yml +++ b/.github/workflows/ci-scenarios.yml @@ -62,7 +62,8 @@ jobs: - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version-file: free/ci/go.mod + go-version: "1.26" # newest 1.26: the cli checkout below needs more than free/ci/go.mod names + check-latest: true cache: false - name: Build the ci plugin from this commit and the cli from main diff --git a/.github/workflows/drills-nightly.yml b/.github/workflows/drills-nightly.yml index f8ddc08c8..e9da99a86 100644 --- a/.github/workflows/drills-nightly.yml +++ b/.github/workflows/drills-nightly.yml @@ -9,8 +9,9 @@ # stops firing shows as a missing run here, not as an `overdue` report. # The pin: CI_VERSION, CI_RELEASE_URL and CI_RELEASE_SHA256 take the `ci-release` handoff (tag plus sha256 of each # platform tarball) of P7-PLUG-69, the first plugins release that holds `--pipeline` and drill jobs (Epic P7-CI D34). -# Moving the pin means changing these three values here, in nself-ci.yml and in nself-ci-source.yml in one commit. If a -# value is ever empty again the first step fails with a clear message instead of running an unpinned binary. +# This pin moves on its own: the drill scripts come from the checkout, so they need a binary that has the flags they call +# (1.2.13 for main-status-audit); nself-ci.yml and nself-ci-source.yml keep their own pin. If a value is ever empty again +# the first step fails with a clear message instead of running an unpinned binary. # See also scripts/drill/README.md and .github/wiki/Catalog.md (Freshness drill). name: drills-nightly @@ -31,9 +32,9 @@ concurrency: env: PIPELINE: drills # the pipeline name in .nself/ci.yaml - CI_VERSION: "1.2.11" # the pinned release (shown in the summary); same pin as nself-ci.yml - CI_RELEASE_URL: "https://github.com/nself-org/plugins/releases/download/v1.2.11/ci-1.2.11-linux-amd64.tar.gz" - CI_RELEASE_SHA256: "0ca65f839b007764f70ba9e8e719aee88f336efb74b932297b2b9097cbdd0016" + CI_VERSION: "1.2.13" # the pinned release (shown in the summary); main-status-audit needs the policy protect --branch flag, new in 1.2.13 + CI_RELEASE_URL: "https://github.com/nself-org/plugins/releases/download/v1.2.13/ci-1.2.13-linux-amd64.tar.gz" + CI_RELEASE_SHA256: "2b2d24a17937cecbe3c1d4ee2e59f1c488fe2a7ea6e3e60bd5c814c56454272e" NSELF_V15: "1" # the plugin binary does not follow the cli flip; kept until v1.6 GVISOR_RELEASE: "20261012.0" # same pin and checksum as ci-isolation-linux.yml GVISOR_SHA512: "9606445eed5aedfcf05310e7ccef61d994089914c6622fc82624c24d2b18db1b2210f40e40a7be27b8b7772859de7d95c501b4eba66423053c960bf671377c5e" @@ -62,10 +63,13 @@ jobs: # The drills run Go (security-suite, truth-sweep, scenarios), docker and podman (isolation-levels) and # gVisor's runsc registered with docker; the steps are the ones ci-isolation-linux.yml runs. + # The scenario drills build nself-org/cli main, whose go.mod needs a newer Go than free/ci's (GOTOOLCHAIN=local in the + # engine's job environment), so take the newest 1.26. - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version-file: free/ci/go.mod - cache-dependency-path: free/ci/go.sum + go-version: "1.26" + check-latest: true + cache: false - name: Container runtimes run: | diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index 883bb7fc9..c584322db 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -55,6 +55,7 @@ missing() { record "$1" error "reason: prerequisite.missing - $2"; } snip() { { grep -E '^(--- FAIL|FAIL|panic|fatal error)' "$1" 2>/dev/null | head -n 4 | cut -c1-160 + grep -E '^[[:space:]]+[A-Za-z0-9_]+_test\.go:[0-9]+:' "$1" 2>/dev/null | head -n 2 | cut -c1-200 grep -E 'prerequisite missing|required' "$1" 2>/dev/null | head -n 1 | cut -c1-160 tail -n 2 "$1" 2>/dev/null | cut -c1-120 } | tr '\n' ' ' From 57e63ad1c0a3bc048544ce7a0b0b42ae6ba279c8 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:28:51 -0400 Subject: [PATCH 07/27] fix(ci): a manual drills-nightly run is the owner's own run (--trust owner on dispatch) --- .github/workflows/drills-nightly.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/drills-nightly.yml b/.github/workflows/drills-nightly.yml index e9da99a86..2c5f3643b 100644 --- a/.github/workflows/drills-nightly.yml +++ b/.github/workflows/drills-nightly.yml @@ -119,11 +119,17 @@ jobs: env: GH_TOKEN: ${{ github.token }} # read-only; the drills that call gh (main-status-audit, truth-sweep) take it from here NSELF_CI_HOME: ${{ runner.temp }}/ci-home # a known run store, so the job logs can be kept below + # The schedule runs on the default branch, which the binary classes owner. A manual run from another branch is a + # merge ref no default-branch commit contains and would be refused (E625); whoever can dispatch this workflow can + # also edit it, so a dispatch is the owner's own run. + NSELF_CI_TRUST: ${{ github.event_name == 'workflow_dispatch' && 'owner' || '' }} run: | set -o pipefail mkdir -p "$RUNNER_TEMP/evidence" status=0 - "$NSELF_CI_BIN" --no-status --pipeline "$PIPELINE" --json > "$RUNNER_TEMP/evidence/$PIPELINE.json" || status=$? + args=(--no-status --pipeline "$PIPELINE" --json) + if [ -n "$NSELF_CI_TRUST" ]; then args+=(--trust "$NSELF_CI_TRUST"); fi + "$NSELF_CI_BIN" "${args[@]}" > "$RUNNER_TEMP/evidence/$PIPELINE.json" || status=$? { echo "### $PIPELINE (nself-ci $CI_VERSION)" echo "exit code: $status" From 0c06b0f70a7061f37c337cd907fb337cfab4c2a7 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:54:27 -0400 Subject: [PATCH 08/27] fix(ci): give drill-security-suite a 60 minute timeout; hand gh a read-only token through its config file --- .github/workflows/drills-nightly.yml | 12 +++++++++++- .nself/ci.yaml | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/drills-nightly.yml b/.github/workflows/drills-nightly.yml index 2c5f3643b..bce4bf0ba 100644 --- a/.github/workflows/drills-nightly.yml +++ b/.github/workflows/drills-nightly.yml @@ -115,9 +115,19 @@ jobs: - name: Install nft (scenario 1 cuts the network) run: command -v nft >/dev/null || (sudo -n apt-get update -qq && sudo -n apt-get install -y -qq nftables) + # The executor hands a job HOME and XDG_* but no token variable, so the drills that call gh (main-status-audit, + # truth-sweep) read their credential from gh's own config file. The token is the job's read-only github.token. + - name: Give gh a read-only token + env: + GH_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$HOME/.config/gh" + chmod 700 "$HOME/.config/gh" + ( umask 077; printf 'github.com:\n oauth_token: %s\n git_protocol: https\n' "$GH_READ_TOKEN" > "$HOME/.config/gh/hosts.yml" ) + - name: Run the pipeline env: - GH_TOKEN: ${{ github.token }} # read-only; the drills that call gh (main-status-audit, truth-sweep) take it from here NSELF_CI_HOME: ${{ runner.temp }}/ci-home # a known run store, so the job logs can be kept below # The schedule runs on the default branch, which the binary classes owner. A manual run from another branch is a # merge ref no default-branch commit contains and would be refused (E625); whoever can dispatch this workflow can diff --git a/.nself/ci.yaml b/.nself/ci.yaml index 51350435b..dfcf6c0fc 100644 --- a/.nself/ci.yaml +++ b/.nself/ci.yaml @@ -138,6 +138,12 @@ jobs: kind: drill run: [./scripts/drill/scenario-12-external-consumer/run] timeout: 30m + # The security suite runs the whole acceptance suite and then plants each regression in a copy of the module (about + # 40 s each, three copies at a time): it needs far more than the 10 minute default. + drill-security-suite: + kind: drill + run: [./scripts/drill/security-suite/run] + timeout: 60m # Per-check configuration. `truth.rules` lists the rule files of the truth check; its # presence is what plans the check as a job when none declares it (ImpliedBy). From 9ea9cb4cac1285eaeea9c4c1c0b84cdfe30686c8 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 09:54:55 -0400 Subject: [PATCH 09/27] test(ci): resolve the executor home tempdir too (E612 on macOS) --- free/ci/internal/gates/node_prepare_test.go | 22 +++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/free/ci/internal/gates/node_prepare_test.go b/free/ci/internal/gates/node_prepare_test.go index 86d50ca42..37df48a3b 100644 --- a/free/ci/internal/gates/node_prepare_test.go +++ b/free/ci/internal/gates/node_prepare_test.go @@ -37,12 +37,7 @@ func newNodeHost(t *testing.T) *nodeHost { if runtime.GOOS == "windows" { t.Skip("POSIX shells only") } - // The executor refuses a workdir whose resolved path differs (E612), and macOS TMPDIR sits behind /var -> /private/var. - root, err := filepath.EvalSymlinks(t.TempDir()) - if err != nil { - t.Fatal(err) - } - h := &nodeHost{t: t, root: root, bin: t.TempDir()} + h := &nodeHost{t: t, root: realTempDir(t), bin: t.TempDir()} for _, tool := range []string{"sh", "ps", "tail", "head", "cat", "mkdir", "sed", "grep", "cut", "wc", "rm", "basename", "awk", "touch", "ln"} { p, err := exec.LookPath(tool) if err != nil { @@ -107,7 +102,7 @@ func (h *nodeHost) run(id, workdir string) result { p := h.plan(id, workdir) var out strings.Builder res := (&ciexec.Executor{}).Run(context.Background(), ciexec.JobSpec{AttemptID: "node-" + strings.ReplaceAll(id, ".", "-"), CoordinatorID: "coord", - Job: model.Job{Kind: "quality", Workdir: workdir, Isolation: "process"}, Command: p.Command(), SourceDir: h.root, Home: h.t.TempDir(), + Job: model.Job{Kind: "quality", Workdir: workdir, Isolation: "process"}, Command: p.Command(), SourceDir: h.root, Home: realTempDir(h.t), Output: func(b []byte) { out.Write(b) }}) r := result{exit: res.ExitCode, out: strings.TrimSpace(out.String())} if res.Err != nil { @@ -396,7 +391,7 @@ func TestMissingManagerE614(t *testing.T) { } // the standalone runner agrees: required and missing is error/infra with E614, never a failure d, _ := DefaultRegistry().Get("node.test") - c := (Runner{Executor: &ciexec.Executor{}, Home: t.TempDir()}).Run(context.Background(), h.root, PlannedCheck{Def: d, Required: true, Workdir: "."}, model.PipelineConfig{}) + c := (Runner{Executor: &ciexec.Executor{}, Home: realTempDir(t)}).Run(context.Background(), h.root, PlannedCheck{Def: d, Required: true, Workdir: "."}, model.PipelineConfig{}) if c.Result != "error" || c.FailureClass != "infra" || !strings.Contains(c.Excerpt, "E614") { t.Fatalf("%+v", c) } @@ -598,3 +593,14 @@ esac t.Fatalf("the pnpm store must survive for the check: %+v", r) } } + +// realTempDir is t.TempDir with symlinks resolved: the executor refuses a workdir or home whose resolved path differs +// (E612), and macOS TMPDIR sits behind /var -> /private/var. +func realTempDir(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} From b4f6776fa84d4678b925e94fd7c51c7d4dbe9fad Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 10:15:49 -0400 Subject: [PATCH 10/27] fix(ci): isolation drill excerpt carries the lines before a failing test; dispatch of ci-isolation-linux is the owner's run --- .github/workflows/ci-isolation-linux.yml | 2 +- scripts/drill/isolation-levels/run | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-isolation-linux.yml b/.github/workflows/ci-isolation-linux.yml index 0fcc99763..b6b9e5a77 100644 --- a/.github/workflows/ci-isolation-linux.yml +++ b/.github/workflows/ci-isolation-linux.yml @@ -86,7 +86,7 @@ jobs: INSTALL_RUNSC: ${{ inputs.install_runsc }} # A pull request checkout is a merge ref no branch contains: the built binary classes it untrusted (E625). # A head in this repository is the owner's own code on a disposable hosted runner; a fork head leaves this empty. - NSELF_CI_TRUST: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'owner' || '' }} + NSELF_CI_TRUST: ${{ (github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) && 'owner' || '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index c584322db..0cdc00986 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -55,7 +55,7 @@ missing() { record "$1" error "reason: prerequisite.missing - $2"; } snip() { { grep -E '^(--- FAIL|FAIL|panic|fatal error)' "$1" 2>/dev/null | head -n 4 | cut -c1-160 - grep -E '^[[:space:]]+[A-Za-z0-9_]+_test\.go:[0-9]+:' "$1" 2>/dev/null | head -n 2 | cut -c1-200 + grep -B8 -E '^--- FAIL' "$1" 2>/dev/null | grep -vE '^(=== |--- |--$)' | head -n 8 | cut -c1-200 grep -E 'prerequisite missing|required' "$1" 2>/dev/null | head -n 1 | cut -c1-160 tail -n 2 "$1" 2>/dev/null | cut -c1-120 } | tr '\n' ' ' From 5173de5241ea31b01734858e13fa0c3b21ca5557 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 10:20:42 -0400 Subject: [PATCH 11/27] fix(ci): main-status-audit audits one day (the seven day window exceeds its 200 commit read) --- .nself/ci.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.nself/ci.yaml b/.nself/ci.yaml index dfcf6c0fc..d477ee249 100644 --- a/.nself/ci.yaml +++ b/.nself/ci.yaml @@ -138,6 +138,14 @@ jobs: kind: drill run: [./scripts/drill/scenario-12-external-consumer/run] timeout: 30m + # The default seven day window holds more than the 200 commits the audit reads on this busy repository (it then + # refuses to answer); the drill runs daily, so one day covers every commit since the last run. + drill-main-status-audit: + kind: drill + run: [./scripts/drill/main-status-audit/run] + env: + AUDIT_DAYS: "1" + timeout: 20m # The security suite runs the whole acceptance suite and then plants each regression in a copy of the module (about # 40 s each, three copies at a time): it needs far more than the 10 minute default. drill-security-suite: From 7ce387b74012bee38442a400cb2e0122d4e75403 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 10:52:13 -0400 Subject: [PATCH 12/27] fix(ci): isolation drill tests run with TMPDIR=/tmp (the executor's private TMPDIR blocks job users) --- scripts/drill/isolation-levels/run | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index 0cdc00986..b1e18017a 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -101,7 +101,9 @@ leg() { why=$(prereq "$level") if [ -n "$why" ]; then missing "$level" "$why"; return; fi out=$work/$level.out - (cd "$root/free/ci" && CGO_ENABLED=0 TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 + # TMPDIR=/tmp: the executor gives a job a private 0700 TMPDIR, and the hygiene test makes job users + # (other uids) walk into a directory below it; they cannot cross a private parent. + (cd "$root/free/ci" && CGO_ENABLED=0 TMPDIR=/tmp TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 rc=$? if [ $rc -ne 0 ]; then record "$level" fail "go test -tags $tags $pkg exited $rc: $(snip "$out")" From 11020f53652c849ccf5feb557ac4633f31b3535f Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 11:19:14 -0400 Subject: [PATCH 13/27] ci: keep the isolation drill's full test output as an artifact --- .github/workflows/drills-nightly.yml | 9 +++++++++ scripts/drill/isolation-levels/run | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/drills-nightly.yml b/.github/workflows/drills-nightly.yml index bce4bf0ba..1efab6ca1 100644 --- a/.github/workflows/drills-nightly.yml +++ b/.github/workflows/drills-nightly.yml @@ -146,6 +146,15 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" exit "$status" + - name: Keep the drill logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: drill-logs-${{ github.run_id }} + path: /tmp/drill-logs/ + if-no-files-found: ignore + retention-days: 14 + - name: Keep the evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index b1e18017a..90d5c9840 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -106,6 +106,8 @@ leg() { (cd "$root/free/ci" && CGO_ENABLED=0 TMPDIR=/tmp TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 rc=$? if [ $rc -ne 0 ]; then + # keep the whole test output where a CI workflow can upload it (the excerpt below is one line) + mkdir -p /tmp/drill-logs && cp "$out" "/tmp/drill-logs/isolation-$level.out" 2>/dev/null || true record "$level" fail "go test -tags $tags $pkg exited $rc: $(snip "$out")" elif grep -q -- '--- SKIP' "$out"; then record "$level" fail "a test skipped; a missing prerequisite must fail, never skip: $(grep -m1 -- '--- SKIP' "$out" | cut -c1-120)" From 4242ebaa9900b5a9af7a2ff7913ba315648a0f04 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 11:31:18 -0400 Subject: [PATCH 14/27] ci(scenarios): remove leftover containers before scenario 1 on the throwaway runner --- .github/workflows/ci-scenarios.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci-scenarios.yml b/.github/workflows/ci-scenarios.yml index 9e1176f85..edb8e9a98 100644 --- a/.github/workflows/ci-scenarios.yml +++ b/.github/workflows/ci-scenarios.yml @@ -90,6 +90,14 @@ jobs: jq -r '"scenario-12: " + ([.data.checks[] | select(.id == "drill-scenario-12-external-consumer") | .result][0] // "no result")' "$RUNNER_TEMP/evidence/scenario-12.json" | tee -a "$GITHUB_STEP_SUMMARY" jq -e '[.data.checks[] | select(.id == "drill-scenario-12-external-consumer") | .result] == ["pass"]' "$RUNNER_TEMP/evidence/scenario-12.json" >/dev/null + # Scenario 12 can leave containers behind; scenario 1 refuses to cut the network of running containers. The runner is + # a throwaway VM that holds nothing else, so remove them. + - name: Remove leftover containers (Linux) + if: runner.os == 'Linux' && !cancelled() + run: | + ids=$(docker ps -aq) + if [ -n "$ids" ]; then docker ps -a --format '{{.ID}} {{.Image}} {{.Status}}'; docker rm -f $ids; fi + - name: Scenario 1, local only (network cut) if: ${{ !cancelled() }} run: | From 79539f948a61d5c3d395df40c2a6b1ae665df826 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 11:49:21 -0400 Subject: [PATCH 15/27] fix(ci): the hygiene leg tests a world-readable copy of free/ci (job users cannot enter the private run store) --- scripts/drill/isolation-levels/run | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index 90d5c9840..77fd0d0d1 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -103,7 +103,15 @@ leg() { out=$work/$level.out # TMPDIR=/tmp: the executor gives a job a private 0700 TMPDIR, and the hygiene test makes job users # (other uids) walk into a directory below it; they cannot cross a private parent. - (cd "$root/free/ci" && CGO_ENABLED=0 TMPDIR=/tmp TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 + # The hygiene test runs commands as job users (other uids) with the package directory as their working directory, + # so the source tree must be traversable by them: inside a drills job it sits in the runner's private run store. + # Test from a world-readable copy. + src=$root/free/ci + if [ "$level" = hygiene ] && [ -d "$src" ]; then + copy=/tmp/drill-iso/ci + rm -rf /tmp/drill-iso && mkdir -p /tmp/drill-iso && cp -R "$src" "$copy" && chmod -R a+rX /tmp/drill-iso && chmod a+rx /tmp/drill-iso && src=$copy + fi + (cd "$src" && CGO_ENABLED=0 TMPDIR=/tmp TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 rc=$? if [ $rc -ne 0 ]; then # keep the whole test output where a CI workflow can upload it (the excerpt below is one line) From acb763e506992266e5c0c5080cfaf9b0e5213f3f Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 12:26:41 -0400 Subject: [PATCH 16/27] test(ci): role grants are not lease grants for the admit chokepoint scan; the android planner test ignores the runner's ANDROID_SDK_ROOT --- free/ci/cmd/build_v15_test.go | 1 + free/ci/internal/dispatch/chokepoint_test.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/free/ci/cmd/build_v15_test.go b/free/ci/cmd/build_v15_test.go index 38d22f099..fa3e27038 100644 --- a/free/ci/cmd/build_v15_test.go +++ b/free/ci/cmd/build_v15_test.go @@ -73,6 +73,7 @@ const requiredUnit = "version: 1\njobs:\n unit:\n kind: test\n run: [sh, func TestBuildV15Synthesized(t *testing.T) { repo := androidRepo(t, requiredUnit) t.Setenv("ANDROID_HOME", "/opt/sdk-fixture") + t.Setenv("ANDROID_SDK_ROOT", "") // GitHub runners set it; the job passes it through, so the count below would be 2 t.Setenv("JAVA_HOME", "") lane, err := planBuildLane(context.Background(), repo, "android", 15*time.Minute) if err != nil { diff --git a/free/ci/internal/dispatch/chokepoint_test.go b/free/ci/internal/dispatch/chokepoint_test.go index bbc813f02..70d164349 100644 --- a/free/ci/internal/dispatch/chokepoint_test.go +++ b/free/ci/internal/dispatch/chokepoint_test.go @@ -42,6 +42,13 @@ var ( "internal/storetest.Admit": true, // test fixtures only; TestStoretestIsTestOnly keeps it out of shipped code } localFiles = map[string]bool{"internal/run/attempt.go": true, "internal/run/dispatch_hook.go": true} + // roleGrantFiles name a TEAM role grant (P7-TRUST-12): "Grant" there is a permission row of a person, not a lease. + // The exemption is for the identifier Grant only; every other admission function stays banned in these files. + roleGrantFiles = map[string]bool{ + "internal/roles/grants.go": true, + "internal/store/trust_roles.go": true, + "cmd/serve_admin_roles.go": true, + } ) // banned are the admission functions only the chokepoint may name. @@ -108,11 +115,11 @@ func scanFile(f *ast.File, rel string, add func(string)) { if !localFiles[rel] { report(name) } - case banned[name] && !chokeFuncs[key]: + case banned[name] && !chokeFuncs[key] && !(name == "Grant" && roleGrantFiles[rel]): report(name) } case *ast.Ident: - if banned[x.Name] && x != declName && !(x.Name == "LocalAdmit" && localFiles[rel]) && (x.Name == "LocalAdmit" || !chokeFuncs[key]) { + if banned[x.Name] && x != declName && !(x.Name == "Grant" && roleGrantFiles[rel]) && !(x.Name == "LocalAdmit" && localFiles[rel]) && (x.Name == "LocalAdmit" || !chokeFuncs[key]) { report(x.Name) } case *ast.BasicLit: From b7c982aa9a6d13cd08fac22cf373f578b74d66e5 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 12:37:28 -0400 Subject: [PATCH 17/27] fix(ci): a cancelled job's orphans no longer hold the cancel for the whole grace period on Linux The executor waits out the grace period after the job leader exits while any remembered descendant is alive. On Linux an orphan is adopted by the agent (subreaper) and stays a zombie until reaped, and a signal-0 probe succeeds on a zombie, so every cancel of a job that had a background child waited the full 10 s grace before reporting its result, past the coordinator's 5 s cancel timeout. Alive now means not a zombie, the orphans are reaped while waiting, and the wait ends as soon as the group is gone. The runs follow tests also stop depending on the host's container runtimes: reading a run recovers after dead coordinators, which runs docker info and ps, about a second on a CI runner with a live daemon. --- free/ci/cmd/runs_wait_test.go | 20 +++++++++++++++++++ free/ci/internal/exec/process.go | 10 +++++++--- .../internal/exec/process_subreaper_darwin.go | 1 + free/ci/internal/exec/process_table_linux.go | 9 +++++++++ free/ci/internal/exec/process_tree_unix.go | 10 +++++++++- 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/free/ci/cmd/runs_wait_test.go b/free/ci/cmd/runs_wait_test.go index 24801d60d..2f2b21ea7 100644 --- a/free/ci/cmd/runs_wait_test.go +++ b/free/ci/cmd/runs_wait_test.go @@ -10,6 +10,7 @@ import ( "encoding/json" "os" "path/filepath" + "runtime" "testing" "time" @@ -17,10 +18,29 @@ import ( "github.com/nself-org/plugins/free/ci/internal/store" ) +// noContainerRuntime puts a docker and a podman that refuse every command first on PATH. Reading a run +// recovers after dead coordinators, which asks each installed container runtime for orphans (`info`, `ps`); +// a host with a live daemon (a CI runner) spends about a second on that, and these tests are about the +// run's stream, not the host's runtimes. +func noContainerRuntime(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + return + } + dir := t.TempDir() + for _, name := range []string{"docker", "podman"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + // runsHome points NSELF_CI_HOME at a fresh directory and speeds the poll up. func runsHome(t *testing.T) string { t.Helper() home := t.TempDir() + noContainerRuntime(t) t.Setenv("NSELF_CI_HOME", home) t.Setenv("NSELF_V15", "1") t.Setenv("NSELF_INVOKE_DEADLINE_MS", "") diff --git a/free/ci/internal/exec/process.go b/free/ci/internal/exec/process.go index df8ffefa5..70f5750c2 100644 --- a/free/ci/internal/exec/process.go +++ b/free/ci/internal/exec/process.go @@ -135,10 +135,14 @@ func (e *Executor) runCommandFiles(ctx context.Context, stdin io.Reader, extra [ } timer.Stop() tracker.snapshot() - if leaderDone && (groupAlive(cmd.Process.Pid) || tracker.anyAlive()) { - if remain := time.Until(deadline); remain > 0 { - time.Sleep(remain) + // The leader is gone: give the rest of the group the grace period to follow it, and stop waiting as + // soon as none is left (an exited orphan is reaped here so it does not count as alive). + for leaderDone && time.Now().Before(deadline) { + tracker.reap() + if !groupAlive(cmd.Process.Pid) && !tracker.anyAlive() { + break } + time.Sleep(10 * time.Millisecond) } _ = killGroup(cmd.Process.Pid) tracker.signalKill() diff --git a/free/ci/internal/exec/process_subreaper_darwin.go b/free/ci/internal/exec/process_subreaper_darwin.go index ab610ebf3..b157e23fe 100644 --- a/free/ci/internal/exec/process_subreaper_darwin.go +++ b/free/ci/internal/exec/process_subreaper_darwin.go @@ -7,4 +7,5 @@ import "syscall" func ensureSubreaper() error { return nil } func reapDescendants([]int) {} func groupAlive(pid int) bool { return syscall.Kill(-pid, 0) == nil } +func processAlive(pid int) bool { return syscall.Kill(pid, 0) == nil } func adoptedDescendants(map[int]int, string) []int { return nil } diff --git a/free/ci/internal/exec/process_table_linux.go b/free/ci/internal/exec/process_table_linux.go index 71916a7d1..0b99a2b40 100644 --- a/free/ci/internal/exec/process_table_linux.go +++ b/free/ci/internal/exec/process_table_linux.go @@ -138,3 +138,12 @@ func groupAlive(pgid int) bool { } return false } + +// processAlive reports whether pid is a process that has not exited. A zombie (exited, not yet +// reaped by its parent) is not alive although a signal-0 probe still succeeds on it. +func processAlive(pid int) bool { + if st, ok := readProcStat(pid); ok { + return st.state != 'Z' + } + return processExists(pid) +} diff --git a/free/ci/internal/exec/process_tree_unix.go b/free/ci/internal/exec/process_tree_unix.go index eeb2a1c52..4e60b11dc 100644 --- a/free/ci/internal/exec/process_tree_unix.go +++ b/free/ci/internal/exec/process_tree_unix.go @@ -137,14 +137,22 @@ func (t *processTree) pids() []int { func (t *processTree) signalTerminate() { t.signal(syscall.SIGTERM) } func (t *processTree) signalKill() { t.signal(syscall.SIGKILL) } + +// anyAlive says whether a remembered process still runs. A zombie does not: on Linux an orphan is +// adopted by this process (the subreaper) and stays a zombie until reaped, and a signal-0 probe +// succeeds on it. func (t *processTree) anyAlive() bool { for _, pid := range t.pids() { - if syscall.Kill(pid, 0) == nil { + if processAlive(pid) { return true } } return false } + +// reap collects the remembered processes that have exited, so none lingers as a zombie. +func (t *processTree) reap() { reapDescendants(t.pids()) } + func (t *processTree) signal(sig syscall.Signal) { for _, pid := range t.pids() { _ = syscall.Kill(pid, sig) From 84b59ca0f6aa05cc5fc4c3c2f542cde131fe4367 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 12:48:54 -0400 Subject: [PATCH 18/27] ci(drill): name the running containers when the network cut refuses --- scripts/drill/fixtures/lib/cut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/drill/fixtures/lib/cut.sh b/scripts/drill/fixtures/lib/cut.sh index b45e67de1..9d1697418 100644 --- a/scripts/drill/fixtures/lib/cut.sh +++ b/scripts/drill/fixtures/lib/cut.sh @@ -49,7 +49,7 @@ cut_preflight() { # The cut drops forwarding out of every docker bridge and the docker daemon's own egress, so it would break other # containers on a shared box. Refuse unless the caller says this host is the drill's own (hosted runner, throwaway VM). if [ -n "$(docker ps -q 2>/dev/null)" ] && [ "${DRILL_CUT_ALLOW_SHARED_HOST:-}" != 1 ]; then - cut_die "containers are running; the cut would cut their network too. Stop them, or set DRILL_CUT_ALLOW_SHARED_HOST=1 on a host that is only for this drill" + cut_die "containers are running ($(docker ps --format "{{.ID}} {{.Image}} {{.Names}} {{.Status}}" 2>/dev/null | tr "\n" ";")); the cut would cut their network too. Stop them, or set DRILL_CUT_ALLOW_SHARED_HOST=1 on a host that is only for this drill" fi ;; Darwin) command -v pfctl >/dev/null || cut_die "pfctl is required" ;; From 93c82a62049fb68abe5e0f801fab1e2dac650ca0 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 13:11:45 -0400 Subject: [PATCH 19/27] fix(ci): log tail window covers Go 1.26's six-byte invalid-byte escape; audited gate process starts join the exec-funnel allowlist --- .../internal/nodes/provision/single_exec_test.go | 4 ++++ free/ci/internal/transport/agent/logwriter.go | 7 ++++--- free/ci/internal/transport/agent/logwriter_test.go | 14 +++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/free/ci/internal/nodes/provision/single_exec_test.go b/free/ci/internal/nodes/provision/single_exec_test.go index a804a0012..936f95bec 100644 --- a/free/ci/internal/nodes/provision/single_exec_test.go +++ b/free/ci/internal/nodes/provision/single_exec_test.go @@ -46,6 +46,10 @@ var dynamicProgram = map[string]bool{ "internal/exec/process.go:runCommand": true, "internal/exec/process.go:runCommandIn": true, "internal/exec/process.go:runCommandSig": true, + "internal/exec/process.go:runCommandFiles": true, // the same gate argv start as runCommand, with extra files (P7-CACHE) + // The orphan sweep removes a stale job container with the runtime binary the sweep resolved + // (docker or podman, checked against runtimeBinary before it is used); never ssh. + "internal/exec/sweep.go:removeStaleContainer": true, // `nself ci --detach` re-executes the plugin binary (os.Executable) in a new // session; the program is the caller's own binary, never ssh (P7-CI-63). "internal/exec/detach.go:StartDetached": true, diff --git a/free/ci/internal/transport/agent/logwriter.go b/free/ci/internal/transport/agent/logwriter.go index f62e46e9a..34c9348ae 100644 --- a/free/ci/internal/transport/agent/logwriter.go +++ b/free/ci/internal/transport/agent/logwriter.go @@ -26,9 +26,10 @@ import ( // tailWindow is how much of the file end OpenLog reads to find the last complete line. // -// A line is at most about three times its frame: the JSON encoder writes U+2028 and U+2029 as six bytes (three in -// the frame) and an invalid byte as the three-byte U+FFFD. -const tailWindow = 4 * protocol.MaxFrame +// A line is at most about six times its frame: the JSON encoder writes U+2028 and U+2029 as six bytes (three in +// the frame) and, up to Go 1.26, an invalid byte as the six-byte escape \ufffd (Go 1.27 writes the three-byte rune). +// Eight frames leave room for the envelope around the worst case. +const tailWindow = 8 * protocol.MaxFrame var unsafeFile = regexp.MustCompile(`[^A-Za-z0-9_.-]`) diff --git a/free/ci/internal/transport/agent/logwriter_test.go b/free/ci/internal/transport/agent/logwriter_test.go index c1641d968..10d5a1527 100644 --- a/free/ci/internal/transport/agent/logwriter_test.go +++ b/free/ci/internal/transport/agent/logwriter_test.go @@ -189,7 +189,7 @@ func writeLines(t *testing.T, path string, n int, textLen int) { func TestLogWriterReadsTheTailOfAFileLargerThanTheWindow(t *testing.T) { path := filepath.Join(t.TempDir(), "big.jsonl") - writeLines(t, path, 5000, 1000) // about 5 MiB, more than the 4 MiB window + writeLines(t, path, 9000, 1000) // about 9 MiB, more than the 8 MiB window if st, _ := os.Stat(path); st.Size() <= tailWindow { t.Fatalf("file is %d bytes, want more than the window", st.Size()) } @@ -197,24 +197,24 @@ func TestLogWriterReadsTheTailOfAFileLargerThanTheWindow(t *testing.T) { if err != nil { t.Fatal(err) } - if w.Last() != 5000 { - t.Fatalf("tail = %d, want 5000", w.Last()) + if w.Last() != 9000 { + t.Fatalf("tail = %d, want 9000", w.Last()) } _ = w.Close() // and with a torn tail beyond the window f, _ := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o600) - _, _ = f.WriteString(`{"line_seq":5001,"stre`) + _, _ = f.WriteString(`{"line_seq":9001,"stre`) _ = f.Close() w, err = OpenLog(path) if err != nil { t.Fatal(err) } defer w.Close() - if w.Last() != 5000 { + if w.Last() != 9000 { t.Fatalf("tail after a torn line = %d", w.Last()) } - if ok, _ := w.Append(logOf(5001, "z")); !ok { - t.Fatal("line 5001 not written") + if ok, _ := w.Append(logOf(9001, "z")); !ok { + t.Fatal("line 9001 not written") } b, _ := os.ReadFile(path) if !strings.HasSuffix(string(b), `"text":"z"}`+"\n") || strings.Contains(string(b), `"stre{`) { From b81829428048f2935fecd2343287ee08b93ba2d2 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 13:53:59 -0400 Subject: [PATCH 20/27] fix(ci): TMPDIR=/tmp only for the hygiene leg of the isolation drill (the network leg passes with the default) --- scripts/drill/isolation-levels/run | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/drill/isolation-levels/run b/scripts/drill/isolation-levels/run index 77fd0d0d1..92047fe16 100755 --- a/scripts/drill/isolation-levels/run +++ b/scripts/drill/isolation-levels/run @@ -101,17 +101,17 @@ leg() { why=$(prereq "$level") if [ -n "$why" ]; then missing "$level" "$why"; return; fi out=$work/$level.out - # TMPDIR=/tmp: the executor gives a job a private 0700 TMPDIR, and the hygiene test makes job users - # (other uids) walk into a directory below it; they cannot cross a private parent. # The hygiene test runs commands as job users (other uids) with the package directory as their working directory, # so the source tree must be traversable by them: inside a drills job it sits in the runner's private run store. # Test from a world-readable copy. src=$root/free/ci + tmpdir=${TMPDIR:-/tmp} if [ "$level" = hygiene ] && [ -d "$src" ]; then + tmpdir=/tmp # the executor's private 0700 TMPDIR cannot be crossed by the job users either copy=/tmp/drill-iso/ci rm -rf /tmp/drill-iso && mkdir -p /tmp/drill-iso && cp -R "$src" "$copy" && chmod -R a+rX /tmp/drill-iso && chmod a+rx /tmp/drill-iso && src=$copy fi - (cd "$src" && CGO_ENABLED=0 TMPDIR=/tmp TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 + (cd "$src" && CGO_ENABLED=0 TMPDIR="$tmpdir" TRUST04_TEST_IMAGE=$image go test -mod=vendor -count=1 -v -timeout 20m -tags "$tags" "$pkg") >"$out" 2>&1 rc=$? if [ $rc -ne 0 ]; then # keep the whole test output where a CI workflow can upload it (the excerpt below is one line) From 6aa84f599f052bf3bdfcbe8971f5719f2e14577c Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 16:05:24 -0400 Subject: [PATCH 21/27] fix(ci): pairing withdraw audits before it wakes the window; TestQuotaStops gets room for a loaded runner; keep the isolation drill output --- .github/workflows/ci-isolation-linux.yml | 9 +++++++++ free/ci/internal/pairing/service.go | 5 ++++- .../ci/internal/providers/github/reconcile/sweep_test.go | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-isolation-linux.yml b/.github/workflows/ci-isolation-linux.yml index b6b9e5a77..fb7bc263e 100644 --- a/.github/workflows/ci-isolation-linux.yml +++ b/.github/workflows/ci-isolation-linux.yml @@ -183,3 +183,12 @@ jobs: for check in isolation.container isolation.sandboxed-container isolation.network; do jq -e --arg c "$check" '.. | objects | select(.id? == $c or .name? == $c) | .verified_at | select(. != null)' "$work/doctor.json" >/dev/null done + + - name: Keep the isolation drill's test output + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: isolation-drill-logs-${{ github.run_id }} + path: /tmp/drill-logs/ + if-no-files-found: ignore + retention-days: 14 diff --git a/free/ci/internal/pairing/service.go b/free/ci/internal/pairing/service.go index 351d8c080..96f64ce9c 100644 --- a/free/ci/internal/pairing/service.go +++ b/free/ci/internal/pairing/service.go @@ -160,8 +160,11 @@ func (s *Service) Withdraw(ctx context.Context, code string) error { return err } } + // The audit row is written before the window is woken: waking it lets `nodes pair` exit and close the store, and an + // append after that fails with "sql: database is closed". + err = s.cfg.Store.AppendAudit(ctx, s.cfg.Actor, "pairing.withdraw", o.ID, "revoked="+boolStr(o.Serial != "")) s.signal() - return s.cfg.Store.AppendAudit(ctx, s.cfg.Actor, "pairing.withdraw", o.ID, "revoked="+boolStr(o.Serial != "")) + return err } func boolStr(b bool) string { diff --git a/free/ci/internal/providers/github/reconcile/sweep_test.go b/free/ci/internal/providers/github/reconcile/sweep_test.go index ffd443710..bec136436 100644 --- a/free/ci/internal/providers/github/reconcile/sweep_test.go +++ b/free/ci/internal/providers/github/reconcile/sweep_test.go @@ -190,7 +190,8 @@ func TestQuotaStops(t *testing.T) { if err = r.e.Demand().Reconcile(context.Background(), r.enq); err != nil || r.f.total() != before { t.Fatalf("demand tick before RetryAt: err=%v requests=%d, want none (the quota stop is shared)", err, r.f.total()-before) } - r.clk.Advance(61 * time.Second) + // RetryAt is stamped with the wall clock, which can be seconds ahead of the fake clock on a loaded runner: pass it with room. + r.clk.Advance(70 * time.Second) r.f.quota = nil if err = r.e.GitHub().Reconcile(context.Background(), r.enq); err != nil { t.Fatal(err) From 1d0f43268b73519164e47678a2eedf8a6dce869b Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 16:26:13 -0400 Subject: [PATCH 22/27] ci(isolation): remove leftover nself-ci containers and networks before the isolation drill --- .github/workflows/ci-isolation-linux.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-isolation-linux.yml b/.github/workflows/ci-isolation-linux.yml index fb7bc263e..8ad386118 100644 --- a/.github/workflows/ci-isolation-linux.yml +++ b/.github/workflows/ci-isolation-linux.yml @@ -167,6 +167,12 @@ jobs: - name: Isolation levels drill run: | set -euo pipefail + # The integration step above ran the same netscope tests; a rootless podman job network it leaves + # behind (nself-ci-t08-hp-podman-net, run 40016512358) makes the drill's own run of that test fail. + for b in docker podman; do + "$b" ps -aq --filter name=nself-ci | xargs -r "$b" rm -f >/dev/null 2>&1 || true + "$b" network ls --format '{{.Name}}' | grep '^nself-ci-' | xargs -r "$b" network rm >/dev/null 2>&1 || true + done drill="" for d in scripts/drill/isolation-levels/run free/ci/scripts/drill/isolation-levels/run; do if [[ -x "$d" ]]; then drill="$d"; break; fi From fa98bf573fcafd83442f904dfeed421c61ead92b Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 16:49:58 -0400 Subject: [PATCH 23/27] fix(exec): podman listings use index .Labels, not the docker-only .Label method (hygiene and sweep failed with exit 125 and left job networks behind) --- free/ci/internal/exec/hygiene_seam.go | 11 ++++++++++- free/ci/internal/exec/hygiene_seam_test.go | 10 ++++++++++ free/ci/internal/exec/sweep.go | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/free/ci/internal/exec/hygiene_seam.go b/free/ci/internal/exec/hygiene_seam.go index 404326f56..b7b80b6dc 100644 --- a/free/ci/internal/exec/hygiene_seam.go +++ b/free/ci/internal/exec/hygiene_seam.go @@ -39,6 +39,15 @@ type Labelled struct { var resourceID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$`) +// labelExpr is the format expression that prints one label's value. Docker's listings offer the +// .Label method; podman's report structs have none (the format fails with exit 125) and expose Labels as a map. +func labelExpr(binary, key string) string { + if binary == "podman" { + return `{{index .Labels "` + key + `"}}` + } + return `{{.Label "` + key + `"}}` +} + // RuntimeListLabelled lists the resources of one kind that carry the attempt label. A non-empty // attempt narrows the list to that attempt; an empty one returns every attempt-labelled resource. func RuntimeListLabelled(ctx context.Context, binary string, kind ResourceKind, attempt string) ([]Labelled, error) { @@ -49,7 +58,7 @@ func RuntimeListLabelled(ctx context.Context, binary string, kind ResourceKind, if attempt != "" { filter += "=" + attempt } - label := `{{.Label "` + AttemptLabel + `"}}` + label := labelExpr(binary, AttemptLabel) var args []string switch kind { case KindContainer: diff --git a/free/ci/internal/exec/hygiene_seam_test.go b/free/ci/internal/exec/hygiene_seam_test.go index 7cb3cd9dc..347a5f0a9 100644 --- a/free/ci/internal/exec/hygiene_seam_test.go +++ b/free/ci/internal/exec/hygiene_seam_test.go @@ -152,3 +152,13 @@ func TestKillAttemptGroupKillsALiveGroupWhoseLeaderExited(t *testing.T) { } } } + +// podman's listings have no .Label method (exit 125, which left job networks behind); docker's do. +func TestLabelExprPerRuntime(t *testing.T) { + if got := labelExpr("docker", "k"); got != `{{.Label "k"}}` { + t.Errorf("docker: %s", got) + } + if got := labelExpr("podman", "k"); got != `{{index .Labels "k"}}` { + t.Errorf("podman: %s", got) + } +} diff --git a/free/ci/internal/exec/sweep.go b/free/ci/internal/exec/sweep.go index 6a0abc493..deb96bccb 100644 --- a/free/ci/internal/exec/sweep.go +++ b/free/ci/internal/exec/sweep.go @@ -117,7 +117,7 @@ func (e *Executor) sweepRuntime(ctx context.Context, current, binary string) err if _, err := RuntimeQuery(ctx, binary, "info"); err != nil { // the shared probe: hygiene and the isolation facts ask the same question return nil } - out, err := RuntimeQuery(ctx, binary, "ps", "-a", "--filter", "label=nself.ci.coordinator", "--filter", "label=nself.ci.attempt", "--format", "{{.ID}} {{.Label \"nself.ci.coordinator\"}}") + out, err := RuntimeQuery(ctx, binary, "ps", "-a", "--filter", "label=nself.ci.coordinator", "--filter", "label=nself.ci.attempt", "--format", "{{.ID}} "+labelExpr(binary, "nself.ci.coordinator")) if err != nil { return coded("E613", binary+" orphan listing failed") } From 1718b972dbb3078bf37d9346f3445603ed802a87 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 17:19:40 -0400 Subject: [PATCH 24/27] ci(scenarios): allow the network cut on the throwaway Linux runner while the engine's probe container winds down --- .github/workflows/ci-scenarios.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-scenarios.yml b/.github/workflows/ci-scenarios.yml index edb8e9a98..63615c718 100644 --- a/.github/workflows/ci-scenarios.yml +++ b/.github/workflows/ci-scenarios.yml @@ -100,6 +100,9 @@ jobs: - name: Scenario 1, local only (network cut) if: ${{ !cancelled() }} + env: + # the runner is a throwaway VM; the engine's own network probe container can still be winding down when the cut preflight looks (run 40022254401) + DRILL_CUT_ALLOW_SHARED_HOST: ${{ runner.os == 'Linux' && '1' || '' }} run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/evidence" From c61420b49be99ced706d25bb73935a9877fa9397 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 17:31:16 -0400 Subject: [PATCH 25/27] fix(drill): the network-cut preflight waits up to 60s for the engine's own probe container to exit --- scripts/drill/fixtures/lib/cut.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/drill/fixtures/lib/cut.sh b/scripts/drill/fixtures/lib/cut.sh index 9d1697418..1e9f27e86 100644 --- a/scripts/drill/fixtures/lib/cut.sh +++ b/scripts/drill/fixtures/lib/cut.sh @@ -48,6 +48,11 @@ cut_preflight() { ! getent group "$CUT_GID" >/dev/null 2>&1 || cut_die "group id $CUT_GID is already in use; set CUT_GID to a free number" # The cut drops forwarding out of every docker bridge and the docker daemon's own egress, so it would break other # containers on a shared box. Refuse unless the caller says this host is the drill's own (hosted runner, throwaway VM). + # The engine's own runtime probe container can still be winding down when this job starts (run 40022254401): wait for it. + if [ "${DRILL_CUT_ALLOW_SHARED_HOST:-}" != 1 ]; then + cut_waited=0 + while [ -n "$(docker ps -q 2>/dev/null)" ] && [ "$cut_waited" -lt 30 ]; do sleep 2; cut_waited=$((cut_waited + 1)); done + fi if [ -n "$(docker ps -q 2>/dev/null)" ] && [ "${DRILL_CUT_ALLOW_SHARED_HOST:-}" != 1 ]; then cut_die "containers are running ($(docker ps --format "{{.ID}} {{.Image}} {{.Names}} {{.Status}}" 2>/dev/null | tr "\n" ";")); the cut would cut their network too. Stop them, or set DRILL_CUT_ALLOW_SHARED_HOST=1 on a host that is only for this drill" fi From 01ab0a4020cf5c977dd9fd95eab8d98cf029c680 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 18:02:01 -0400 Subject: [PATCH 26/27] fix(exec): podman network listings keep the label map under .Network.Labels --- free/ci/internal/exec/hygiene_seam.go | 7 +++++-- free/ci/internal/exec/hygiene_seam_test.go | 7 +++++-- free/ci/internal/exec/sweep.go | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/free/ci/internal/exec/hygiene_seam.go b/free/ci/internal/exec/hygiene_seam.go index b7b80b6dc..12b7d6d2f 100644 --- a/free/ci/internal/exec/hygiene_seam.go +++ b/free/ci/internal/exec/hygiene_seam.go @@ -41,8 +41,11 @@ var resourceID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$`) // labelExpr is the format expression that prints one label's value. Docker's listings offer the // .Label method; podman's report structs have none (the format fails with exit 125) and expose Labels as a map. -func labelExpr(binary, key string) string { +func labelExpr(binary string, kind ResourceKind, key string) string { if binary == "podman" { + if kind == KindNetwork { // podman 5: a network's .Labels is a flat list, the map is under .Network + return `{{index .Network.Labels "` + key + `"}}` + } return `{{index .Labels "` + key + `"}}` } return `{{.Label "` + key + `"}}` @@ -58,7 +61,7 @@ func RuntimeListLabelled(ctx context.Context, binary string, kind ResourceKind, if attempt != "" { filter += "=" + attempt } - label := labelExpr(binary, AttemptLabel) + label := labelExpr(binary, kind, AttemptLabel) var args []string switch kind { case KindContainer: diff --git a/free/ci/internal/exec/hygiene_seam_test.go b/free/ci/internal/exec/hygiene_seam_test.go index 347a5f0a9..07391e415 100644 --- a/free/ci/internal/exec/hygiene_seam_test.go +++ b/free/ci/internal/exec/hygiene_seam_test.go @@ -155,10 +155,13 @@ func TestKillAttemptGroupKillsALiveGroupWhoseLeaderExited(t *testing.T) { // podman's listings have no .Label method (exit 125, which left job networks behind); docker's do. func TestLabelExprPerRuntime(t *testing.T) { - if got := labelExpr("docker", "k"); got != `{{.Label "k"}}` { + if got := labelExpr("docker", KindContainer, "k"); got != `{{.Label "k"}}` { t.Errorf("docker: %s", got) } - if got := labelExpr("podman", "k"); got != `{{index .Labels "k"}}` { + if got := labelExpr("podman", KindContainer, "k"); got != `{{index .Labels "k"}}` { t.Errorf("podman: %s", got) } + if got := labelExpr("podman", KindNetwork, "k"); got != `{{index .Network.Labels "k"}}` { + t.Errorf("podman network: %s", got) + } } diff --git a/free/ci/internal/exec/sweep.go b/free/ci/internal/exec/sweep.go index deb96bccb..5f6d10cc5 100644 --- a/free/ci/internal/exec/sweep.go +++ b/free/ci/internal/exec/sweep.go @@ -117,7 +117,7 @@ func (e *Executor) sweepRuntime(ctx context.Context, current, binary string) err if _, err := RuntimeQuery(ctx, binary, "info"); err != nil { // the shared probe: hygiene and the isolation facts ask the same question return nil } - out, err := RuntimeQuery(ctx, binary, "ps", "-a", "--filter", "label=nself.ci.coordinator", "--filter", "label=nself.ci.attempt", "--format", "{{.ID}} "+labelExpr(binary, "nself.ci.coordinator")) + out, err := RuntimeQuery(ctx, binary, "ps", "-a", "--filter", "label=nself.ci.coordinator", "--filter", "label=nself.ci.attempt", "--format", "{{.ID}} "+labelExpr(binary, KindContainer, "nself.ci.coordinator")) if err != nil { return coded("E613", binary+" orphan listing failed") } From f1740e86298ace27ca603e76090d8a2070a72d66 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 23 Oct 2026 18:32:38 -0400 Subject: [PATCH 27/27] ci(isolation): the drill job's own pass line decides the step (a single-job run is a partial selection and exits 1) --- .github/workflows/ci-isolation-linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-isolation-linux.yml b/.github/workflows/ci-isolation-linux.yml index 8ad386118..a289ba98e 100644 --- a/.github/workflows/ci-isolation-linux.yml +++ b/.github/workflows/ci-isolation-linux.yml @@ -183,7 +183,9 @@ jobs: fi work="$(mktemp -d)" (cd free/ci && CGO_ENABLED=0 go build -mod=vendor -o "$work/nself-ci" ./cmd) - NSELF_CI_BIN="$work/nself-ci" NSELF_CI_HOME="$work/home" "$drill" + # A single-job run is a partial selection, so nself-ci exits 1 even when the job passes: the job's own line decides. + NSELF_CI_BIN="$work/nself-ci" NSELF_CI_HOME="$work/home" "$drill" 2>&1 | tee "$work/drill.txt" || true + grep -Eq '^ +pass +drill-isolation-levels( |$)' "$work/drill.txt" || { echo "drill-isolation-levels did not pass" >&2; exit 1; } # doctor exits 12 when any unrelated check warns; the jq checks below decide the step. NSELF_V15=1 NSELF_CI_HOME="$work/home" "$work/nself-ci" doctor --json | tee "$work/doctor.json" || [[ ${PIPESTATUS[0]} -eq 12 ]] for check in isolation.container isolation.sandboxed-container isolation.network; do