From 8f06e9fc27b94e99ce0fb77f513e87e909861c09 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:05:13 -0600 Subject: [PATCH 1/3] fix(apply-rulesets): --all skips GHSA forks and FLEET_EXCLUDE repos `--all` listed every non-archived org repo, including the private `-ghsa-` forks GitHub creates for security advisories. Their rulesets can't be read ("failed to resolve ruleset ID"), so --all exited 1 with "5 repo(s) failed" even when every real repo succeeded. Detect them as combsmith/.github-private#2462 does: GHSA id suffix, private, issues off, and parent repo present in the org. Also add FLEET_EXCLUDE (software-factory, software-factory-canary): they have no dependency-audit workflow, so code-quality's required Audit gate check would block every merge. An explicit --repo still applies to them. Co-Authored-By: Claude Opus 5.5 --- scripts/apply-rulesets.sh | 48 ++++++++-- .../apply-rulesets/apply-rulesets.bats | 90 ++++++++++++++++++- 2 files changed, 130 insertions(+), 8 deletions(-) diff --git a/scripts/apply-rulesets.sh b/scripts/apply-rulesets.sh index 06c240de4..821392283 100755 --- a/scripts/apply-rulesets.sh +++ b/scripts/apply-rulesets.sh @@ -37,7 +37,8 @@ set -euo pipefail # Usage: # GH_TOKEN= ./scripts/apply-rulesets.sh --repo owner/repo [--dry-run] [...] # GH_TOKEN= ./scripts/apply-rulesets.sh [--dry-run] # bare name → $ORG/ -# GH_TOKEN= ./scripts/apply-rulesets.sh --all [--dry-run] # every non-archived org repo +# GH_TOKEN= ./scripts/apply-rulesets.sh --all [--dry-run] # every non-archived org repo, +# # minus GHSA forks and FLEET_EXCLUDE # # Env: # GH_TOKEN token with administration:write on the target repo(s) (required for writes) @@ -57,6 +58,12 @@ DRY_RUN="${DRY_RUN:-false}" # wide — it is applied only when named explicitly (e.g. `--repo release-channel-tags`). readonly FLEET_RULESETS=(code-quality pr-quality) +# Repos --all leaves alone. They have no dependency-audit workflow, so the +# code-quality ruleset's required `dependency-audit / Audit gate` check would +# never report and would block every merge; they keep their existing rulesets. +# An explicit --repo still applies to them. +readonly FLEET_EXCLUDE=(software-factory software-factory-canary) + # The code-scanning threshold floor (cs_floor_payload): code-quality.json's # code_scanning rule is a minimum, so an update keeps any stricter live threshold. # shellcheck source=lib/code-scanning.sh @@ -198,6 +205,27 @@ apply_repo() { return "$rc" } +# classify_fleet_repos — read `gh repo list --json name,isPrivate,hasIssuesEnabled` +# on stdin and print one `\t` line per repo, action being apply, +# ghsa (a GHSA temporary private fork) or excluded (in FLEET_EXCLUDE). +# GitHub creates a private fork named -ghsa- for each security advisory, +# without the fork flag and with issues disabled; its rulesets can't be read, so +# applying to it always fails. A GHSA-shaped name alone is not enough: the repo +# must also be private, have issues off, and have its parent repo in the org +# (same check as dev-lead-retry.sh in combsmith/.github-private#2462). +classify_fleet_repos() { + jq -r --args '"-ghsa(-[23456789cfghjmpqrvwx]{4}){3}$" as $ghsa + | $ARGS.positional as $exclude + | [.[].name] as $names + | .[] + | (if (.isPrivate == true and .hasIssuesEnabled == false + and (.name | test($ghsa)) + and ((.name | sub($ghsa; "")) as $p | any($names[]; . == $p))) + then "ghsa" + elif (.name as $n | any($exclude[]; . == $n)) then "excluded" + else "apply" end) + "\t" + .name' "${FLEET_EXCLUDE[@]}" +} + # resolve_repo — echo owner/repo: pass through an owner/repo, else prefix $ORG. resolve_repo() { case "$1" in */*) printf '%s' "$1" ;; *) printf '%s/%s' "$ORG" "$1" ;; esac @@ -240,14 +268,24 @@ main() { if [ "$all" = true ]; then [ -z "$target" ] || { echo "::error::--all and a repo argument are mutually exclusive" >&2; return 2; } echo "[apply-rulesets] fetching non-archived repos in ${ORG} ..." - local repos repo failed=0 repos_rc=0 - repos="$(gh repo list "$ORG" --no-archived --json name -q '.[].name' --limit 500)" && repos_rc=0 || repos_rc=$? + local repos_json classified action repo failed=0 repos_rc=0 + local repos=() + repos_json="$(gh repo list "$ORG" --no-archived --json name,isPrivate,hasIssuesEnabled --limit 500)" && repos_rc=0 || repos_rc=$? if [ "$repos_rc" -ne 0 ]; then echo "::error::failed to list repositories for organization ${ORG} (exit code ${repos_rc})" >&2 return "$repos_rc" fi - [ -n "$repos" ] || { echo "::error::no repositories found in ${ORG} — check GH_TOKEN" >&2; return 1; } - for repo in $repos; do + classified="$(classify_fleet_repos <<< "$repos_json")" \ + || { echo "::error::failed to parse the repository list for ${ORG}" >&2; return 1; } + while IFS=$'\t' read -r action repo; do + case "$action" in + apply) repos+=("$repo") ;; + ghsa) echo "[apply-rulesets] skip ${ORG}/${repo}: GHSA temporary private fork" ;; + excluded) echo "[apply-rulesets] skip ${ORG}/${repo}: in FLEET_EXCLUDE (no dependency-audit workflow)" ;; + esac + done <<< "$classified" + [ "${#repos[@]}" -gt 0 ] || { echo "::error::no repositories found in ${ORG} — check GH_TOKEN" >&2; return 1; } + for repo in "${repos[@]}"; do apply_repo "${ORG}/${repo}" "${files[@]}" || { failed=$((failed + 1)); echo "::warning::failed on ${ORG}/${repo}"; } done [ "$failed" -eq 0 ] || { echo "::error::${failed} repo(s) failed" >&2; return 1; } diff --git a/test/scripts/apply-rulesets/apply-rulesets.bats b/test/scripts/apply-rulesets/apply-rulesets.bats index 1d129f5bd..a1cf557b7 100644 --- a/test/scripts/apply-rulesets/apply-rulesets.bats +++ b/test/scripts/apply-rulesets/apply-rulesets.bats @@ -16,7 +16,8 @@ teardown() { [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN"; return 0; } # gh stub: records writes (POST/PUT) to $CALLS (args line then --input body); # returns $RULESETS_LIST for the rulesets LIST (default [] → create path); # returns $RULESET_DETAIL for one ruleset by id (default {}; the literal FAIL -# makes that read fail); returns $REPO_LIST for `repo list`; serves the repo's +# makes that read fail); fails every rulesets call whose args contain +# $RULESETS_FAIL (unset → none fail); returns $REPO_LIST for `repo list`; serves the repo's # .github/required-checks.json from $DECLARED (unset → HTTP 404, i.e. the repo # declares nothing; the literal FAIL → HTTP 500) and logs that read to $READS. _stub_gh() { @@ -38,6 +39,8 @@ case "\$args" in [ -z "\${DECLARED:-}" ] && { echo "gh: Not Found (HTTP 404)" >&2; exit 1; } printf '%s' "\$DECLARED" ;; *"repo list"*) printf '%s' "\${REPO_LIST:-}" ;; + *"\${RULESETS_FAIL:-//none//}"*"rulesets"*) + echo "failed to fetch rulesets: failed to resolve ruleset ID (HTTP 404)" >&2; exit 1 ;; *"/rulesets/"*) [ "\${RULESET_DETAIL:-}" = "FAIL" ] && { echo "gh: Server Error (HTTP 500)" >&2; exit 1; } empty='{}'; printf '%s' "\${RULESET_DETAIL:-\$empty}" ;; @@ -254,9 +257,14 @@ DEP_AUDIT_WF="$SCRIPT_DIR/.github/workflows/dependency-audit.yml" } # ── fleet mode ──────────────────────────────────────────────────────────────── +# _repo — one `gh repo list --json` entry. +_repo() { jq -cn --arg n "$1" --argjson p "$2" --argjson i "$3" '{name: $n, isPrivate: $p, hasIssuesEnabled: $i}'; } +_repo_list() { jq -cs . <<< "$(printf '%s\n' "$@")"; } + @test "apply --all: iterates every non-archived org repo" { _stub_gh - export RULESETS_LIST='[]' REPO_LIST=$'alpha\nbeta' + export RULESETS_LIST='[]' + REPO_LIST="$(_repo_list "$(_repo alpha false true)" "$(_repo beta true true)")"; export REPO_LIST run bash "$APPLY" --all --dry-run [ "$status" -eq 0 ] [[ "$output" == *"repo=combsmith/alpha"* ]] @@ -264,6 +272,82 @@ DEP_AUDIT_WF="$SCRIPT_DIR/.github/workflows/dependency-audit.yml" [[ "$output" == *"across the fleet"* ]] } +# GHSA temporary private forks (-ghsa-) answer "failed to resolve +# ruleset ID" on every rulesets read, so applying to them made --all exit 1 +# ("N repo(s) failed") even when every real repo succeeded. +@test "apply --all: skips GHSA temporary private forks with a log line and exits 0" { + _stub_gh + export RULESETS_LIST='[]' RULESETS_FAIL='-ghsa-' + REPO_LIST="$(_repo_list \ + "$(_repo TalkTerm false true)" \ + "$(_repo TalkTerm-ghsa-pc5f-cw67-7cmm true false)" \ + "$(_repo .github false true)" \ + "$(_repo .github-ghsa-qpjq-vrh6-8wcg true false)")"; export REPO_LIST + run bash "$APPLY" --all --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"skip combsmith/TalkTerm-ghsa-pc5f-cw67-7cmm: GHSA temporary private fork"* ]] + [[ "$output" == *"skip combsmith/.github-ghsa-qpjq-vrh6-8wcg: GHSA temporary private fork"* ]] + [[ "$output" == *"repo=combsmith/TalkTerm "* ]] + [[ "$output" == *"repo=combsmith/.github "* ]] + [[ "$output" != *"repo=combsmith/TalkTerm-ghsa"* ]] + [[ "$output" != *"repo=combsmith/.github-ghsa"* ]] + [[ "$output" != *"failed"* ]] +} + +@test "apply --all: a GHSA-shaped name alone never skips a repo" { + _stub_gh + export RULESETS_LIST='[]' + # public; issues on; no parent repo in the org; 'a'/'b' are not in the GHSA id alphabet + REPO_LIST="$(_repo_list \ + "$(_repo app false true)" \ + "$(_repo app-ghsa-pc5f-cw67-7cmm false false)" \ + "$(_repo app-ghsa-rw84-v39h-45cp true true)" \ + "$(_repo gone-ghsa-fx78-wx9r-525p true false)" \ + "$(_repo app-ghsa-aaaa-bbbb-aaaa true false)")"; export REPO_LIST + run bash "$APPLY" --all --dry-run + [ "$status" -eq 0 ] + [[ "$output" != *"skip "* ]] + local r + for r in app-ghsa-pc5f-cw67-7cmm app-ghsa-rw84-v39h-45cp gone-ghsa-fx78-wx9r-525p app-ghsa-aaaa-bbbb-aaaa; do + [[ "$output" == *"repo=combsmith/${r} "* ]] + done +} + +@test "apply --all: a real repo failing still fails the run" { + _stub_gh + export RULESETS_LIST='[]' RULESETS_FAIL='combsmith/beta/' + REPO_LIST="$(_repo_list "$(_repo alpha false true)" "$(_repo beta false true)")"; export REPO_LIST + run bash "$APPLY" --all --dry-run + [ "$status" -eq 1 ] + [[ "$output" == *"1 repo(s) failed"* ]] +} + +# software-factory and software-factory-canary have no dependency-audit workflow; +# the code-quality ruleset would block every merge there. +@test "apply --all: skips FLEET_EXCLUDE repos with a log line, makes no calls for them" { + _stub_gh + export RULESETS_LIST='[]' RULESETS_FAIL='software-factory' + REPO_LIST="$(_repo_list \ + "$(_repo alpha false true)" \ + "$(_repo software-factory false true)" \ + "$(_repo software-factory-canary false true)")"; export REPO_LIST + run bash "$APPLY" --all + [ "$status" -eq 0 ] + [[ "$output" == *"skip combsmith/software-factory: in FLEET_EXCLUDE"* ]] + [[ "$output" == *"skip combsmith/software-factory-canary: in FLEET_EXCLUDE"* ]] + [[ "$output" != *"repo=combsmith/software-factory"* ]] + run grep -c 'repos/combsmith/software-factory' "$CALLS" + [ "$output" = "0" ] +} + +@test "apply --repo: an explicit --repo still applies to a FLEET_EXCLUDE repo" { + _stub_gh + export RULESETS_LIST='[]' + run bash "$APPLY" --repo combsmith/software-factory --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"repo=combsmith/software-factory "* ]] +} + @test "apply: --all and --repo are mutually exclusive" { _stub_gh run bash "$APPLY" --all --repo combsmith/acme @@ -273,7 +357,7 @@ DEP_AUDIT_WF="$SCRIPT_DIR/.github/workflows/dependency-audit.yml" @test "apply: a bare token after --all is a ruleset-name filter (unknown → errors)" { _stub_gh - export REPO_LIST=$'alpha' + REPO_LIST="$(_repo_list "$(_repo alpha false true)")"; export REPO_LIST run bash "$APPLY" --all not-a-ruleset --dry-run [ "$status" -ne 0 ] [[ "$output" == *"no ruleset file not-a-ruleset.json"* ]] From 16bd7b2200876761217cbfd5c1c848b20f31b9dd Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:12:16 -0600 Subject: [PATCH 2/3] test(apply-rulesets): assert no gh call of any kind targets FLEET_EXCLUDE repos The excluded-repo test checked $CALLS, which records only writes, so a read against an excluded repo (e.g. a required-checks GET answered with a handled 404) would pass unnoticed. The stub now logs every call to $GH_LOG. Co-Authored-By: Claude Opus 5.5 --- test/scripts/apply-rulesets/apply-rulesets.bats | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/scripts/apply-rulesets/apply-rulesets.bats b/test/scripts/apply-rulesets/apply-rulesets.bats index a1cf557b7..83cbd2f02 100644 --- a/test/scripts/apply-rulesets/apply-rulesets.bats +++ b/test/scripts/apply-rulesets/apply-rulesets.bats @@ -10,10 +10,11 @@ setup() { STUB_BIN="$(mktemp -d)"; export PATH="$STUB_BIN:$PATH" CALLS="$STUB_BIN/calls.log"; export CALLS READS="$STUB_BIN/reads.log"; export READS + GH_LOG="$STUB_BIN/gh.log"; export GH_LOG } teardown() { [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN"; return 0; } -# gh stub: records writes (POST/PUT) to $CALLS (args line then --input body); +# gh stub: logs every call's args to $GH_LOG; records writes (POST/PUT) to $CALLS (args line then --input body); # returns $RULESETS_LIST for the rulesets LIST (default [] → create path); # returns $RULESET_DETAIL for one ruleset by id (default {}; the literal FAIL # makes that read fail); fails every rulesets call whose args contain @@ -24,6 +25,7 @@ _stub_gh() { cat > "$STUB_BIN/gh" <> "$GH_LOG" case "\$args" in *"--method POST"*|*"--method PUT"*) echo "\$args" >> "$CALLS" @@ -324,7 +326,7 @@ _repo_list() { jq -cs . <<< "$(printf '%s\n' "$@")"; } # software-factory and software-factory-canary have no dependency-audit workflow; # the code-quality ruleset would block every merge there. -@test "apply --all: skips FLEET_EXCLUDE repos with a log line, makes no calls for them" { +@test "apply --all: skips FLEET_EXCLUDE repos with a log line, makes no gh calls for them" { _stub_gh export RULESETS_LIST='[]' RULESETS_FAIL='software-factory' REPO_LIST="$(_repo_list \ @@ -336,7 +338,10 @@ _repo_list() { jq -cs . <<< "$(printf '%s\n' "$@")"; } [[ "$output" == *"skip combsmith/software-factory: in FLEET_EXCLUDE"* ]] [[ "$output" == *"skip combsmith/software-factory-canary: in FLEET_EXCLUDE"* ]] [[ "$output" != *"repo=combsmith/software-factory"* ]] - run grep -c 'repos/combsmith/software-factory' "$CALLS" + # Every gh call, reads included (a handled 404 on required-checks.json would + # not fail the run), so none may target an excluded repo. + grep -q 'repos/combsmith/alpha/' "$GH_LOG" + run grep -c 'combsmith/software-factory' "$GH_LOG" [ "$output" = "0" ] } From f9c133dba937ab11e74aafbe7e676c575890e6c7 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:44:05 -0600 Subject: [PATCH 3/3] fix(apply-rulesets): find a GHSA fork's archived parent --all listed repos with --no-archived, so a fork whose parent was archived after the advisory opened had no parent in the lookup, was applied to, and failed the run again. List archived repos too (limit 1000) for the parent lookup only; archived repos are never applied to. Co-Authored-By: Claude Opus 5.5 --- scripts/apply-rulesets.sh | 14 +++++++---- .../apply-rulesets/apply-rulesets.bats | 24 ++++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/scripts/apply-rulesets.sh b/scripts/apply-rulesets.sh index 821392283..99452e4e9 100755 --- a/scripts/apply-rulesets.sh +++ b/scripts/apply-rulesets.sh @@ -205,9 +205,11 @@ apply_repo() { return "$rc" } -# classify_fleet_repos — read `gh repo list --json name,isPrivate,hasIssuesEnabled` -# on stdin and print one `\t` line per repo, action being apply, -# ghsa (a GHSA temporary private fork) or excluded (in FLEET_EXCLUDE). +# classify_fleet_repos — read `gh repo list --json name,isArchived,isPrivate,hasIssuesEnabled` +# (archived repos included) on stdin and print one `\t` line per +# repo, action being apply, archived, ghsa (a GHSA temporary private fork) or +# excluded (in FLEET_EXCLUDE). Archived repos are listed only so that a fork +# whose parent was archived after the advisory opened is still recognised. # GitHub creates a private fork named -ghsa- for each security advisory, # without the fork flag and with issues disabled; its rulesets can't be read, so # applying to it always fails. A GHSA-shaped name alone is not enough: the repo @@ -218,7 +220,8 @@ classify_fleet_repos() { | $ARGS.positional as $exclude | [.[].name] as $names | .[] - | (if (.isPrivate == true and .hasIssuesEnabled == false + | (if .isArchived == true then "archived" + elif (.isPrivate == true and .hasIssuesEnabled == false and (.name | test($ghsa)) and ((.name | sub($ghsa; "")) as $p | any($names[]; . == $p))) then "ghsa" @@ -270,7 +273,7 @@ main() { echo "[apply-rulesets] fetching non-archived repos in ${ORG} ..." local repos_json classified action repo failed=0 repos_rc=0 local repos=() - repos_json="$(gh repo list "$ORG" --no-archived --json name,isPrivate,hasIssuesEnabled --limit 500)" && repos_rc=0 || repos_rc=$? + repos_json="$(gh repo list "$ORG" --json name,isArchived,isPrivate,hasIssuesEnabled --limit 1000)" && repos_rc=0 || repos_rc=$? if [ "$repos_rc" -ne 0 ]; then echo "::error::failed to list repositories for organization ${ORG} (exit code ${repos_rc})" >&2 return "$repos_rc" @@ -280,6 +283,7 @@ main() { while IFS=$'\t' read -r action repo; do case "$action" in apply) repos+=("$repo") ;; + archived) ;; ghsa) echo "[apply-rulesets] skip ${ORG}/${repo}: GHSA temporary private fork" ;; excluded) echo "[apply-rulesets] skip ${ORG}/${repo}: in FLEET_EXCLUDE (no dependency-audit workflow)" ;; esac diff --git a/test/scripts/apply-rulesets/apply-rulesets.bats b/test/scripts/apply-rulesets/apply-rulesets.bats index 83cbd2f02..3a2943433 100644 --- a/test/scripts/apply-rulesets/apply-rulesets.bats +++ b/test/scripts/apply-rulesets/apply-rulesets.bats @@ -18,7 +18,8 @@ teardown() { [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN"; return 0; } # returns $RULESETS_LIST for the rulesets LIST (default [] → create path); # returns $RULESET_DETAIL for one ruleset by id (default {}; the literal FAIL # makes that read fail); fails every rulesets call whose args contain -# $RULESETS_FAIL (unset → none fail); returns $REPO_LIST for `repo list`; serves the repo's +# $RULESETS_FAIL (unset → none fail); returns $REPO_LIST for `repo list` +# (minus isArchived entries under --no-archived); serves the repo's # .github/required-checks.json from $DECLARED (unset → HTTP 404, i.e. the repo # declares nothing; the literal FAIL → HTTP 500) and logs that read to $READS. _stub_gh() { @@ -40,6 +41,7 @@ case "\$args" in [ "\${DECLARED:-}" = "FAIL" ] && { echo "gh: Server Error (HTTP 500)" >&2; exit 1; } [ -z "\${DECLARED:-}" ] && { echo "gh: Not Found (HTTP 404)" >&2; exit 1; } printf '%s' "\$DECLARED" ;; + *"repo list"*"--no-archived"*) jq -c 'map(select(.isArchived != true))' <<< "\${REPO_LIST:-[]}" ;; *"repo list"*) printf '%s' "\${REPO_LIST:-}" ;; *"\${RULESETS_FAIL:-//none//}"*"rulesets"*) echo "failed to fetch rulesets: failed to resolve ruleset ID (HTTP 404)" >&2; exit 1 ;; @@ -259,8 +261,8 @@ DEP_AUDIT_WF="$SCRIPT_DIR/.github/workflows/dependency-audit.yml" } # ── fleet mode ──────────────────────────────────────────────────────────────── -# _repo — one `gh repo list --json` entry. -_repo() { jq -cn --arg n "$1" --argjson p "$2" --argjson i "$3" '{name: $n, isPrivate: $p, hasIssuesEnabled: $i}'; } +# _repo [] — one `gh repo list --json` entry. +_repo() { jq -cn --arg n "$1" --argjson p "$2" --argjson i "$3" --argjson a "${4:-false}" '{name: $n, isArchived: $a, isPrivate: $p, hasIssuesEnabled: $i}'; } _repo_list() { jq -cs . <<< "$(printf '%s\n' "$@")"; } @test "apply --all: iterates every non-archived org repo" { @@ -296,6 +298,22 @@ _repo_list() { jq -cs . <<< "$(printf '%s\n' "$@")"; } [[ "$output" != *"failed"* ]] } +# The listing includes archived repos so the parent lookup still finds a parent +# that was archived after the advisory's fork was created. +@test "apply --all: skips a GHSA fork whose parent is archived; never applies to archived repos" { + _stub_gh + export RULESETS_LIST='[]' RULESETS_FAIL='-ghsa-' + REPO_LIST="$(_repo_list \ + "$(_repo alpha false true)" \ + "$(_repo oldapp false true true)" \ + "$(_repo oldapp-ghsa-pc5f-cw67-7cmm true false)")"; export REPO_LIST + run bash "$APPLY" --all --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"skip combsmith/oldapp-ghsa-pc5f-cw67-7cmm: GHSA temporary private fork"* ]] + [[ "$output" == *"repo=combsmith/alpha "* ]] + [[ "$output" != *"repo=combsmith/oldapp"* ]] +} + @test "apply --all: a GHSA-shaped name alone never skips a repo" { _stub_gh export RULESETS_LIST='[]'