fix(agent-shield): fail at the configured min-severity (#1436)

.github/scripts/agent-shield/lib/severity-gate.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Severity gate for the AgentShield reusable (issue #1436).
+#
+# The scanner's --min-severity only filters the report, and its exit code only
+# reflects critical findings. This gate reads the scanner's JSON report and
+# decides pass or fail at the configured threshold.
+#
+# Pure decision: no network calls. Anything it cannot verify fails closed.
+#
+# Report schema read here (ecc-agentshield 1.4.0 and 1.6.0, `--format json`):
+#   { "findings": [ { "severity": "critical|high|medium|low|info", "title": "...",
+#                     "description": "...", "file": "...", "line": 12 } ],
+#     "summary": { "totalFindings": 1, ... } }
+#
+# Finding text comes from the scanned repository, so it is cleaned before it
+# reaches the log, and `evidence` is never printed.
+#
+# Tested by test/workflows/agent-shield/severity-gate.bats.
+
+# agent_shield_severity_gate THRESHOLD REPORT_FILE
+#   Returns 0 when no finding is at or above THRESHOLD, 1 otherwise. Also returns
+#   1 for an unknown threshold, or a report that is unreadable or off-schema.
+agent_shield_severity_gate() {
+  local threshold="$1" report="$2" out verdict
+
+  case "$threshold" in
+    critical | high | medium | low | info) ;;
+    *)
+      echo "::error::AgentShield: min-severity must be one of critical, high, medium, low, info"
+      return 1
+      ;;
+  esac
+
+  # -s: exactly one JSON document is accepted, so stray output around the report
+  # is treated as off-schema rather than ignored.
+  out=$(jq -rs --arg threshold "$threshold" '
+    def levels: ["critical", "high", "medium", "low", "info"];
+    def rank: . as $s | levels | index($s);
+    def safe: tostring
+      | gsub("[[:cntrl:]]+"; " ")
+      | gsub(":(?=:)"; ":​")
+      | gsub("##\\["; "#​#[");
+    def location: (.file // "unknown" | safe)
+      + (if .line then ":" + (.line | safe) else "" end);
+
+    if length != 1 then "invalid\texpected one JSON document"
+    else .[0] as $report
+      | if ($report | type) != "object" or ($report.findings | type) != "array" then
+          "invalid\tfindings list missing"
+        elif ($report.summary.totalFindings // null) != ($report.findings | length) then
+          "invalid\tsummary does not match the findings list"
+        elif any($report.findings[];
+            (type != "object") or ((.severity | type) != "string") or (.severity | rank) == null) then
+          "invalid\tfinding with an unknown severity"
+        else
+          ($threshold | rank) as $limit
+          | [$report.findings[] | select((.severity | rank) <= $limit)] as $hits
+          | if ($hits | length) == 0 then
+              "pass\t\($report.findings | length)"
+            else
+              "fail\t\($hits | length)",
+              ($hits | sort_by(.severity | rank)[]
+                | "  [\(.severity | ascii_upcase)] \(.title // "untitled" | safe) (\(location))",
+                  "    \(.description // "" | safe)")
+            end
+        end
+    end
+  ' "$report" 2>/dev/null) || out=""
+
+  verdict="${out%%$'\n'*}"
+  case "$verdict" in
+    pass$'\t'*)
+      echo "AgentShield: no findings at or above ${threshold} (${verdict#*$'\t'} reported)"
+      return 0
+      ;;
+    fail$'\t'*)
+      echo "::error::AgentShield: ${verdict#*$'\t'} finding(s) at or above ${threshold}"
+      printf '%s\n' "${out#*$'\n'}"
+      return 1
+      ;;
+    invalid$'\t'*)
+      echo "::error::AgentShield: cannot trust the scan report: ${verdict#*$'\t'}"
+      return 1
+      ;;
+    *)
+      echo "::error::AgentShield: could not read the scan report"
+      return 1
+      ;;
+  esac
+}

.github/workflows/agent-shield-reusable.yml

@@ -71,7 +71,40 @@ jobs:
           npx --prefix="$npm_prefix" --ignore-scripts "ecc-agentshield@${AS_VERSION}" scan \
             --path "$scan_target" \
             --min-severity "${AS_SEVERITY}" \
-            --format terminal
+            --format json > "$RUNNER_TEMP/agentshield-report.json" || scan_rc=$?
+          # Exit 2 only means the report has critical findings, which the gate
+          # below handles. Any other non-zero exit is a scanner failure.
+          case "${scan_rc:-0}" in
+            0 | 2) ;;
+            *)
+              echo "::error::AgentShield scanner failed (exit ${scan_rc})"
+              exit "$scan_rc"
+              ;;
+          esac
+
+      # The scanner's exit code ignores min-severity, so the threshold is enforced
+      # here from its JSON report (#1436). Checked out after the scan so the tooling
+      # is never part of the scan target, and sparse so later checks ignore it.
+      - name: Checkout AgentShield gate tooling
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: combsmith/.github
+          # This reusable's own commit, so the gate matches the channel the caller
+          # pinned. `job.workflow_sha`, not `github.job_workflow_sha`, which is
+          # empty at step level (#839).
+          ref: ${{ job.workflow_sha }}
+          path: .agent-shield-tooling
+          sparse-checkout: .github/scripts/agent-shield
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Enforce AgentShield severity threshold
+        env:
+          AS_SEVERITY: ${{ inputs.min-severity }}
+        run: |
+          # shellcheck source=/dev/null
+          . .agent-shield-tooling/.github/scripts/agent-shield/lib/severity-gate.sh
+          agent_shield_severity_gate "$AS_SEVERITY" "$RUNNER_TEMP/agentshield-report.json"
 
       # --- Org-specific structural checks ---
       - name: Check required agent files exist

.github/workflows/agent-shield-tests.yml

@@ -0,0 +1,47 @@
+# Quality gates for the AgentShield reusable and its severity gate (issue #1436).
+name: AgentShield Gate Tests
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/agent-shield-*.yml'
+      - '.github/scripts/agent-shield/**'
+      - 'test/workflows/agent-shield/**'
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/agent-shield-*.yml'
+      - '.github/scripts/agent-shield/**'
+      - 'test/workflows/agent-shield/**'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: agent-shield-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Lint and bats
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Install bats, shellcheck, and jq (yq is preinstalled on the runner)
+        run: |
+          set -euo pipefail
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends bats shellcheck jq
+
+      - name: shellcheck
+        run: shellcheck -x .github/scripts/agent-shield/lib/severity-gate.sh
+
+      - name: Run bats suite
+        run: bats --print-output-on-failure test/workflows/agent-shield/

standards/canary-rings.json

@@ -100,7 +100,7 @@
           }
         ],
         "_suspect_note": "suspect_failure_classes (#668 increment 2): same shape as benign classes (`id`, `workflow` ERE, `step` ERE) plus a `guidance` discriminating question. A counted (non-benign) failure that matches a suspect class at differs=1 is triaged SUSPECT instead of REGRESSION \u2014 it still BLOCKS and still needs a human (labelled `needs-human`, NOT auto-cleared like `version_independent`), but the blocker issue renders the `guidance` so the human confirm is a 30-second check rather than a from-scratch investigation. Default-absent (this key omitted) = today's behavior, byte-identical for agents that do not opt in.",
-        "_confirm_note": "require_confirmation (#668 increment 3, Layer 3): opt-in human go/no-go at ONE ring boundary for a correctness-sensitive agent. When a transition sets `require_confirmation: true` and the reliability verdict is otherwise PROMOTE, the frontier holds in state AWAITING_CONFIRMATION \u2014 the scheduled promote-all never auto-advances it; a deliberate `promote <agent> --confirm` dispatch (Actions-audit-logged) does. The confirmation IS the dispatch (no state store). `--confirm` is NOT `--override`: it advances ONLY a reliability-clean AWAITING_CONFIRMATION state and can never bypass a BLOCKED gate. sync-issues files an evidence-carrying `canary-confirm:<agent>` issue (compare diff link + run stats; labelled `needs-human`) that auto-closes on promote or candidate change. dev-lead only initially; reliability still clears next/ring0/ring1 unattended. Default-absent (this key omitted) = fully autonomous, byte-identical for agents that do not opt in.",
+        "_confirm_note": "require_confirmation (#668 increment 3, Layer 3): opt-in human go/no-go at ONE ring boundary for a correctness-sensitive agent. When a transition sets `require_confirmation: true` and the reliability verdict is otherwise PROMOTE, the frontier holds in state AWAITING_CONFIRMATION \u2014 the scheduled promote-all never auto-advances it; a deliberate `promote <agent> --confirm` dispatch (Actions-audit-logged) does. The confirmation IS the dispatch (no state store). `--confirm` is NOT `--override`: it advances ONLY a reliability-clean AWAITING_CONFIRMATION state and can never bypass a BLOCKED gate. sync-issues files an evidence-carrying `canary-confirm:<agent>` issue (compare diff link + run stats; labelled `needs-human`) that auto-closes on promote or candidate change. dev-lead initially; agent-shield also opts in because a release can make its required check fail on findings that already exist in stable-only repos; reliability still clears next/ring0/ring1 unattended. Default-absent (this key omitted) = fully autonomous, byte-identical for agents that do not opt in.",
         "transitions": {
           "next->ring0": {
             "dwell_hours": 4,
@@ -179,7 +179,8 @@
           },
           "ring1->stable": {
             "dwell_hours": 12,
-            "sample_min": 1
+            "sample_min": 1,
+            "require_confirmation": true
           }
         }
       }