diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 0b06f3f7c..279138897 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -2,7 +2,8 @@ # (issue #894): every curl download in ci.yml must carry bounded retries so a # transient GitHub Releases blip retries instead of failing the job. # See test/workflows/ci/install-resilience.bats. Also runs the gitleaks config -# detection tests (issue #1395, test/workflows/ci/secret-scan-config.bats). +# detection tests (issue #1395, test/workflows/ci/secret-scan-config.bats) and +# the Agent Security Scan severity gate (test/workflows/ci/agent-security-gate.bats). name: CI Workflow Tests on: @@ -13,6 +14,7 @@ on: - '.github/workflows/ci-tests.yml' - '.gitleaks.toml' - 'standards/gitleaks.toml' + - '.github/scripts/agent-shield/**' push: branches: [main] paths: @@ -21,6 +23,7 @@ on: - '.github/workflows/ci-tests.yml' - '.gitleaks.toml' - 'standards/gitleaks.toml' + - '.github/scripts/agent-shield/**' permissions: {} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1494adc7a..5f96036d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,8 @@ jobs: timeout-minutes: 10 permissions: contents: read + env: + AGENTSHIELD_MIN_SEVERITY: high steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -173,7 +175,27 @@ jobs: npm_config_fetch_retry_maxtimeout: '30000' npm_config_fetch_timeout: '60000' run: | - npx --ignore-scripts ecc-agentshield@1.4.0 scan --path . --format json --min-severity high + npx --ignore-scripts ecc-agentshield@1.4.0 scan --path . --format json \ + --min-severity "$AGENTSHIELD_MIN_SEVERITY" \ + > "$RUNNER_TEMP/agentshield-report.json" || scan_rc=$? + # Exit 2 only means the report has critical findings, which the gate + # below handles. Any other non-zero exit is a scanner failure. + case "${scan_rc:-0}" in + 0 | 2) ;; + *) + echo "::error::AgentShield scanner failed (exit ${scan_rc})" + exit "$scan_rc" + ;; + esac + + # The scanner's exit code ignores --min-severity, so the threshold is enforced + # here from its JSON report with the gate the AgentShield reusable uses (#1436). + # See test/workflows/ci/agent-security-gate.bats. + - name: Enforce AgentShield severity threshold + run: | + # shellcheck source=/dev/null + . .github/scripts/agent-shield/lib/severity-gate.sh + agent_shield_severity_gate "$AGENTSHIELD_MIN_SEVERITY" "$RUNNER_TEMP/agentshield-report.json" secret-scan: name: Secret scan (gitleaks) diff --git a/test/workflows/ci/agent-security-gate.bats b/test/workflows/ci/agent-security-gate.bats new file mode 100644 index 000000000..7d0fa274f --- /dev/null +++ b/test/workflows/ci/agent-security-gate.bats @@ -0,0 +1,111 @@ +#!/usr/bin/env bats +# ci.yml's Agent Security Scan job must fail on a high-severity finding. +# +# The scanner's --min-severity only filters its report, and it exits non-zero +# only for critical findings, so the job enforces the threshold from the JSON +# report with the shared gate in .github/scripts/agent-shield/lib/severity-gate.sh +# (the same gate the AgentShield reusable uses, #1436). These tests run the job's +# real steps against a stubbed scanner. + +bats_require_minimum_version 1.5.0 + +load 'helpers/setup' + +# job_step NAME — prints the run: script of the agent-security step called NAME. +job_step() { + yq -r ".jobs[\"agent-security\"].steps[] | select(.name == \"$1\") | .run" "$TT_WORKFLOW" +} + +# run_job_steps STUB_EXIT REPORT_JSON — runs the scan and gate steps from the repo +# root, with a stub npx that prints REPORT_JSON and exits STUB_EXIT. +run_job_steps() { + local stub_exit="$1" report_json="$2" + local tmp="$BATS_TEST_TMPDIR" scan gate severity + scan=$(job_step "Run AgentShield") + gate=$(job_step "Enforce AgentShield severity threshold") + severity=$(yq -r '.jobs["agent-security"].env.AGENTSHIELD_MIN_SEVERITY' "$TT_WORKFLOW") + [ -n "$scan" ] && [ "$scan" != "null" ] + [ -n "$gate" ] && [ "$gate" != "null" ] + + mkdir -p "$tmp/bin" "$tmp/runner-temp" + printf '%s\n' "$report_json" >"$tmp/stub-report.json" + cat >"$tmp/bin/npx" <"$tmp/npx.args" +cat "$tmp/stub-report.json" +exit $stub_exit +STUB + chmod +x "$tmp/bin/npx" + printf '%s\n' "$scan" >"$tmp/scan.sh" + printf '%s\n' "$gate" >"$tmp/gate.sh" + + # Each step runs the way the runner runs it: its own `bash -e`, stopping the job. + # shellcheck disable=SC2016 # expanded by the inner bash, not here + run env RUNNER_TEMP="$tmp/runner-temp" AGENTSHIELD_MIN_SEVERITY="$severity" \ + PATH="$tmp/bin:$PATH" bash -c \ + 'cd "$1" && bash -e "$2/scan.sh" && bash -e "$2/gate.sh"' _ "$TT_REPO_ROOT" "$tmp" +} + +finding() { + printf '{"findings":[{"severity":"%s","title":"t","description":"d","file":"f"}],"summary":{"totalFindings":1}}' "$1" +} + +REPORT_CLEAN='{"findings":[],"summary":{"totalFindings":0}}' + +@test "agent-security: the job fails at high" { + [ "$(yq -r '.jobs["agent-security"].env.AGENTSHIELD_MIN_SEVERITY' "$TT_WORKFLOW")" = "high" ] +} + +@test "agent-security: the scanner exits 0 with a high finding, and the job still fails" { + run_job_steps 0 "$(finding high)" + [ "$status" -eq 1 ] + [[ "$output" == *"1 finding(s) at or above high"* ]] + [[ "$output" == *"[HIGH]"* ]] +} + +@test "agent-security: the scanner exits 2 for a critical finding, and the gate reports it" { + run_job_steps 2 "$(finding critical)" + [ "$status" -eq 1 ] + [[ "$output" == *"[CRITICAL]"* ]] +} + +@test "agent-security: a medium finding passes" { + run_job_steps 0 "$(finding medium)" + [ "$status" -eq 0 ] + [[ "$output" == *"no findings at or above high (1 reported)"* ]] +} + +@test "agent-security: a clean scan passes" { + run_job_steps 0 "$REPORT_CLEAN" + [ "$status" -eq 0 ] + [[ "$output" == *"no findings at or above high (0 reported)"* ]] +} + +@test "agent-security: a scanner failure stops the job with the scanner's exit code" { + run_job_steps 4 "$REPORT_CLEAN" + [ "$status" -eq 4 ] + [[ "$output" == *"AgentShield scanner failed (exit 4)"* ]] +} + +@test "agent-security: a scanner that prints no report fails" { + run_job_steps 0 "" + [ "$status" -eq 1 ] +} + +@test "agent-security: the scan keeps --ignore-scripts and asks for JSON at the threshold" { + run_job_steps 0 "$REPORT_CLEAN" + [ "$status" -eq 0 ] + local args + args=$(cat "$BATS_TEST_TMPDIR/npx.args") + [[ "$args" == *"--ignore-scripts"* ]] + [[ "$args" == *"--min-severity high"* ]] + [[ "$args" == *"--format json"* ]] +} + +@test "agent-security: the gate step uses the shared severity gate" { + local gate + gate=$(job_step "Enforce AgentShield severity threshold") + [[ "$gate" == *". .github/scripts/agent-shield/lib/severity-gate.sh"* ]] + # shellcheck disable=SC2016 # a literal string in the step script + [[ "$gate" == *'agent_shield_severity_gate "$AGENTSHIELD_MIN_SEVERITY"'* ]] +}