diff --git a/.github/scripts/dependency-audit/lib/dependency-audit.sh b/.github/scripts/dependency-audit/lib/dependency-audit.sh new file mode 100644 index 000000000..46f22bf86 --- /dev/null +++ b/.github/scripts/dependency-audit/lib/dependency-audit.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# Discovery, coverage and gate logic for the dependency-audit reusable workflow +# (.github/workflows/dependency-audit-reusable.yml). +# +# The reusable checks this repo out at its own commit (`job.workflow_sha`) and +# sources this file, so the logic always matches the pinned workflow version. +# The workflow steps are thin glue: they install the scanners and call the +# runners below. Everything that decides WHAT gets audited, and whether the run +# as a whole passed, lives here so it is covered by bats +# (test/workflows/dependency-audit/) instead of being trapped inline in YAML. +# +# External commands are resolved through PATH (`cargo`, `pip-audit`, `jq`, +# `find`, `sort`), so tests substitute stubs for the scanners. +# +# Paths are handled NUL-delimited end to end: a directory name may legally +# contain a newline, and a newline-split record would point the guards at a +# path that does not exist and silently skip the real project (#1404). +# +# Compatible with bash 3.2+ (no associative arrays, no `mapfile -d`). +# +# Issues: #1393 (Cargo coverage), #1394 (committed Cargo.lock), #1404 (Python +# discovery), #1406 (aggregate gate). + +# dep_audit_display PATH +# Prints PATH with CR and LF escaped as \r and \n, for log lines and workflow +# commands. A raw newline in a crafted directory name would otherwise end the +# `::group::` line and start a new line the runner parses as a command. +dep_audit_display() { + local s="$1" + s="${s//$'\r'/\\r}" + s="${s//$'\n'/\\n}" + printf '%s' "$s" +} + +# dep_audit_dirs_of_files +# Reads NUL-delimited file paths on stdin and prints the NUL-delimited, sorted, +# de-duplicated set of their parent directories. +dep_audit_dirs_of_files() { + local f + while IFS= read -r -d '' f; do + case "$f" in + */*) printf '%s\0' "${f%/*}" ;; + *) printf '.\0' ;; + esac + done | LC_ALL=C sort -z -u +} + +# dep_audit_python_projects ROOT +# Prints, NUL-delimited, every directory under ROOT that holds a +# pyproject.toml or requirements.txt (regular file or symlink), skipping +# virtualenvs (*/.venv/*, */venv/*). Non-zero if the walk itself fails, so an +# unreadable subtree is not mistaken for "no projects there". +dep_audit_python_projects() { + local root="$1" + ( + set -o pipefail + find "$root" \( -name 'pyproject.toml' -o -name 'requirements.txt' \) \ + \( -type f -o -type l \) \ + -not -path '*/.venv/*' -not -path '*/venv/*' -print0 | + dep_audit_dirs_of_files + ) +} + +# dep_audit_python_run ROOT +# Runs pip-audit for every Python project under ROOT: `pip-audit DIR` when the +# directory has a pyproject.toml and `pip-audit -r DIR/requirements.txt` when +# it has a requirements.txt (both when both are present). Returns non-zero if +# discovery fails, any audit fails, or a discovered project cannot be audited +# — an unprocessable project is an error, never a silent skip (#1404). +dep_audit_python_run() { + local root="$1" list dir shown audited status=0 + list="$(mktemp)" || return 1 + if ! dep_audit_python_projects "$root" >"$list"; then + echo "::error::Python project discovery failed under $(dep_audit_display "$root")" + rm -f "$list" + return 1 + fi + while IFS= read -r -d '' -u 3 dir; do + shown="$(dep_audit_display "$dir")" + echo "::group::pip-audit ${shown}" + audited=0 + if [ -f "$dir/pyproject.toml" ]; then + audited=1 + pip-audit "$dir" 0)' +} + +# dep_audit_cargo_audit_roots ROOT +# Prints, NUL-delimited, the de-duplicated workspace roots that together +# cover every Cargo.toml under ROOT, in manifest order. Each manifest is +# resolved with `cargo metadata --no-deps` (no network, no lockfile write). +# A manifest Cargo cannot resolve is reported with `::error::` and makes the +# function return non-zero; every resolvable root is still printed, so the +# caller can audit what it can and fail the job. +dep_audit_cargo_audit_roots() { + local root="$1" list dir meta ws seen s status=0 + local -a roots=() + list="$(mktemp)" || return 1 + if ! dep_audit_cargo_manifest_dirs "$root" >"$list"; then + echo "::error::Cargo manifest discovery failed under $(dep_audit_display "$root")" >&2 + rm -f "$list" + return 1 + fi + while IFS= read -r -d '' -u 3 dir; do + if ! meta="$(cargo metadata --no-deps --format-version 1 \ + --manifest-path "$dir/Cargo.toml" &2 + status=1 + continue + fi + # Sentinel keeps a trailing newline in the path intact through $(...). + ws="$(printf '%s' "$meta" | dep_audit_cargo_workspace_root && printf x)" + if [ "${ws%x}" = "$ws" ] || [ -z "${ws%x}" ]; then + echo "::error::cargo metadata for $(dep_audit_display "$dir")/Cargo.toml has no workspace_root" >&2 + status=1 + continue + fi + ws="${ws%x}" + seen=0 + for s in ${roots[@]+"${roots[@]}"}; do + if [ "$s" = "$ws" ]; then + seen=1 + break + fi + done + if [ "$seen" -eq 0 ]; then + roots+=("$ws") + printf '%s\0' "$ws" + fi + done 3<"$list" + rm -f "$list" + return "$status" +} + +# dep_audit_cargo_audit_dir DIR +# Audits the lockfile in DIR. A committed Cargo.lock is audited exactly as +# committed — regenerating it first would audit whatever versions resolve +# today instead of the reviewed ones (#1394). Only when DIR has no Cargo.lock +# (typically a library crate) is one generated so there is something to audit. +dep_audit_cargo_audit_dir() { + local dir="$1" + ( + cd -- "$dir" || exit 1 + if [ ! -e Cargo.lock ]; then + echo "No committed Cargo.lock in $(dep_audit_display "$dir"); generating one to audit." + if ! cargo generate-lockfile "$list" || status=1 + while IFS= read -r -d '' -u 3 dir; do + echo "::group::cargo audit $(dep_audit_display "$dir")" + dep_audit_cargo_audit_dir "$dir" || status=1 + echo "::endgroup::" + done 3<"$list" + rm -f "$list" + return "$status" +} + +# dep_audit_gate_verdict +# Reads the reusable's `toJSON(needs)` on stdin — {job_id: {result, outputs}} +# for the detect job and every audit job — and prints one "job: result" line +# per job. Returns 0 only when `detect` succeeded and every other job either +# succeeded or was skipped (its ecosystem is absent). `failure`, `cancelled`, +# a missing result, malformed input or a missing `detect` entry all fail: +# this is the status that can stand in for the whole audit (#1406). +dep_audit_gate_verdict() { + local pairs job result status=0 saw_detect=0 + if ! pairs="$(jq -r -e ' + if type == "object" and length > 0 then + to_entries[] | "\(.key)\t\(.value.result // "missing")" + else + error("needs is not a non-empty object") + end')"; then + echo "::error::Audit gate could not read the needed jobs' results" + return 1 + fi + while IFS=$'\t' read -r job result; do + [ -n "$job" ] || continue + echo "${job}: ${result}" + if [ "$job" = "detect" ]; then + saw_detect=1 + if [ "$result" != "success" ]; then + echo "::error::Ecosystem detection did not succeed (${result}); no audit can be trusted" + status=1 + fi + continue + fi + case "$result" in + success | skipped) ;; + *) + echo "::error::Dependency audit job '${job}' did not pass (${result})" + status=1 + ;; + esac + done </dev/null \ - | xargs -n1 dirname 2>/dev/null \ - | sort -u - ) - - mapfile -t NON_WORKSPACES < <( - find . -name Cargo.toml -not -path '*/target/*' -print0 \ - | xargs -0 grep -L '^\[workspace\]' 2>/dev/null \ - | xargs -n1 dirname 2>/dev/null \ - | sort -u - ) - - # Standalone crates: those whose dir is not under any workspace root - STANDALONE=() - for dir in "${NON_WORKSPACES[@]}"; do - covered=0 - for ws in "${WORKSPACES[@]}"; do - # Trim trailing slash from $ws for accurate prefix match - ws_trim="${ws%/}" - case "$dir/" in - "$ws_trim/"*) covered=1; break ;; - esac - done - if [ "$covered" -eq 0 ]; then - STANDALONE+=("$dir") - fi - done - - for dir in "${WORKSPACES[@]}" "${STANDALONE[@]}"; do - echo "::group::cargo audit $dir" - if ! (cd "$dir" && cargo generate-lockfile 2>/dev/null; cargo audit); then - status=1 - fi - echo "::endgroup::" - done - exit $status + # One audit per workspace root that covers a Cargo.toml, as reported + # by `cargo metadata` (a crate the parent workspace excludes is its own + # root and is audited on its own, #1393). A committed Cargo.lock is + # audited as committed; one is generated only when none exists (#1394). + # shellcheck source=/dev/null + . ../.dependency-audit-tooling/.github/scripts/dependency-audit/lib/dependency-audit.sh + dep_audit_cargo_run . audit-pip: name: pip-audit @@ -222,39 +202,69 @@ jobs: if: needs.detect.outputs.pip == 'true' runs-on: ubuntu-latest steps: + # The audited repo goes in `repo/` so the tooling checkout below is never + # part of the tree being scanned. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: repo + persist-credentials: false + + # Same tooling source as audit-cargo. It also carries the hash-locked + # pip-audit requirements, which the audited repo does not have. + - name: Check out dependency-audit tooling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: combsmith/.github + ref: ${{ job.workflow_sha }} + path: .dependency-audit-tooling + fetch-depth: 1 + persist-credentials: false - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.x" - name: Install pip-audit - run: 'pip install --require-hashes --only-binary :all: -r scripts/pip-audit-requirements.txt' + run: 'pip install --require-hashes --only-binary :all: -r .dependency-audit-tooling/scripts/pip-audit-requirements.txt' - name: Audit Python dependencies + working-directory: repo run: | - status=0 # Audit each Python project found in the repo. Some projects ship # both pyproject.toml (tooling/build) and requirements.txt (pinned # runtime deps) — audit BOTH when both are present rather than - # short-circuiting on pyproject.toml. - while IFS= read -r dir; do - echo "::group::pip-audit $dir" - if [ -f "$dir/pyproject.toml" ]; then - if ! pip-audit "$dir"; then - status=1 - fi - fi - if [ -f "$dir/requirements.txt" ]; then - if ! pip-audit -r "$dir/requirements.txt"; then - status=1 - fi - fi - echo "::endgroup::" - done < <( - { - find . -name 'pyproject.toml' -not -path '*/.venv/*' -not -path '*/venv/*' -exec dirname {} \; - find . -name 'requirements.txt' -not -path '*/.venv/*' -not -path '*/venv/*' -exec dirname {} \; - } | sort -u - ) - exit $status + # short-circuiting on pyproject.toml. Discovery is NUL-delimited and + # a discovered project that cannot be audited fails the job (#1404). + # shellcheck source=/dev/null + . ../.dependency-audit-tooling/.github/scripts/dependency-audit/lib/dependency-audit.sh + dep_audit_python_run . + + # Aggregate result of the whole audit (#1406). `Detect ecosystems` succeeds + # as soon as discovery finishes, whatever the scanners find; this job fails + # when detection or any scanner failed or was cancelled (a scanner skipped + # because its ecosystem is absent is fine). `always()` — not `!cancelled()` — + # so a cancelled run reports a failure here instead of a skip. Every audit-* + # job MUST be listed in `needs` (pinned by test/workflows/dependency-audit/). + audit-gate: + name: Audit gate + needs: [detect, audit-npm, audit-pnpm, audit-go, audit-cargo, audit-pip] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out dependency-audit tooling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: combsmith/.github + ref: ${{ job.workflow_sha }} + path: .dependency-audit-tooling + fetch-depth: 1 + persist-credentials: false + + - name: Require every audit to pass + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: | + # shellcheck source=/dev/null + . .dependency-audit-tooling/.github/scripts/dependency-audit/lib/dependency-audit.sh + printf '%s' "$NEEDS_JSON" | dep_audit_gate_verdict diff --git a/.github/workflows/dependency-audit-tests.yml b/.github/workflows/dependency-audit-tests.yml new file mode 100644 index 000000000..782561fc9 --- /dev/null +++ b/.github/workflows/dependency-audit-tests.yml @@ -0,0 +1,67 @@ +# Quality gates for the reusable dependency-audit workflow and its supporting +# discovery/coverage/gate library. +# +# Triggered on any PR that touches: +# - .github/workflows/dependency-audit*.yml +# - .github/scripts/dependency-audit/** +# - test/workflows/dependency-audit/** +# - standards/workflows/dependency-audit.yml +# +# Gates (all must pass before merge): +# 1. shellcheck — static analysis for the library and the test helpers +# 2. bats — unit tests for the discovery/coverage/gate logic, plus the +# reusable's own `run:` steps executed against fixtures with +# stubbed scanners (#1393, #1394, #1404, #1406) + +name: Dependency Audit Tests + +on: + pull_request: + paths: + - '.github/workflows/dependency-audit*.yml' + - '.github/scripts/dependency-audit/**' + - 'test/workflows/dependency-audit/**' + - 'standards/workflows/dependency-audit.yml' + push: + branches: [main] + paths: + - '.github/workflows/dependency-audit*.yml' + - '.github/scripts/dependency-audit/**' + - 'test/workflows/dependency-audit/**' + - 'standards/workflows/dependency-audit.yml' + +permissions: {} + +concurrency: + group: dependency-audit-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Lint and bats + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Install bats, shellcheck, and jq (yq is preinstalled on the runner) + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends bats shellcheck jq + + - name: shellcheck + run: | + set -euo pipefail + shellcheck -x \ + .github/scripts/dependency-audit/lib/dependency-audit.sh \ + test/workflows/dependency-audit/helpers/setup.bash + + - name: Run bats suite + run: bats --print-output-on-failure test/workflows/dependency-audit/ diff --git a/standards/github-settings.md b/standards/github-settings.md index 480024f0f..60a6576f1 100644 --- a/standards/github-settings.md +++ b/standards/github-settings.md @@ -315,7 +315,7 @@ the reclassification table and sequencing note that follow. | **SonarCloud** | ✅ Required (codified) | `SonarCloud` | Code quality, maintainability, security hotspots. The context is exactly as codified in `code-quality.json`; verify with `gh pr checks ` if a repo's SonarCloud workflow reports under a different name | | **CodeQL** | ✅ Required (codified) | `CodeQL` | SAST via GitHub-managed default setup — auto-detects all supported languages (see [ci-standards.md §2](ci-standards.md#2-codeql-analysis-github-managed-default-setup)) | | **AgentShield** | ✅ Required (codified) | `agent-shield / AgentShield` | Deep agent-config security scan on every PR | -| **Dependency Audit** | ✅ Required (codified) | `dependency-audit / Detect ecosystems` | Only the unconditional `Detect ecosystems` job is required; per-ecosystem audit jobs are gated on lockfile presence and would fail as required-but-skipped | +| **Dependency Audit** | ✅ Required (codified) | `dependency-audit / Detect ecosystems` | Only the unconditional `Detect ecosystems` job is required; per-ecosystem audit jobs are gated on lockfile presence and would fail as required-but-skipped. `Detect ecosystems` passes once discovery finishes, whatever the scanners find. The reusable also publishes `dependency-audit / Audit gate`, which runs on every run and fails when detection or any audit job failed or was cancelled. It is not required yet: switching the required context to it is a pending decision (#1406), to be made only after the channel the fleet pins ships the job | The following are **intentionally NOT** in `code-quality.json` today: diff --git a/test/workflows/dependency-audit/dependency-audit.bats b/test/workflows/dependency-audit/dependency-audit.bats new file mode 100644 index 000000000..19b06ae2f --- /dev/null +++ b/test/workflows/dependency-audit/dependency-audit.bats @@ -0,0 +1,296 @@ +#!/usr/bin/env bats +# Tests for .github/scripts/dependency-audit/lib/dependency-audit.sh and the +# steps of .github/workflows/dependency-audit-reusable.yml that call it. +# +# Pins: +# #1393 a crate the parent workspace `exclude`s (or that is not a member) is +# audited on its own, not dropped because it sits under the +# workspace directory. +# #1394 a committed Cargo.lock is audited as committed, never regenerated. +# #1404 Python discovery is NUL-delimited; a directory whose name contains a +# newline is audited, and an unprocessable project fails the job. +# #1406 the aggregate `Audit gate` job fails when any audit failed or was +# cancelled, and depends on every audit job. + +load 'helpers/setup' + +setup() { + da_setup_stubs + # shellcheck source=/dev/null + . "$DA_LIB" + WS="${BATS_TEST_TMPDIR}/ws" + REPO="${WS}/repo" + mkdir -p "$REPO" +} + +# A Cargo workspace at $REPO with one member (crates/member) and one crate it +# excludes (tools/excluded), each with its own committed lockfile state. +make_cargo_fixture() { + mkdir -p "$REPO/crates/member" "$REPO/tools/excluded" + cat >"$REPO/Cargo.toml" <<'TOML' +[workspace] +members = ["crates/*"] +exclude = ["tools/excluded"] +TOML + echo '# workspace lock: clean' >"$REPO/Cargo.lock" + printf '[package]\nname = "member"\nversion = "0.1.0"\n' >"$REPO/crates/member/Cargo.toml" + printf '[package]\nname = "excluded"\nversion = "0.1.0"\n' >"$REPO/tools/excluded/Cargo.toml" + # Committed lockfile pins a vulnerable version. + echo '# excluded lock: VULN pinned' >"$REPO/tools/excluded/Cargo.lock" + da_ws_member "$REPO/crates/member" "$REPO" + da_ws_member "$REPO" "$REPO" +} + +# ── display ────────────────────────────────────────────────────────────────── + +@test "display: escapes CR and LF so a path cannot start a new log line" { + run dep_audit_display $'a\nb\rc' + [ "$status" -eq 0 ] + [ "$output" = 'a\nb\rc' ] +} + +# ── Python discovery (#1404) ──────────────────────────────────────────────── + +@test "python discovery: a directory name with a newline is one NUL-delimited record" { + mkdir -p "$REPO/svc"$'\n'"api" "$REPO/both" "$REPO/.venv/lib" "$REPO/x/venv/y" + touch "$REPO/svc"$'\n'"api/requirements.txt" "$REPO/both/pyproject.toml" \ + "$REPO/both/requirements.txt" "$REPO/.venv/lib/pyproject.toml" \ + "$REPO/x/venv/y/requirements.txt" + cd "$REPO" + local got=() d + while IFS= read -r -d '' d; do got+=("$d"); done < <(dep_audit_python_projects .) + [ "${#got[@]}" -eq 2 ] + [ "${got[0]}" = "./both" ] + [ "${got[1]}" = "./svc"$'\n'"api" ] +} + +@test "python discovery: a directory named like a manifest is not a project" { + mkdir -p "$REPO/requirements.txt" + cd "$REPO" + run dep_audit_python_projects . + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "python run: a vulnerable project under a newline-named directory fails the audit" { + mkdir -p "$REPO/svc"$'\n'"api" + echo 'VULN==1.0' >"$REPO/svc"$'\n'"api/requirements.txt" + cd "$REPO" + run dep_audit_python_run . + [ "$status" -eq 1 ] + da_calls | grep -qF 'pip-audit -r ./svc\napi/requirements.txt' +} + +@test "python run: audits pyproject.toml AND requirements.txt when both exist" { + mkdir -p "$REPO/both" + touch "$REPO/both/pyproject.toml" "$REPO/both/requirements.txt" + cd "$REPO" + run dep_audit_python_run . + [ "$status" -eq 0 ] + da_calls | grep -qxF "pip-audit ./both @ $(pwd -P)" + da_calls | grep -qxF "pip-audit -r ./both/requirements.txt @ $(pwd -P)" +} + +@test "python run: a discovered project with no readable manifest fails instead of skipping" { + mkdir -p "$REPO/broken" + ln -s does-not-exist "$REPO/broken/requirements.txt" + cd "$REPO" + run dep_audit_python_run . + [ "$status" -eq 1 ] + [[ "$output" == *"::error::Discovered Python project ./broken"* ]] + [ ! -s "$CALLS" ] +} + +@test "python run: a discovery failure fails the audit" { + [ "$(id -u)" -ne 0 ] || skip "root can read a mode-000 directory" + mkdir -p "$REPO/locked/inner" + touch "$REPO/locked/inner/requirements.txt" + chmod 000 "$REPO/locked" + cd "$REPO" + run dep_audit_python_run . + chmod 755 "$REPO/locked" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::Python project discovery failed"* ]] +} + +@test "python run: a crafted directory name cannot emit a workflow command line" { + mkdir -p "$REPO/x"$'\n'"::error::injected" + touch "$REPO/x"$'\n'"::error::injected/requirements.txt" + cd "$REPO" + run dep_audit_python_run . + [ "$status" -eq 0 ] + [ "$(printf '%s\n' "$output" | grep -cx '::error::injected')" -eq 0 ] + [[ "$output" == *'::group::pip-audit ./x\n::error::injected'* ]] +} + +# ── Cargo coverage (#1393) ────────────────────────────────────────────────── + +@test "cargo workspace_root: extracts the field verbatim" { + run dep_audit_cargo_workspace_root <<<'{"workspace_root":"/w/a\nb","packages":[]}' + [ "$status" -eq 0 ] + [ "$output" = "/w/a"$'\n'"b" ] +} + +@test "cargo workspace_root: missing, empty or non-string is an error" { + run dep_audit_cargo_workspace_root <<<'{"packages":[]}' + [ "$status" -ne 0 ] + run dep_audit_cargo_workspace_root <<<'{"workspace_root":""}' + [ "$status" -ne 0 ] + run dep_audit_cargo_workspace_root <<<'{"workspace_root":7}' + [ "$status" -ne 0 ] +} + +@test "cargo roots: an excluded crate under the workspace dir is its own audit root" { + make_cargo_fixture + cd "$REPO" + local got=() d + while IFS= read -r -d '' d; do got+=("$d"); done < <(dep_audit_cargo_audit_roots .) + [ "${#got[@]}" -eq 2 ] + [ "${got[0]}" = "$(pwd -P)" ] + [ "${got[1]}" = "$(pwd -P)/tools/excluded" ] +} + +@test "cargo roots: build output under target/ is ignored" { + mkdir -p "$REPO/target/debug/build/x" + printf '[package]\nname = "x"\n' >"$REPO/target/debug/build/x/Cargo.toml" + cd "$REPO" + run dep_audit_cargo_audit_roots . + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "cargo roots: a manifest cargo cannot resolve fails, other roots still listed" { + make_cargo_fixture + (cd "$REPO/tools/excluded" && pwd -P) >>"$CARGO_METADATA_FAIL" + cd "$REPO" + local out="${BATS_TEST_TMPDIR}/roots" err="${BATS_TEST_TMPDIR}/roots.err" rc=0 + dep_audit_cargo_audit_roots . >"$out" 2>"$err" || rc=$? + [ "$rc" -eq 1 ] + grep -qF '::error::cargo metadata failed for ./tools/excluded/Cargo.toml' "$err" + [ "$(tr '\0' '\n' <"$out")" = "$(pwd -P)" ] +} + +@test "cargo run: excluded crate is audited and its vulnerable committed lock fails the job" { + make_cargo_fixture + cd "$REPO" + run dep_audit_cargo_run . + [ "$status" -eq 1 ] + da_calls | grep -qxF "cargo audit @ $(pwd -P)/tools/excluded" + # Member crates are covered by the workspace audit, not re-audited. + [ "$(da_calls | grep -c '^cargo audit ')" -eq 2 ] +} + +# ── Cargo lockfile (#1394) ────────────────────────────────────────────────── + +@test "cargo audit dir: a committed Cargo.lock is audited as committed" { + mkdir -p "$REPO/c" + echo '# VULN pinned' >"$REPO/c/Cargo.lock" + run dep_audit_cargo_audit_dir "$REPO/c" + [ "$status" -eq 1 ] + [ "$(da_calls | grep -c generate-lockfile)" -eq 0 ] + [ "$(cat "$REPO/c/Cargo.lock")" = '# VULN pinned' ] +} + +@test "cargo audit dir: with no Cargo.lock one is generated, then audited" { + mkdir -p "$REPO/lib" + run dep_audit_cargo_audit_dir "$REPO/lib" + [ "$status" -eq 0 ] + [ "$(da_calls | sed 's/ @ .*//')" = "$(printf 'cargo generate-lockfile\ncargo audit')" ] +} + +# ── Gate verdict (#1406) ──────────────────────────────────────────────────── + +@test "gate: detect success with audits success/skipped passes" { + run dep_audit_gate_verdict <<<'{"detect":{"result":"success","outputs":{}}, + "audit-npm":{"result":"success"},"audit-pnpm":{"result":"skipped"}, + "audit-go":{"result":"skipped"},"audit-cargo":{"result":"success"}, + "audit-pip":{"result":"skipped"}}' + [ "$status" -eq 0 ] + [[ "$output" == *"audit-cargo: success"* ]] +} + +@test "gate: a failed audit fails the gate" { + run dep_audit_gate_verdict <<<'{"detect":{"result":"success"},"audit-pip":{"result":"failure"}}' + [ "$status" -eq 1 ] + [[ "$output" == *"::error::Dependency audit job 'audit-pip' did not pass (failure)"* ]] +} + +@test "gate: a cancelled audit fails the gate" { + run dep_audit_gate_verdict <<<'{"detect":{"result":"success"},"audit-go":{"result":"cancelled"}}' + [ "$status" -eq 1 ] +} + +@test "gate: detect anything but success fails, even when audits were skipped" { + local r + for r in failure cancelled skipped; do + run dep_audit_gate_verdict <<<"{\"detect\":{\"result\":\"$r\"},\"audit-npm\":{\"result\":\"skipped\"}}" + [ "$status" -eq 1 ] + done +} + +@test "gate: missing detect, missing result or malformed input fails closed" { + run dep_audit_gate_verdict <<<'{"audit-npm":{"result":"success"}}' + [ "$status" -eq 1 ] + run dep_audit_gate_verdict <<<'{"detect":{"result":"success"},"audit-npm":{}}' + [ "$status" -eq 1 ] + run dep_audit_gate_verdict <<<'{}' + [ "$status" -eq 1 ] + run dep_audit_gate_verdict <<<'not json' + [ "$status" -eq 1 ] + run dep_audit_gate_verdict "$REPO/svc"$'\n'"api/requirements.txt" + run da_run_step audit-pip "Audit Python dependencies" "$WS" + [ "$status" -ne 0 ] + da_calls | grep -qF 'requirements.txt' +} + +@test "workflow step: Require every audit to pass fails when an audit failed" { + export NEEDS_JSON='{"detect":{"result":"success"},"audit-cargo":{"result":"failure"}}' + run da_run_step audit-gate "Require every audit to pass" "$WS" + [ "$status" -eq 1 ] + export NEEDS_JSON='{"detect":{"result":"success"},"audit-cargo":{"result":"skipped"}}' + run da_run_step audit-gate "Require every audit to pass" "$WS" + [ "$status" -eq 0 ] +} diff --git a/test/workflows/dependency-audit/helpers/setup.bash b/test/workflows/dependency-audit/helpers/setup.bash new file mode 100644 index 000000000..f2ce95f96 --- /dev/null +++ b/test/workflows/dependency-audit/helpers/setup.bash @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Common helpers for the dependency-audit bats suite. +# +# Scanners are replaced by stubs on PATH that record every call: +# cargo metadata -> answers workspace_root from $CARGO_WS_MAP +# ("\t" lines, both as +# `pwd -P` prints them); the manifest's own directory +# when it is not listed. Fails for a manifest dir +# listed in $CARGO_METADATA_FAIL. +# generate-lockfile -> writes a clean Cargo.lock +# audit -> fails when ./Cargo.lock contains "VULN" +# pip-audit fails when the audited pyproject.toml / requirements file +# contains "VULN" +# Calls are appended to $CALLS, one record per call, NUL-terminated, as +# " @ ". + +DA_REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../../.." && pwd)" +DA_LIB="${DA_REPO_ROOT}/.github/scripts/dependency-audit/lib/dependency-audit.sh" +DA_WORKFLOW="${DA_REPO_ROOT}/.github/workflows/dependency-audit-reusable.yml" +export DA_REPO_ROOT DA_LIB DA_WORKFLOW + +da_setup_stubs() { + STUB_BIN="${BATS_TEST_TMPDIR}/bin" + CALLS="${BATS_TEST_TMPDIR}/calls" + CARGO_WS_MAP="${BATS_TEST_TMPDIR}/ws-map" + CARGO_METADATA_FAIL="${BATS_TEST_TMPDIR}/metadata-fail" + mkdir -p "$STUB_BIN" + : >"$CALLS" + : >"$CARGO_WS_MAP" + : >"$CARGO_METADATA_FAIL" + export STUB_BIN CALLS CARGO_WS_MAP CARGO_METADATA_FAIL + + cat >"$STUB_BIN/cargo" <<'STUB' +#!/usr/bin/env bash +printf 'cargo %s @ %s\0' "$*" "$(pwd -P)" >>"$CALLS" +case "$1" in + metadata) + manifest="" + while [ $# -gt 0 ]; do + [ "$1" = "--manifest-path" ] && manifest="$2" + shift + done + dir="$(cd -- "${manifest%/*}" && pwd -P)" || exit 101 + if grep -qxF -- "$dir" "$CARGO_METADATA_FAIL"; then + echo "error: failed to parse manifest" >&2 + exit 101 + fi + ws="$dir" + while IFS=$'\t' read -r m r; do + [ "$m" = "$dir" ] && ws="$r" + done <"$CARGO_WS_MAP" + jq -n --arg ws "$ws" '{packages: [], workspace_root: $ws}' + ;; + generate-lockfile) + echo "# generated: clean" >Cargo.lock + ;; + audit) + [ -f Cargo.lock ] || { echo "error: no Cargo.lock" >&2; exit 1; } + ! grep -q VULN Cargo.lock + ;; + *) exit 2 ;; +esac +STUB + + cat >"$STUB_BIN/pip-audit" <<'STUB' +#!/usr/bin/env bash +printf 'pip-audit %s @ %s\0' "$*" "$(pwd -P)" >>"$CALLS" +if [ "$1" = "-r" ]; then + target="$2" +else + target="$1/pyproject.toml" +fi +[ -f "$target" ] || exit 2 +! grep -q VULN "$target" +STUB + chmod +x "$STUB_BIN/cargo" "$STUB_BIN/pip-audit" + PATH="${STUB_BIN}:${PATH}" + export PATH +} + +# Prints the recorded calls, one per line, with embedded newlines shown as \n. +da_calls() { + local c + while IFS= read -r -d '' c; do + printf '%s\n' "${c//$'\n'/\\n}" + done <"$CALLS" +} + +# da_ws_member MANIFEST_DIR WORKSPACE_DIR — tell the cargo stub that the crate +# in MANIFEST_DIR belongs to the workspace rooted at WORKSPACE_DIR. +da_ws_member() { + printf '%s\t%s\n' "$(cd -- "$1" && pwd -P)" "$(cd -- "$2" && pwd -P)" >>"$CARGO_WS_MAP" +} + +# da_step_script JOB_ID STEP_NAME — prints the `run:` script of that step of +# the reusable workflow, exactly as committed. +da_step_script() { + JOB="$1" STEP="$2" yq -r \ + '.jobs[strenv(JOB)].steps[] | select(.name == strenv(STEP)) | .run' \ + "$DA_WORKFLOW" +} + +# da_run_step JOB_ID STEP_NAME WORKSPACE — runs that step the way the runner +# does (`bash -e`, in the step's working-directory under WORKSPACE), with the +# tooling checkout at WORKSPACE/.dependency-audit-tooling pointing at this repo. +da_run_step() { + local job="$1" step="$2" ws="$3" wd script + script="${BATS_TEST_TMPDIR}/step.sh" + da_step_script "$job" "$step" >"$script" + [ -s "$script" ] || { echo "step '$step' not found in job '$job'" >&2; return 99; } + wd="$(JOB="$job" STEP="$step" yq -r \ + '.jobs[strenv(JOB)].steps[] | select(.name == strenv(STEP)) | .["working-directory"] // ""' \ + "$DA_WORKFLOW")" + ln -sfn "$DA_REPO_ROOT" "$ws/.dependency-audit-tooling" + (cd -- "$ws/${wd:-.}" && bash -e "$script") +} diff --git a/test/workflows/dependency-hardening.bats b/test/workflows/dependency-hardening.bats index 2dd02c680..e5c330ab6 100644 --- a/test/workflows/dependency-hardening.bats +++ b/test/workflows/dependency-hardening.bats @@ -96,9 +96,11 @@ workflow_command_lines() { path="${path#\"}" path="${path%\'}" path="${path%\"}" - # Reusable workflows check the tooling out under .feature-ideation-tooling/; - # map that runtime prefix back to the repo root for the existence check. + # Reusable workflows check the tooling out under .feature-ideation-tooling/ + # or .dependency-audit-tooling/; map that runtime prefix back to the repo + # root for the existence check. path="${path#.feature-ideation-tooling/}" + path="${path#.dependency-audit-tooling/}" if [ -z "$path" ]; then echo "EMPTY path extracted from: ${file##*/}: $cmd"; fail=1 elif [ ! -f "$REPO_ROOT/$path" ]; then