diff --git a/.github/workflows/ci-failure-analyst-reusable.yml b/.github/workflows/ci-failure-analyst-reusable.yml index f7c368595..0a29d45c9 100644 --- a/.github/workflows/ci-failure-analyst-reusable.yml +++ b/.github/workflows/ci-failure-analyst-reusable.yml @@ -36,52 +36,36 @@ jobs: REPO: ${{ github.repository }} steps: + # The payload resolution lives in scripts/ci-failure-analyst-resolve.sh + # (tests: tests/ci_failure_analyst_resolve.bats). Fetch it from THIS + # reusable's own commit, so a caller pinned to a channel always runs the + # script that matches the workflow it pinned. .github-private is public, + # so the caller's token can read it. + - name: Checkout analyst tooling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: combsmith/.github-private + ref: ${{ job.workflow_sha }} + path: .ci-analyst-tooling + sparse-checkout: scripts/ci-failure-analyst-resolve.sh + sparse-checkout-cone-mode: false + fetch-depth: 1 + persist-credentials: false + + # Resolves the head SHA, the non-fork PR and the failing run from either a + # check_run payload (third-party checks) or a workflow_run payload (failed + # GitHub Actions runs). Anything else ends in a "skipping" notice, exit 0. - name: Resolve non-fork PR id: pr - env: - HEAD_SHA: ${{ github.event.check_run.head_sha }} - CHECK_RUN_PRS: ${{ toJson(github.event.check_run.pull_requests) }} - run: | - set -euo pipefail - # Prefer the pull_requests array embedded in the check_run payload; - # filter to non-fork PRs (head repo must equal base repo). - pr_number=$(echo "$CHECK_RUN_PRS" \ - | jq -r '.[] | select(.head.repo.id == .base.repo.id) | .number' \ - | head -n 1) - if [ -z "$pr_number" ]; then - # Fallback: commits-to-pulls API handles edge cases (e.g. a second - # push arrived before this check_run completed). - # shellcheck disable=SC2016 # $ENV.REPO is jq built-in syntax, not a shell variable - pr_number=$(gh api "repos/$REPO/commits/$HEAD_SHA/pulls" \ - --jq '.[] | select(.head.repo.full_name == $ENV.REPO) | .number' \ - 2>/dev/null | head -n 1 || true) - fi - if [ -z "$pr_number" ]; then - echo "::notice::No non-fork PR for SHA $HEAD_SHA — skipping" - echo "skip=true" >> "$GITHUB_OUTPUT" - else - echo "skip=false" >> "$GITHUB_OUTPUT" - echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT" - fi + run: bash .ci-analyst-tooling/scripts/ci-failure-analyst-resolve.sh resolve - name: Check idempotency if: steps.pr.outputs.skip == 'false' id: idem env: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} - HEAD_SHA: ${{ github.event.check_run.head_sha }} - run: | - set -euo pipefail - existing=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \ - --paginate \ - --jq ".[] | select(.body | startswith(\"\")) | .id" \ - 2>/dev/null | head -1 || true) - if [ -n "$existing" ]; then - echo "::notice::Diagnostic comment already posted (id=$existing) — idempotency skip" - echo "exists=true" >> "$GITHUB_OUTPUT" - else - echo "exists=false" >> "$GITHUB_OUTPUT" - fi + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: bash .ci-analyst-tooling/scripts/ci-failure-analyst-resolve.sh idempotency - name: Install Claude Code if: steps.pr.outputs.skip == 'false' && steps.idem.outputs.exists == 'false' @@ -97,10 +81,11 @@ jobs: if: steps.pr.outputs.skip == 'false' && steps.idem.outputs.exists == 'false' env: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} - HEAD_SHA: ${{ github.event.check_run.head_sha }} - WORKFLOW_NAME: ${{ github.event.check_run.name }} - DETAILS_URL: ${{ github.event.check_run.details_url }} - CHECK_RUN_ID: ${{ github.event.check_run.id }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + WORKFLOW_NAME: ${{ steps.pr.outputs.check_name }} + DETAILS_URL: ${{ steps.pr.outputs.details_url }} + RUN_ID: ${{ steps.pr.outputs.run_id }} + FAILED_JOBS: ${{ steps.pr.outputs.failed_jobs }} run: | set -euo pipefail export PATH="$HOME/.npm-global/bin:$PATH" @@ -113,16 +98,15 @@ jobs: - Head SHA: $HEAD_SHA - Workflow/check name: $WORKFLOW_NAME - Details URL: $DETAILS_URL - - Check run ID: $CHECK_RUN_ID + - GitHub Actions run ID: $RUN_ID + - Failed jobs (job / failing step): $FAILED_JOBS Follow these steps in order: ### 1. Fetch failure logs - Extract the GitHub Actions run ID from the details URL: - run_id=$(echo "$DETAILS_URL" | grep -oP '(?<=runs/)\d+' || true) - If a run_id is found, fetch the failed-step logs: - gh run view "$run_id" --log-failed --repo $REPO 2>/dev/null | tail -200 - If no run_id (e.g. external quality gate like SonarCloud), use the PR diff: + If the run ID above is not empty, fetch the failed-step logs: + gh run view $RUN_ID --log-failed --repo $REPO 2>/dev/null | tail -200 + If the run ID is empty (e.g. an external quality gate like SonarCloud), use the PR diff: gh pr diff $PR_NUMBER --repo $REPO 2>/dev/null | head -200 ### 2. Identify the failing step and error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 571a600e4..97d86aed3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,9 @@ jobs: chmod +x actionlint rm actionlint.tar.gz shopt -s extglob - ./actionlint -ignore 'SC2129' .github/workflows/!(*.lock).yml + # `workflow_sha` is a valid `job` context property (the + # reusable's own commit); the actionlint schema omits it. + ./actionlint -ignore 'SC2129' -ignore '"workflow_sha" is not defined' .github/workflows/!(*.lock).yml shellcheck: name: ShellCheck diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3a34195b9..baaf3ce70 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -124,6 +124,7 @@ jobs: tests/fleet_monitor_run.bats \ tests/test_fleet_paused_run.bats \ tests/template_stub_drift.bats \ + tests/ci_failure_analyst_resolve.bats \ tests/canary_rollout.bats \ tests/test_initiative_canary.bats \ tests/test_initiative_driver_canary.bats \ @@ -336,7 +337,9 @@ jobs: chmod +x actionlint rm actionlint.tar.gz shopt -s extglob - ./actionlint -ignore 'SC2129' .github/workflows/!(*.lock).yml + # `workflow_sha` is a valid `job` context property (the + # reusable's own commit); the actionlint schema omits it. + ./actionlint -ignore 'SC2129' -ignore '"workflow_sha" is not defined' .github/workflows/!(*.lock).yml validate-workflow-schedules: # Off-peak scheduling compliance (#724, epic #722): fail any PR that diff --git a/docs/aw/ci-failure-analyst.md b/docs/aw/ci-failure-analyst.md index 495e6ecd9..55d5e0eb1 100644 --- a/docs/aw/ci-failure-analyst.md +++ b/docs/aw/ci-failure-analyst.md @@ -4,9 +4,9 @@ Agentic workflow that diagnoses CI failures and posts a root-cause comment on th ## What it does -When a check run completes with `conclusion: failure` on a non-fork PR, the workflow: +When a check run or a watched GitHub Actions workflow run completes with `conclusion: failure` on a non-fork PR, the workflow: -1. Resolves the PR number from the check run payload (with a commits-to-pulls API fallback) +1. Resolves the head SHA, the PR number and the failing run from the `check_run` or `workflow_run` payload, with a commits-to-pulls API fallback for the PR (open PRs only) 2. Checks idempotency — skips if a `` comment already exists for this SHA 3. Fetches the failed run logs from the GitHub Actions API 4. Identifies the specific step that failed and its error message @@ -88,6 +88,81 @@ if: >- !startsWith(github.event.check_run.name, 'CI Failure Analyst') ``` +## Which failures are covered + +| Failure | Event | Covered | +|---|---|---| +| A check from another app (SonarCloud and the like) | `check_run` | Yes, through the current stub | +| A failed GitHub Actions job (CI, Lint, Tests, …) | `workflow_run` | Once the stub carries the `workflow_run` trigger below | + +GitHub does not start a workflow for an event raised by GitHub Actions' own token, so a check run +created by an Actions job never delivers `check_run` to the stub. The `check_run` trigger therefore +only ever sees third-party checks. Failed Actions runs reach the analyst through `workflow_run` (#2235). + +The reusable accepts either payload. `scripts/ci-failure-analyst-resolve.sh` (tests: +`tests/ci_failure_analyst_resolve.bats`) reads it and resolves: + +- **`check_run`**: head SHA, PR (the payload's non-fork `pull_requests`), check name, details URL, and + the Actions run id when the details URL points at one. A third-party check has no run, so the + analysis falls back to the PR diff. +- **`workflow_run`**: head SHA from `workflow_run.head_sha`, PR from `workflow_run.pull_requests` + (non-fork only), run id from `workflow_run.id`, and the run's failed jobs and their failing steps. + +When the payload has no non-fork PR, the commits-to-pulls lookup is tried, counting open PRs only. +A failure on the default branch is therefore not reported on the PR already merged into it. A +payload that carries neither event, or a run with no open non-fork PR, ends in a "skipping" notice +with exit 0. + +The reusable fetches the script from its own commit (`job.workflow_sha`), so a caller pinned +to a channel runs the script that matches the workflow it pinned. + +### The `workflow_run` trigger and the watched-workflow list + +The stub adds `workflow_run` next to `check_run`: + +```yaml +on: + check_run: + types: [completed] + # Each repo may edit this list: name every workflow whose failures should be + # analysed. GitHub requires names (the `name:` of each workflow file); there + # is no wildcard. Never list "CI Failure Analyst" itself. + workflow_run: + workflows: ["CI", "Lint", "Tests"] + types: [completed] + +concurrency: + group: "ci-failure-analyst-${{ github.event.check_run.head_sha || github.event.workflow_run.head_sha }}" + cancel-in-progress: false + +jobs: + analyze: + if: >- + (github.event_name == 'check_run' && + github.event.check_run.conclusion == 'failure' && + !startsWith(github.event.check_run.name, 'CI Failure Analyst')) || + (github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'failure' && + !startsWith(github.event.workflow_run.name, 'CI Failure Analyst')) +``` + +To change which Actions failures are analysed, edit the `workflows:` list in your repo's stub. Each +entry must match the target workflow's `name:` exactly. A workflow that is not listed is not analysed. +The `workflows:` list is the one part of the stub a repo is expected to change. + +When several watched workflows fail on the same commit, each failure starts an analyst run. The +per-SHA concurrency group runs them one at a time, and the `` marker +makes every run after the first an idempotency skip, giving one comment per commit. + +**Rollout order.** The stubs pin `ci-failure-analyst/v1-stable`. A trigger takes effect as soon as +the stub merges, but the reusable it calls is whatever that channel points at. The reusable at +v1.0.0 reads only `check_run` and fails a `workflow_run` call with `jq: Cannot iterate over null`. +So the `workflow_run` trigger goes into `templates/ci-failure-analyst.yml` and this repo's +`ci-failure-analyst.lock.yml` only after the pinned channel has been promoted to a commit carrying +the dual-payload reusable. That stub change also needs `caller-stub-freeze`'s baseline regenerated, +plus matching updates to `interaction-contracts/ci-failure-analyst.yml` and the §4 row of the +[agentic interaction model](../agentic-interaction-model.md). + ## Root cause categories | Category | Indicators | diff --git a/scripts/ci-failure-analyst-resolve.sh b/scripts/ci-failure-analyst-resolve.sh new file mode 100755 index 000000000..3f45f4b64 --- /dev/null +++ b/scripts/ci-failure-analyst-resolve.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -euo pipefail +# ci-failure-analyst-resolve.sh — payload resolution for the CI Failure Analyst +# reusable (.github/workflows/ci-failure-analyst-reusable.yml, #2235). +# +# The reusable is called from a stub that may fire on either event: +# check_run — a check created by another app (SonarCloud and the like). +# GitHub does not deliver check_run for checks created by +# GitHub Actions' own token, so this never sees Actions jobs. +# workflow_run — a GitHub Actions run completed; this is how the analyst +# learns about failed Actions jobs. +# The event is detected from the payload itself ($GITHUB_EVENT_PATH; a reusable +# sees its caller's payload), so a payload that carries neither ends in the +# "skipping" notice with exit 0 instead of a jq iteration error. +# +# Subcommands (results go to $GITHUB_OUTPUT): +# resolve skip=true|false, and when false: event, pr_number, head_sha, +# check_name, details_url, run_id, check_run_id, failed_jobs +# idempotency exists=true|false — whether PR_NUMBER already has a comment +# starting with ``. That marker +# keeps it to one analysis per commit however many workflows fail. +# +# Env: REPO (owner/repo), GITHUB_EVENT_PATH, GITHUB_OUTPUT, GH_TOKEN for gh; +# PR_NUMBER and HEAD_SHA for `idempotency`. + +REPO="${REPO:-${GITHUB_REPOSITORY:-}}" +GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" +export REPO + +# emit — write a key=value pair to GITHUB_OUTPUT, stripping carriage returns. +emit() { printf '%s=%s\n' "$1" "$(printf '%s' "$2" | tr -d '\r\n')" >> "$GITHUB_OUTPUT"; } + +# payload_field — read one value from the event payload; empty on a +# missing, empty or non-JSON payload. +payload_field() { + local event_path="${GITHUB_EVENT_PATH:-}" + [ -n "$event_path" ] && [ -s "$event_path" ] || return 0 + jq -r "$1 // empty" "$event_path" 2>/dev/null || true +} + +# failed_jobs — "job / failing step; job; ..." for the run's failed jobs. +# A failed jobs lookup returns nonzero rather than reading as "no failed jobs". +failed_jobs() { + [ -n "$1" ] || return 0 + local jobs + jobs="$(gh api "repos/$REPO/actions/runs/$1/jobs" --paginate \ + --jq '.jobs[]? | select(.conclusion == "failure") + | .name + ([(.steps // [])[] | select(.conclusion == "failure") | " / " + .name][0] // "")')" || return 1 + printf '%s\n' "$jobs" | paste -sd';' - | sed 's/;/; /g' +} + +# resolve — parse the GitHub event payload and emit outputs needed for analysis. +# Detects check_run or workflow_run events, resolves to an open PR, collects failed jobs. +# Emits skip=true if no open PR is found or the payload is unsupported. +resolve() { + local event key head_sha pr_number pr_state pulls candidates candidate name details_url jobs_summary run_id="" check_run_id="" + event="$(payload_field 'if (.check_run | type) == "object" then "check_run" + elif (.workflow_run | type) == "object" then "workflow_run" + else null end')" + case "$event" in + check_run) key=check_run ;; + workflow_run) key=workflow_run ;; + *) + echo "::notice::Event payload carries neither a check_run nor a workflow_run — skipping" + emit skip true + return 0 + ;; + esac + + head_sha="$(payload_field ".$key.head_sha")" + if [ -z "$head_sha" ]; then + echo "::notice::No head SHA in the $event payload — skipping" + emit skip true + return 0 + fi + + # Prefer the pull_requests array embedded in the payload, filtered to + # non-fork PRs (head repo must equal base repo). The embedded entries are + # minimal objects with no `state` field, so openness is re-checked against + # the API: the run may finish after its PR merged or closed. + # A head SHA can belong to several PRs, so every candidate is checked and the + # first open one wins; with none open, the commits-to-pulls lookup below runs. + candidates="$(payload_field '(.'"$key"'.pull_requests // [])[] + | select(.head.repo.id == .base.repo.id) | .number')" + pr_number="" + while IFS= read -r candidate; do + [ -n "$candidate" ] || continue + # A failed lookup is not "closed": it fails the step rather than skipping. + if ! pr_state="$(gh api "repos/$REPO/pulls/$candidate" --jq '.state')"; then + echo "::error::Could not look up the state of PR #$candidate" + return 1 + fi + if [ "$pr_state" = "open" ]; then + pr_number="$candidate" + break + fi + done <<< "$candidates" + if [ -z "$pr_number" ]; then + # Fallback: the commits-to-pulls API covers a second push that arrived + # before this event, and a workflow_run whose payload omits the PR. Only + # an open PR counts, so a failure on the default branch is not reported + # on the PR that was already merged into it. A failed lookup fails the step + # rather than reading as "no PR". + # shellcheck disable=SC2016 # $ENV.REPO is jq built-in syntax, not a shell variable + if ! pulls="$(gh api "repos/$REPO/commits/$head_sha/pulls" \ + --jq '.[] | select(.state == "open" and .head.repo.full_name == $ENV.REPO) | .number')"; then + echo "::error::Could not look up the PRs for SHA $head_sha" + return 1 + fi + pr_number="$(printf '%s\n' "$pulls" | sed -n '1p')" + fi + if [ -z "$pr_number" ]; then + echo "::notice::No non-fork PR for SHA $head_sha — skipping" + emit skip true + return 0 + fi + + name="$(payload_field ".$key.name")" + if [ "$event" = check_run ]; then + details_url="$(payload_field ".check_run.details_url // .check_run.html_url")" + check_run_id="$(payload_field ".check_run.id")" + # An Actions-hosted check links to its run; a third-party check does not. + run_id="$(printf '%s' "$details_url" | grep -oE '/actions/runs/[0-9]+' | grep -oE '[0-9]+$' || true)" + else + details_url="$(payload_field ".workflow_run.html_url")" + run_id="$(payload_field ".workflow_run.id")" + fi + + if ! jobs_summary="$(failed_jobs "$run_id")"; then + echo "::error::Could not look up the failed jobs for run $run_id" + return 1 + fi + + emit skip false + emit event "$event" + emit pr_number "$pr_number" + emit head_sha "$head_sha" + emit check_name "$name" + emit details_url "$details_url" + emit run_id "$run_id" + emit check_run_id "$check_run_id" + emit failed_jobs "$jobs_summary" +} + +# idempotency — check whether a comment with the SHA marker already exists on the PR. +# Prevents duplicate analysis comments for the same commit across multiple failures. +# Fails closed (assumes exists=true) on API errors to avoid posting duplicates. +idempotency() { + local existing ids + # Fail closed: a failed lookup must not read as "no marker yet", or a second + # diagnostic gets posted for a commit that already has one. + if ! ids="$(gh api "repos/$REPO/issues/${PR_NUMBER:?PR_NUMBER is required}/comments" \ + --paginate \ + --jq ".[] | select(.body | startswith(\"\")) | .id" \ + 2>/dev/null)"; then + echo "::warning::Comment lookup failed — skipping to avoid a duplicate diagnostic" + emit exists true + return 0 + fi + existing="$(printf '%s\n' "$ids" | sed -n '1p')" + if [ -n "$existing" ]; then + echo "::notice::Diagnostic comment already posted (id=$existing) — idempotency skip" + emit exists true + else + emit exists false + fi +} + +case "${1:-}" in + resolve) resolve ;; + idempotency) idempotency ;; + *) + echo "usage: $0 resolve|idempotency" >&2 + exit 2 + ;; +esac diff --git a/tests/aw/ci-failure-analyst/scenarios.md b/tests/aw/ci-failure-analyst/scenarios.md index d612d9869..827dd08fd 100644 --- a/tests/aw/ci-failure-analyst/scenarios.md +++ b/tests/aw/ci-failure-analyst/scenarios.md @@ -120,3 +120,44 @@ Written before the workflow per the TDD cycle in issue #233. - Workflow is a **no-op** — no comment is posted anywhere. - The analyst must not comment on issues or any other thread. + +--- + +## Scenario 3: A watched GitHub Actions workflow fails on a PR (`workflow_run`, #2235) + +**Input event:** + +```json +{ + "action": "completed", + "workflow_run": { + "id": 3000, + "name": "Lint", + "conclusion": "failure", + "head_sha": "www111", + "html_url": "https://github.com/combsmith/example/actions/runs/3000", + "pull_requests": [ + { "number": 77, "head": { "repo": { "id": 1 } }, "base": { "repo": { "id": 1 } } } + ] + } +} +``` + +**Expected output:** + +- A comment is posted to PR #77. +- Comment begins with ``. +- The failing job and step come from run `3000`'s failed jobs and `--log-failed` logs. +- Comment includes a link to `https://github.com/combsmith/example/actions/runs/3000`. +- If another watched workflow (or a third-party check) also fails on `www111`, no second comment is posted. + +--- + +## Edge case: `workflow_run` on the default branch, or a payload with neither event + +**Input event:** a `workflow_run` for a push to `main` (`pull_requests: []`, and the commit's only +associated PR is already merged), or a payload with no `check_run` and no `workflow_run` key. + +**Expected output:** + +- Workflow is a **no-op**: a "skipping" notice and exit 0, never `jq: Cannot iterate over null`. diff --git a/tests/ci_failure_analyst_resolve.bats b/tests/ci_failure_analyst_resolve.bats new file mode 100644 index 000000000..fa8abd92f --- /dev/null +++ b/tests/ci_failure_analyst_resolve.bats @@ -0,0 +1,381 @@ +#!/usr/bin/env bats +# Tests for scripts/ci-failure-analyst-resolve.sh (#2235). +# +# The CI Failure Analyst reusable is called from a stub that triggers on +# check_run (third-party checks such as SonarCloud) and, once the stubs gain it, +# on workflow_run (failed GitHub Actions runs). The resolver reads whichever +# payload the run carries and must end every unusable payload in the +# "skipping" notice with exit 0, never in `jq: Cannot iterate over null`. +# +# gh is stubbed: each `gh api ` call is answered from a fixture file and +# the call's own --jq filter is applied with real jq, so the filters the script +# passes are exercised, not mocked away. + +REPO_ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" +SCRIPT="$REPO_ROOT/scripts/ci-failure-analyst-resolve.sh" + +SHA="0123456789abcdef0123456789abcdef01234567" + +# setup — initialize test environment with temporary directories, gh stub, and fixtures. +setup() { + TMP="$(mktemp -d)" || { echo "Failed to create temp directory" >&2; exit 1; } + STUB_BIN="$TMP/bin" + GH_FIXTURES="$TMP/fixtures" + GH_CALLS="$TMP/gh-calls.log" + mkdir -p "$STUB_BIN" "$GH_FIXTURES" + : > "$GH_CALLS" + export GH_FIXTURES GH_CALLS + + # gh stub: `gh api [--paginate] [--jq ]`. The fixture for + # is $GH_FIXTURES/.json; a missing fixture is a + # failed call (like a 404). + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +echo "$*" >> "$GH_CALLS" +[ "${1:-}" = "api" ] || exit 1 +shift +path="" expr="" +while [ $# -gt 0 ]; do + case "$1" in + --jq) expr="$2"; shift 2 ;; + --paginate) shift ;; + -*) shift ;; + *) [ -z "$path" ] && path="$1"; shift ;; + esac +done +fixture="$GH_FIXTURES/$(printf '%s' "$path" | tr '/' '_').json" +[ -f "$fixture" ] || { echo "gh: Not Found (HTTP 404)" >&2; exit 1; } +if [ -n "$expr" ]; then + jq -r "$expr" "$fixture" +else + cat "$fixture" +fi +GHEOF + chmod +x "$STUB_BIN/gh" + export PATH="$STUB_BIN:$PATH" + + export REPO="combsmith/.github-private" + export GITHUB_OUTPUT="$TMP/output" + : > "$GITHUB_OUTPUT" + export GITHUB_EVENT_PATH="$TMP/event.json" + unset PR_NUMBER HEAD_SHA +} + +# teardown — clean up temporary directory and all test artifacts. +teardown() { + [ -n "${TMP:-}" ] && rm -rf "$TMP" +} + +# fixture — write a mock gh api response fixture for testing. +fixture() { + printf '%s\n' "$2" > "$GH_FIXTURES/$(printf '%s' "$1" | tr '/' '_').json" +} + +# out — retrieve the last value written for to GITHUB_OUTPUT. +out() { + grep "^$1=" "$GITHUB_OUTPUT" | tail -n 1 | cut -d= -f2- +} + +check_run_event() { + cat > "$GITHUB_EVENT_PATH" < [head_branch] +workflow_run_event() { + cat > "$GITHUB_EVENT_PATH" < "$TMP/e" && mv "$TMP/e" "$GITHUB_EVENT_PATH" + failed_jobs_fixture + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out run_id)" = "777" ] + [ "$(out failed_jobs)" = "shellcheck / Run tests; template-drift" ] +} + +@test "check_run: a failed jobs lookup fails the step instead of emitting empty failed_jobs" { + check_run_event + fixture "repos/$REPO/pulls/42" '{"state": "open"}' + jq '.check_run.details_url = "https://github.com/combsmith/.github-private/actions/runs/777/job/9"' \ + "$GITHUB_EVENT_PATH" > "$TMP/e" && mv "$TMP/e" "$GITHUB_EVENT_PATH" + run bash "$SCRIPT" resolve + [ "$status" -ne 0 ] + [[ "$output" == *"Could not look up the failed jobs"* ]] + [ ! -s "$GITHUB_OUTPUT" ] +} + +@test "check_run: empty pull_requests falls back to the commits-to-pulls lookup" { + check_run_event + jq '.check_run.pull_requests = []' "$GITHUB_EVENT_PATH" > "$TMP/e" && mv "$TMP/e" "$GITHUB_EVENT_PATH" + fixture "repos/$REPO/commits/$SHA/pulls" '[ + {"number": 43, "state": "open", "head": {"repo": {"full_name": "combsmith/.github-private"}}} + ]' + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "false" ] + [ "$(out pr_number)" = "43" ] +} + +# ── workflow_run ───────────────────────────────────────────────────────────── + +@test "workflow_run: SHA, PR, run id and failed jobs come from the workflow_run payload" { + workflow_run_event '[{"number": 99, "state": "open", "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}]' + fixture "repos/$REPO/pulls/99" '{"state": "open"}' + failed_jobs_fixture + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "false" ] + [ "$(out event)" = "workflow_run" ] + [ "$(out pr_number)" = "99" ] + [ "$(out head_sha)" = "$SHA" ] + [ "$(out check_name)" = "Lint" ] + [ "$(out details_url)" = "https://github.com/combsmith/.github-private/actions/runs/777" ] + [ "$(out run_id)" = "777" ] + [ "$(out check_run_id)" = "" ] + [ "$(out failed_jobs)" = "shellcheck / Run tests; template-drift" ] +} + +@test "workflow_run: a fork PR in the payload is ignored and the lookup decides" { + workflow_run_event '[{"number": 5, "head": {"repo": {"id": 8}}, "base": {"repo": {"id": 7}}}]' + fixture "repos/$REPO/commits/$SHA/pulls" '[ + {"number": 5, "state": "open", "head": {"repo": {"full_name": "someone/fork"}}} + ]' + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] + [[ "$output" == *"::notice::No non-fork PR for SHA $SHA"*"skipping"* ]] +} + +@test "workflow_run: empty pull_requests falls back to the commits-to-pulls lookup" { + workflow_run_event '[]' + fixture "repos/$REPO/commits/$SHA/pulls" '[ + {"number": 101, "state": "open", "head": {"repo": {"full_name": "combsmith/.github-private"}}} + ]' + failed_jobs_fixture + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "false" ] + [ "$(out pr_number)" = "101" ] +} + +@test "workflow_run on the default branch with no PR: skipping notice, exit 0" { + workflow_run_event '[]' main + # A push to main: the commit's only associated PR is the one already merged. + fixture "repos/$REPO/commits/$SHA/pulls" '[ + {"number": 88, "state": "closed", "head": {"repo": {"full_name": "combsmith/.github-private"}}} + ]' + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] + [ "$(out pr_number)" = "" ] + [[ "$output" == *"::notice::No non-fork PR for SHA $SHA"*"skipping"* ]] +} + +@test "workflow_run with null pull_requests never reaches a jq iteration error" { + workflow_run_event 'null' + fixture "repos/$REPO/commits/$SHA/pulls" '[]' + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] + [[ "$output" != *"Cannot iterate"* ]] +} + +@test "workflow_run: an embedded PR that has since closed is not reported" { + workflow_run_event '[{"number": 99, "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}]' + fixture "repos/$REPO/pulls/99" '{"state": "closed"}' + fixture "repos/$REPO/commits/$SHA/pulls" '[]' + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] +} + +@test "workflow_run: a closed first embedded PR does not hide a later open one" { + workflow_run_event '[{"number": 98, "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}, + {"number": 99, "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}]' + fixture "repos/$REPO/pulls/98" '{"state": "closed"}' + fixture "repos/$REPO/pulls/99" '{"state": "open"}' + failed_jobs_fixture + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "false" ] + [ "$(out pr_number)" = "99" ] +} + +@test "workflow_run: a failed PR-state lookup fails loudly instead of skipping" { + workflow_run_event '[{"number": 99, "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}]' + run bash "$SCRIPT" resolve + [ "$status" -ne 0 ] + [ "$(out skip)" = "" ] +} + +@test "workflow_run: a failed commits-to-pulls lookup fails loudly instead of skipping" { + workflow_run_event '[]' + run bash "$SCRIPT" resolve + [ "$status" -ne 0 ] + [ "$(out skip)" = "" ] +} + +# ── unusable payloads ──────────────────────────────────────────────────────── + +@test "empty payload: skipping notice, exit 0, no jq error, no API call" { + echo '{}' > "$GITHUB_EVENT_PATH" + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] + [[ "$output" == *"::notice::"*"skipping"* ]] + [[ "$output" != *"Cannot iterate"* ]] + [ ! -s "$GH_CALLS" ] +} + +@test "zero-byte, non-JSON and missing payload files all skip with exit 0" { + : > "$GITHUB_EVENT_PATH" + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] + + : > "$GITHUB_OUTPUT" + echo 'not json' > "$GITHUB_EVENT_PATH" + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] + + : > "$GITHUB_OUTPUT" + rm -f "$GITHUB_EVENT_PATH" + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + [ "$(out skip)" = "true" ] +} + +# ── idempotency: one analysis per commit ───────────────────────────────────── + +@test "idempotency: no marker comment yet means the analysis runs" { + export PR_NUMBER=99 HEAD_SHA="$SHA" + fixture "repos/$REPO/issues/99/comments" '[{"id": 1, "body": "unrelated"}]' + run bash "$SCRIPT" idempotency + [ "$status" -eq 0 ] + [ "$(out exists)" = "false" ] +} + +@test "two failures on one SHA: the second trigger finds the marker and skips" { + fixture "repos/$REPO/issues/99/comments" '[]' + + # First failure: a workflow_run for Lint. + workflow_run_event '[{"number": 99, "state": "open", "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}]' + fixture "repos/$REPO/pulls/99" '{"state": "open"}' + failed_jobs_fixture + run bash "$SCRIPT" resolve + [ "$status" -eq 0 ] + first_sha="$(out head_sha)" + PR_NUMBER="$(out pr_number)" HEAD_SHA="$first_sha" run bash "$SCRIPT" idempotency + [ "$status" -eq 0 ] + [ "$(out exists)" = "false" ] + + # The first analysis posts its comment. + fixture "repos/$REPO/issues/99/comments" "[{\"id\": 321, \"body\": \"\\n## CI Failure: Lint\"}]" + + # Second failure on the same commit: a different workflow (Tests), and a + # third-party check_run, both resolve to the same SHA and so the same marker. + : > "$GITHUB_OUTPUT" + workflow_run_event '[{"number": 99, "state": "open", "head": {"repo": {"id": 7}}, "base": {"repo": {"id": 7}}}]' + fixture "repos/$REPO/pulls/99" '{"state": "open"}' + jq '.workflow_run.name = "Tests" | .workflow_run.id = 778' "$GITHUB_EVENT_PATH" > "$TMP/e" && mv "$TMP/e" "$GITHUB_EVENT_PATH" + fixture "repos/$REPO/actions/runs/778/jobs" '{"jobs": []}' + run bash "$SCRIPT" resolve + [ "$(out head_sha)" = "$first_sha" ] + PR_NUMBER="$(out pr_number)" HEAD_SHA="$(out head_sha)" run bash "$SCRIPT" idempotency + [ "$status" -eq 0 ] + [ "$(out exists)" = "true" ] + [[ "$output" == *"already posted (id=321)"* ]] +} + +@test "idempotency: a marker for a different SHA does not suppress this one" { + export PR_NUMBER=99 HEAD_SHA="$SHA" + fixture "repos/$REPO/issues/99/comments" '[{"id": 2, "body": ""}]' + run bash "$SCRIPT" idempotency + [ "$status" -eq 0 ] + [ "$(out exists)" = "false" ] +} + +@test "idempotency: a failed comment lookup fails closed instead of posting a duplicate" { + export PR_NUMBER=99 HEAD_SHA="$SHA" + # no fixture for the comments path: the gh stub answers with a 404 failure + run bash "$SCRIPT" idempotency + [ "$status" -eq 0 ] + [ "$(out exists)" = "true" ] + [[ "$output" == *"lookup failed"* ]] +} + +@test "unknown subcommand is a usage error" { + run bash "$SCRIPT" bogus + [ "$status" -eq 2 ] +}