feat: implement issue #2235 — CI Failure Analyst never runs for failed GitHub Actions jobs: the check_run trigger only fires for third-party checks

.github/workflows/ci-failure-analyst-reusable.yml

@@ -36,52 +36,36 @@
       REPO: ${{ github.repository }}
 
     steps:
+      # The payload resolution lives in scripts/ci-failure-analyst-resolve.sh
+      # (tests: tests/ci_failure_analyst_resolve.bats). Fetch it from THIS
+      # reusable's own commit, so a caller pinned to a channel always runs the
+      # script that matches the workflow it pinned. .github-private is public,
+      # so the caller's token can read it.
+      - name: Checkout analyst tooling
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: combsmith/.github-private
+          ref: ${{ job.workflow_sha }}
+          path: .ci-analyst-tooling
+          sparse-checkout: scripts/ci-failure-analyst-resolve.sh
+          sparse-checkout-cone-mode: false
+          fetch-depth: 1
+          persist-credentials: false
+
+      # Resolves the head SHA, the non-fork PR and the failing run from either a
+      # check_run payload (third-party checks) or a workflow_run payload (failed
+      # GitHub Actions runs). Anything else ends in a "skipping" notice, exit 0.
       - name: Resolve non-fork PR
         id: pr
-        env:
-          HEAD_SHA: ${{ github.event.check_run.head_sha }}
-          CHECK_RUN_PRS: ${{ toJson(github.event.check_run.pull_requests) }}
-        run: |
-          set -euo pipefail
-          # Prefer the pull_requests array embedded in the check_run payload;
-          # filter to non-fork PRs (head repo must equal base repo).
-          pr_number=$(echo "$CHECK_RUN_PRS" \
-            | jq -r '.[] | select(.head.repo.id == .base.repo.id) | .number' \
-            | head -n 1)
-          if [ -z "$pr_number" ]; then
-            # Fallback: commits-to-pulls API handles edge cases (e.g. a second
-            # push arrived before this check_run completed).
-            # shellcheck disable=SC2016  # $ENV.REPO is jq built-in syntax, not a shell variable
-            pr_number=$(gh api "repos/$REPO/commits/$HEAD_SHA/pulls" \
-              --jq '.[] | select(.head.repo.full_name == $ENV.REPO) | .number' \
-              2>/dev/null | head -n 1 || true)
-          fi
-          if [ -z "$pr_number" ]; then
-            echo "::notice::No non-fork PR for SHA $HEAD_SHA — skipping"
-            echo "skip=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "skip=false" >> "$GITHUB_OUTPUT"
-            echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT"
-          fi
+        run: bash .ci-analyst-tooling/scripts/ci-failure-analyst-resolve.sh resolve
 
       - name: Check idempotency
         if: steps.pr.outputs.skip == 'false'
         id: idem
         env:
           PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
-          HEAD_SHA: ${{ github.event.check_run.head_sha }}
-        run: |
-          set -euo pipefail
-          existing=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \
-            --paginate \
-            --jq ".[] | select(.body | startswith(\"<!-- ci-analyst sha=$HEAD_SHA -->\")) | .id" \
-            2>/dev/null | head -1 || true)
-          if [ -n "$existing" ]; then
-            echo "::notice::Diagnostic comment already posted (id=$existing) — idempotency skip"
-            echo "exists=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "exists=false" >> "$GITHUB_OUTPUT"
-          fi
+          HEAD_SHA: ${{ steps.pr.outputs.head_sha }}
+        run: bash .ci-analyst-tooling/scripts/ci-failure-analyst-resolve.sh idempotency
 
       - name: Install Claude Code
         if: steps.pr.outputs.skip == 'false' && steps.idem.outputs.exists == 'false'
@@ -97,10 +81,11 @@
         if: steps.pr.outputs.skip == 'false' && steps.idem.outputs.exists == 'false'
         env:
           PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
-          HEAD_SHA: ${{ github.event.check_run.head_sha }}
-          WORKFLOW_NAME: ${{ github.event.check_run.name }}
-          DETAILS_URL: ${{ github.event.check_run.details_url }}
-          CHECK_RUN_ID: ${{ github.event.check_run.id }}
+          HEAD_SHA: ${{ steps.pr.outputs.head_sha }}
+          WORKFLOW_NAME: ${{ steps.pr.outputs.check_name }}
+          DETAILS_URL: ${{ steps.pr.outputs.details_url }}
+          RUN_ID: ${{ steps.pr.outputs.run_id }}
+          FAILED_JOBS: ${{ steps.pr.outputs.failed_jobs }}
         run: |
           set -euo pipefail
           export PATH="$HOME/.npm-global/bin:$PATH"
@@ -113,16 +98,15 @@
           - Head SHA: $HEAD_SHA
           - Workflow/check name: $WORKFLOW_NAME
           - Details URL: $DETAILS_URL
-          - Check run ID: $CHECK_RUN_ID
+          - GitHub Actions run ID: $RUN_ID
+          - Failed jobs (job / failing step): $FAILED_JOBS
 
           Follow these steps in order:
 
           ### 1. Fetch failure logs
-          Extract the GitHub Actions run ID from the details URL:
-            run_id=$(echo "$DETAILS_URL" | grep -oP '(?<=runs/)\d+' || true)
-          If a run_id is found, fetch the failed-step logs:
-            gh run view "$run_id" --log-failed --repo $REPO 2>/dev/null | tail -200
-          If no run_id (e.g. external quality gate like SonarCloud), use the PR diff:
+          If the run ID above is not empty, fetch the failed-step logs:
+            gh run view $RUN_ID --log-failed --repo $REPO 2>/dev/null | tail -200
+          If the run ID is empty (e.g. an external quality gate like SonarCloud), use the PR diff:
             gh pr diff $PR_NUMBER --repo $REPO 2>/dev/null | head -200
 
           ### 2. Identify the failing step and error

.github/workflows/ci.yml

@@ -63,7 +63,9 @@ jobs:
           chmod +x actionlint
           rm actionlint.tar.gz
           shopt -s extglob
-          ./actionlint -ignore 'SC2129' .github/workflows/!(*.lock).yml
+          # `workflow_sha` is a valid `job` context property (the
+          # reusable's own commit); the actionlint schema omits it.
+          ./actionlint -ignore 'SC2129' -ignore '"workflow_sha" is not defined' .github/workflows/!(*.lock).yml
 
   shellcheck:
     name: ShellCheck

.github/workflows/lint.yml

@@ -124,6 +124,7 @@
             tests/fleet_monitor_run.bats \
             tests/test_fleet_paused_run.bats \
             tests/template_stub_drift.bats \
+            tests/ci_failure_analyst_resolve.bats \
             tests/canary_rollout.bats \
             tests/test_initiative_canary.bats \
             tests/test_initiative_driver_canary.bats \
@@ -336,7 +337,9 @@
           chmod +x actionlint
           rm actionlint.tar.gz
           shopt -s extglob
-          ./actionlint -ignore 'SC2129' .github/workflows/!(*.lock).yml
+          # `workflow_sha` is a valid `job` context property (the
+          # reusable's own commit); the actionlint schema omits it.
+          ./actionlint -ignore 'SC2129' -ignore '"workflow_sha" is not defined' .github/workflows/!(*.lock).yml
 
   validate-workflow-schedules:
     # Off-peak scheduling compliance (#724, epic #722): fail any PR that
@@ -366,7 +369,7 @@
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         # NOTE: persist-credentials is intentionally left at the default (true) —
         # vci_resolve_reusable needs `git fetch origin` to authenticate against this
         # private repo when resolving same-repo channel tags. Setting
         # persist-credentials: false would silently push every such caller into the

docs/aw/ci-failure-analyst.md

@@ -4,9 +4,9 @@ Agentic workflow that diagnoses CI failures and posts a root-cause comment on th
 
 ## What it does
 
-When a check run completes with `conclusion: failure` on a non-fork PR, the workflow:
+When a check run or a watched GitHub Actions workflow run completes with `conclusion: failure` on a non-fork PR, the workflow:
 
-1. Resolves the PR number from the check run payload (with a commits-to-pulls API fallback)
+1. Resolves the head SHA, the PR number and the failing run from the `check_run` or `workflow_run` payload, with a commits-to-pulls API fallback for the PR (open PRs only)
 2. Checks idempotency — skips if a `<!-- ci-analyst sha=<HEAD_SHA> -->` comment already exists for this SHA
 3. Fetches the failed run logs from the GitHub Actions API
 4. Identifies the specific step that failed and its error message
@@ -88,6 +88,81 @@ if: >-
   !startsWith(github.event.check_run.name, 'CI Failure Analyst')
 ```
 
+## Which failures are covered
+
+| Failure | Event | Covered |
+|---|---|---|
+| A check from another app (SonarCloud and the like) | `check_run` | Yes, through the current stub |
+| A failed GitHub Actions job (CI, Lint, Tests, …) | `workflow_run` | Once the stub carries the `workflow_run` trigger below |
+
+GitHub does not start a workflow for an event raised by GitHub Actions' own token, so a check run
+created by an Actions job never delivers `check_run` to the stub. The `check_run` trigger therefore
+only ever sees third-party checks. Failed Actions runs reach the analyst through `workflow_run` (#2235).
+
+The reusable accepts either payload. `scripts/ci-failure-analyst-resolve.sh` (tests:
+`tests/ci_failure_analyst_resolve.bats`) reads it and resolves:
+
+- **`check_run`**: head SHA, PR (the payload's non-fork `pull_requests`), check name, details URL, and
+  the Actions run id when the details URL points at one. A third-party check has no run, so the
+  analysis falls back to the PR diff.
+- **`workflow_run`**: head SHA from `workflow_run.head_sha`, PR from `workflow_run.pull_requests`
+  (non-fork only), run id from `workflow_run.id`, and the run's failed jobs and their failing steps.
+
+When the payload has no non-fork PR, the commits-to-pulls lookup is tried, counting open PRs only.
+A failure on the default branch is therefore not reported on the PR already merged into it. A
+payload that carries neither event, or a run with no open non-fork PR, ends in a "skipping" notice
+with exit 0.
+
+The reusable fetches the script from its own commit (`job.workflow_sha`), so a caller pinned
+to a channel runs the script that matches the workflow it pinned.
+
+### The `workflow_run` trigger and the watched-workflow list
+
+The stub adds `workflow_run` next to `check_run`:
+
+```yaml
+on:
+  check_run:
+    types: [completed]
+  # Each repo may edit this list: name every workflow whose failures should be
+  # analysed. GitHub requires names (the `name:` of each workflow file); there
+  # is no wildcard. Never list "CI Failure Analyst" itself.
+  workflow_run:
+    workflows: ["CI", "Lint", "Tests"]
+    types: [completed]
+
+concurrency:
+  group: "ci-failure-analyst-${{ github.event.check_run.head_sha || github.event.workflow_run.head_sha }}"
+  cancel-in-progress: false
+
+jobs:
+  analyze:
+    if: >-
+      (github.event_name == 'check_run' &&
+       github.event.check_run.conclusion == 'failure' &&
+       !startsWith(github.event.check_run.name, 'CI Failure Analyst')) ||
+      (github.event_name == 'workflow_run' &&
+       github.event.workflow_run.conclusion == 'failure' &&
+       !startsWith(github.event.workflow_run.name, 'CI Failure Analyst'))
+```
+
+To change which Actions failures are analysed, edit the `workflows:` list in your repo's stub. Each
+entry must match the target workflow's `name:` exactly. A workflow that is not listed is not analysed.
+The `workflows:` list is the one part of the stub a repo is expected to change.
+
+When several watched workflows fail on the same commit, each failure starts an analyst run. The
+per-SHA concurrency group runs them one at a time, and the `<!-- ci-analyst sha=... -->` marker
+makes every run after the first an idempotency skip, giving one comment per commit.
+
+**Rollout order.** The stubs pin `ci-failure-analyst/v1-stable`. A trigger takes effect as soon as
+the stub merges, but the reusable it calls is whatever that channel points at. The reusable at
+v1.0.0 reads only `check_run` and fails a `workflow_run` call with `jq: Cannot iterate over null`.
+So the `workflow_run` trigger goes into `templates/ci-failure-analyst.yml` and this repo's
+`ci-failure-analyst.lock.yml` only after the pinned channel has been promoted to a commit carrying
+the dual-payload reusable. That stub change also needs `caller-stub-freeze`'s baseline regenerated,
+plus matching updates to `interaction-contracts/ci-failure-analyst.yml` and the §4 row of the
+[agentic interaction model](../agentic-interaction-model.md).
+
 ## Root cause categories
 
 | Category | Indicators |