CI Failure Triage #37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Failure Triage | |
| # Watches the E2E workflow. On a red run an agent diagnoses each failure from | |
| # the run logs and the uploaded report artifact, and for failures it classifies | |
| # as fixable spec issues it applies the fix, re-runs the test, and opens a PR. | |
| # It never auto-merges. | |
| # | |
| # The `workflow_run` trigger and the secret names must live here: a | |
| # `workflow_run` trigger is illegal inside a reusable workflow, and secret names | |
| # are per-repo. Everything else lives in ShiplightAI/ci-triage. | |
| on: | |
| workflow_run: | |
| workflows: # exact `name:` of each workflow to watch | |
| - Demo Tests | |
| - Showcase Tests | |
| - Spec Drift Fixture # red on main, green on its auto-fix PR | |
| - App Regression Fixture # permanently red by design | |
| types: [completed] | |
| jobs: | |
| triage: | |
| # SECURITY GATE — do not remove. | |
| # | |
| # workflow_run runs from the default branch with FULL access to secrets and | |
| # a write token, even when the run that triggered it came from a fork's | |
| # pull request. The triage job then checks out | |
| # github.event.workflow_run.head_sha — the triggering commit — and runs an | |
| # agent over it with `--permission-mode bypassPermissions`, the Claude OAuth | |
| # token in its environment, and an MCP config read from that checkout. | |
| # | |
| # Without this guard, anyone who can open a PR on this public repo could | |
| # reach that agent with attacker-controlled files (a crafted .mcp.json | |
| # defines commands to execute) and exfiltrate the token. | |
| # | |
| # Restrict it to runs whose head commit lives in THIS repository. Branch | |
| # pushes and PRs from collaborators still trigger triage; forks never do. | |
| if: github.event.workflow_run.head_repository.full_name == github.repository | |
| uses: ShiplightAI/ci-triage/.github/workflows/triage.yml@v1.3 | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| with: | |
| # The Shiplight project is not at the repository root. | |
| working-directory: yaml-examples | |
| triage-runner: ubuntu-latest # read-only diagnosis, no browser needed | |
| autofix-runner: shiplight-small # re-runs tests, so needs browsers/network | |
| node-version: "22" | |
| # Relative to working-directory. Hard guard: a change anywhere else aborts | |
| # the PR. Scoped to what the fix agent is actually allowed to touch — the | |
| # test sources and the templates they reference, not helpers/ or fixtures. | |
| # triage-fixtures is included so autofix can repair the spec-drift | |
| # fixture — that PR is the demonstration. Do not merge it. | |
| allowed-paths: "demo showcase templates triage-fixtures" | |
| verify-workers: "2" | |
| secrets: | |
| claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| openai_api_key: ${{ secrets.OPENAI_API_KEY }} # Codex fallback | |
| autofix_github_token: ${{ secrets.AUTOFIX_GITHUB_TOKEN }} # falls back to GITHUB_TOKEN | |
| # No `extra_env`: the demo suite logs into saucedemo with public | |
| # credentials and the showcase suite hits public sites, so the autofix | |
| # re-run needs no app secrets. Add KEY=VALUE lines here if that changes. | |
| # Caller-owned notification. ci-triage deliberately posts nowhere itself; it | |
| # publishes artifacts and lets the consumer decide what to route where, using | |
| # its own credentials. Keeping SLACK_RELEASE_BOT_TOKEN in this job — and out | |
| # of `extra_env` — means the autofix agent never sees it. | |
| notify-slack: | |
| needs: triage | |
| # Repeats the fork guard: `always()` would otherwise run this job even when | |
| # the triage job above was skipped, handing the Slack bot token to a | |
| # fork-triggered run. | |
| if: >- | |
| ${{ always() && | |
| github.event.workflow_run.head_repository.full_name == github.repository && | |
| (github.event.workflow_run.conclusion == 'failure' || | |
| github.event.workflow_run.conclusion == 'timed_out') }} | |
| continue-on-error: true # Slack being down must never mask a triage result | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| steps: | |
| - name: Checkout integration script | |
| uses: actions/checkout@v5 | |
| - name: Download triage context | |
| id: fetch | |
| continue-on-error: true # triage may have died before uploading | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: triage-context | |
| path: /tmp/triage-context | |
| # `needs: triage` waits for the whole reusable workflow, autofix included, | |
| # so these artifacts already exist by the time this job runs. There are | |
| # none when nothing was fixable — hence continue-on-error. | |
| - name: Download autofix outcomes | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: autofix-result-* | |
| path: /tmp/autofix | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| - name: Post triage report to Slack | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_RELEASE_BOT_TOKEN }} | |
| SLACK_CHANNEL: ${{ vars.SLACK_ISSUE_CHANNEL_ID }} # org variable, not a secret | |
| WORKFLOW: ${{ github.event.workflow_run.name }} | |
| BRANCH: ${{ github.event.workflow_run.head_branch }} | |
| RUN_URL: ${{ github.event.workflow_run.html_url }} | |
| TRIAGE_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| CONCLUSION: ${{ github.event.workflow_run.conclusion }} | |
| run: | | |
| set -euo pipefail | |
| node .github/scripts/post-slack-triage.ts \ | |
| /tmp/triage-context/triage.md \ | |
| /tmp/autofix | |
| # Non-fixable incidents belong in Linear for human follow-up. OAuth client | |
| # credentials stay in this caller-owned job, so neither the triage nor | |
| # autofix agent can read them. Open incidents are updated by the verdict's | |
| # stable dedup_key; completed/cancelled incidents do not suppress a newly | |
| # recurring failure. | |
| notify-linear: | |
| needs: triage | |
| if: >- | |
| ${{ always() && | |
| (github.event.workflow_run.conclusion == 'failure' || | |
| github.event.workflow_run.conclusion == 'timed_out') }} | |
| continue-on-error: true # Linear must never mask the triage/autofix result | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| steps: | |
| - name: Checkout integration script | |
| uses: actions/checkout@v5 | |
| - name: Download triage verdict | |
| id: fetch | |
| continue-on-error: true # triage may have died before uploading | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: triage-context | |
| path: /tmp/triage-context | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| - name: Create or update Linear incidents | |
| env: | |
| LINEAR_CLIENT_ID: ${{ vars.LINEAR_CLIENT_ID }} | |
| LINEAR_CLIENT_SECRET: ${{ secrets.LINEAR_CLIENT_SECRET }} | |
| LINEAR_TEAM_KEY: ${{ vars.LINEAR_TEAM_KEY }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${LINEAR_CLIENT_ID:-}" ] || [ -z "${LINEAR_CLIENT_SECRET:-}" ] || [ -z "${LINEAR_TEAM_KEY:-}" ]; then | |
| echo "Linear OAuth credentials or LINEAR_TEAM_KEY are unset — skipping Linear publishing." | |
| exit 0 | |
| fi | |
| if [ ! -s /tmp/triage-context/verdict.json ]; then | |
| echo "Triage produced no verdict.json — skipping Linear publishing." | |
| exit 0 | |
| fi | |
| node .github/scripts/publish-linear-incidents.ts /tmp/triage-context/verdict.json |