ci: enrich scheduled-failure issues with an LLM triage pass

.github/workflows/ai-failure-enrich.yaml

@@ -0,0 +1,70 @@
+name: AI-enrich scheduled-failure notification
+
+# Called by notify-scheduled-failure.yaml, which is called by scheduled workflows on failure.
+# - notify-scheduled-failure.yaml creates or comments on an issue before calling this workflow.
+# - github.workflow and github.run_id refer to the scheduled workflow that failed.
+
+on:
+  workflow_call:
+    inputs:
+      # What the notifier did, when it got that far. Optional: an empty value
+      # means the script looks the issue up itself, which is what happens when
+      # `notify` fails before it can create one.
+      issue:
+        description: The issue the notifier created or commented on.
+        required: false
+        type: string
+      origin:
+        description: Either "new" or "comment", matching what the notifier did.
+        required: false
+        type: string
+    secrets:
+      OPENROUTER_API_KEY:
+        required: false
+
+permissions: {}
+
+jobs:
+  enrich:
+    runs-on: ubuntu-latest
+    environment: ai-failure-triage  # Required to access secrets.OPENROUTER_API_KEY
+    permissions:
+      issues: write
+      actions: read
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39  # v8.2.0
+      - name: Extract signature, dedupe, ask the LLM, apply
+        id: enrich
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+          WORKFLOW_NAME: ${{ github.workflow }}  # The workflow that failed.
+          NOTIFY_ISSUE: ${{ inputs.issue }}
+          NOTIFY_ORIGIN: ${{ inputs.origin }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          # It takes two things to get this key here, and it's tempting to
+          # remove the second one, so please don't:
+          #
+          #   - `environment:` on this job holds the real value. A caller
+          #     can't see it, because a job without an `environment:` gets
+          #     an empty string for `secrets.OPENROUTER_API_KEY`.
+          #   - every call site in the chain names the secret, and it's
+          #     declared above. That only ever passes the empty string, which
+          #     is why it looks like it does nothing.
+          #
+          # Take the second one out and the key arrives empty, and nothing
+          # complains: we just get a plain notice instead of an enriched one.
+          # If enrichment stops happening for no obvious reason, look here.
+          #
+          # Each repo that adopts this provisions its own key. Without one,
+          # the script writes the plain fallback body and never calls
+          # OpenRouter (see the package's cli.main()).
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          OPENROUTER_MODEL: ${{ vars.OPENROUTER_MODEL }}
+        run: >-
+          uvx --from "git+https://github.com/canonical/charm-tech-code@3702a80516503509a0472f62dcffbcf022654925#subdirectory=ai-failure-notifier"
+          ai-failure-notifier

.github/workflows/example-charm-charmcraft-test.yaml

@@ -71,4 +71,8 @@ jobs:
     needs: [integration]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml

.github/workflows/example-charm-integration-tests.yaml

@@ -84,4 +84,8 @@ jobs:
     needs: [machine-examples-integration, k8s-examples-integration]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml

.github/workflows/integration.yaml

@@ -73,4 +73,8 @@ jobs:
     needs: [integration]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml

.github/workflows/notify-scheduled-failure.yaml

@@ -1,26 +1,84 @@
 name: Notify on scheduled failure
 
-# Reusable workflow: opens an issue when a scheduled workflow fails.
-# Callers gate the invocation with `if: failure() && github.event_name == 'schedule'`
-# and pass `permissions: issues: write`.
+# Reusable workflow: opens (or comments on) an issue when a scheduled
+# workflow fails. Callers gate the invocation with
+# `if: failure() && github.event_name == 'schedule'` and pass
+# `permissions: issues: write` plus `actions: read`, and grant access to
+# `secrets.OPENROUTER_API_KEY`.
+#
+# 1. `notify` opens or comments on an issue. No LLM and no secrets: once it
+#    has run, a notification exists.
+# 2. `enrich` (ai-failure-enrich.yaml) rewrites that issue into something
+#    worth reading. If its key or environment is missing, we still have 1.
 
 on:
   workflow_call:
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY:
+        required: false
 
 jobs:
-  open-issue:
+  notify:
     runs-on: ubuntu-latest
     permissions:
       issues: write
+    # Handed to `enrich` below, so that it's told which issue to rewrite
+    # rather than searching for the marker this job has just stamped.
+    # GitHub's issue search index doesn't necessarily show you what you have
+    # only just written, so that search can miss a marker from seconds ago
+    # and open a second issue for a run that already has one.
+    outputs:
+      issue: ${{ steps.notify.outputs.issue }}
+      origin: ${{ steps.notify.outputs.origin }}
     steps:
-      - name: Create issue on failure
+      - name: Create or comment on issue, deduping coarsely by workflow name
+        id: notify
         env:
           GH_TOKEN: ${{ github.token }}
           WORKFLOW_NAME: ${{ github.workflow }}  # The workflow that called this one.
           REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
-          gh issue create \
-            --repo "$REPO" \
-            --title "Scheduled workflow '$WORKFLOW_NAME' failed" \
-            --body "Scheduled workflow '$WORKFLOW_NAME' failed: $RUN_URL"
+          set -euo pipefail
+          marker="<!-- ai-failure-notifications:run=$RUN_ID:origin="
+
+          # Search for the workflow name in issue bodies or titles.
+          match=$(gh issue list --repo "$REPO" --state open \
+            --search "\"$WORKFLOW_NAME\"" \
+            --json number --limit 1 -q '.[0].number // empty')
+
+          if [[ -n "$match" ]]; then
+            comment_body="Scheduled workflow '$WORKFLOW_NAME' failed again: $RUN_URL"$'\n\n'"${marker}comment -->"
+            gh issue comment "$match" --repo "$REPO" --body "$comment_body"
+            echo "issue=$match" >> "$GITHUB_OUTPUT"
+            echo "origin=comment" >> "$GITHUB_OUTPUT"
+          else
+            issue_body="Scheduled workflow '$WORKFLOW_NAME' failed: $RUN_URL"$'\n\n'"${marker}new -->"
+            # `gh issue create` prints the new issue's URL; the number is its
+            # last path segment.
+            issue_url=$(gh issue create --repo "$REPO" \
+              --title "Scheduled workflow '$WORKFLOW_NAME' failed" \
+              --body "$issue_body")
+            echo "issue=${issue_url##*/}" >> "$GITHUB_OUTPUT"
+            echo "origin=new" >> "$GITHUB_OUTPUT"
+          fi
+
+  enrich:
+    needs: [notify]
+    permissions:
+      issues: write
+      actions: read
+    # Passes an empty string, and is still required -- see the note on
+    # OPENROUTER_API_KEY in ai-failure-enrich.yaml.
+    secrets:
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+    # `notify` has to have succeeded to get here, so these are always set for
+    # this repo. The script can still look the issue up if they're empty,
+    # which is what a repo that adopts the enricher before it passes these
+    # will get.
+    with:
+      issue: ${{ needs.notify.outputs.issue }}
+      origin: ${{ needs.notify.outputs.origin }}
+    uses: ./.github/workflows/ai-failure-enrich.yaml

.github/workflows/smoke.yaml

@@ -50,4 +50,8 @@ jobs:
     needs: [test]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml

.github/workflows/tiobe.yaml

@@ -41,4 +41,8 @@ jobs:
     needs: [TICS]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml

.github/workflows/update-best-practice-doc.yaml

@@ -62,4 +62,8 @@ jobs:
     needs: [update-docs]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml

.github/workflows/update-charm-tests.yaml

@@ -59,4 +59,8 @@ jobs:
     needs: [update-pins]
     permissions:
       issues: write
+      actions: read  # for ai-failure-enrich.yaml
+    secrets:
+      # For ai-failure-enrich.yaml -- can only be read in the ai-failure-triage environment.
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     uses: ./.github/workflows/notify-scheduled-failure.yaml