diff --git a/.github/justfile b/.github/justfile index 979945d743f..e543943aa70 100644 --- a/.github/justfile +++ b/.github/justfile @@ -4,6 +4,7 @@ set working-directory := '..' check: actionlint sh/gh/alert.sh check-coverage + bun test sh/gh/alert.test.ts bun sh/gh/check-runs.ts # Tests of repository tooling; nightly, not on every pull request. @@ -40,7 +41,7 @@ test: release *args: sh/gh/release.sh "$@" -# Post a failed run to Discord, or check that alert.yml covers every workflow. +# Post a failed run to Discord, or check cron and publishing alert coverage. [positional-arguments] alert *args: sh/gh/alert.sh "$@" diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index d0cdc38dec0..acf165f40da 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -1,33 +1,19 @@ name: Alert -# Posts to the Discord #alerts channel when a workflow fails outside of a pull -# request. -# -# Why this exists: GitHub only notifies the user who *triggered* a run, and the -# runs that matter most here are triggered by moq-bot (release tags) or by cron, -# so without this nobody is notified when a release breaks. -# -# Pull request failures are deliberately excluded, both pull_request and -# pull_request_target. The author already sees them as a check on the PR, and -# that noise is what drives people to mute Actions notifications wholesale. +# Posts cron, release, and deployment failures to the Discord #alerts channel. +# PRs, merge queues, branch validation, and cache warming stay on GitHub. # # Needs a DISCORD_WEBHOOK repo secret (Discord: Channel > Integrations > # Webhooks). Nothing else reads it. on: workflow_run: - # Matched by workflow `name:`, not filename. Every workflow that can run - # outside a pull request belongs here; the pull-request-only ones (Android) - # are omitted so they don't spawn a skipped Alert run on every - # push to every PR. `just gh check` enforces both halves of that. - # - # Check and Platform also run for merge groups, and OBS and Platform can - # run on release pushes. Failures outside PRs need an explicit alert. + # Matched by workflow `name:`, not filename. Watch cron and publishing + # workflows only, avoiding a skipped Alert run on every CI completion. + # `just gh check` enforces coverage and the exclusions in alert.sh. workflows: - apt-repo - - Cache - Cachix - - Check - Docker - Interop - moq-c @@ -36,9 +22,6 @@ on: - moq-relay - moq.sh - Nightly - - OBS - - Platform - - Quest - release-brew - Release Cpp - Release Dart @@ -72,11 +55,16 @@ jobs: # # Note the chained cases (a release tag -> release-brew -> here) sit at the # third and final level workflow_run supports; a fourth would never fire. + # Cron suites alert only for schedules; publishing workflows also alert + # when dispatched manually or chained after another release. if: >- (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out') && github.event.workflow_run.event != 'pull_request' && - github.event.workflow_run.event != 'pull_request_target' + github.event.workflow_run.event != 'pull_request_target' && + github.event.workflow_run.event != 'merge_group' && + (github.event.workflow_run.event == 'schedule' || + !contains(fromJSON('["Interop", "Nightly", "TS", "Video"]'), github.event.workflow_run.name)) runs-on: ubuntu-latest steps: diff --git a/sh/gh/alert.sh b/sh/gh/alert.sh index 5d4e812bf15..2bf7998b823 100755 --- a/sh/gh/alert.sh +++ b/sh/gh/alert.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # -# Discord alerting for workflow failures outside of pull requests. +# Discord alerting for cron, release, and deployment failures. # Usage: # alert.sh discord Post the current workflow_run failure to Discord. -# alert.sh check-coverage Verify alert.yml watches every non-PR workflow. +# alert.sh check-coverage Verify alert.yml watches every alertable workflow. # # Environment (discord): # DISCORD_WEBHOOK Required. The #alerts channel webhook URL. @@ -59,36 +59,34 @@ discord() { "$DISCORD_WEBHOOK" } -# Every workflow that can run outside a pull request must be watched by -# alert.yml, and pull-request-only workflows must not be (each entry costs a -# skipped Alert run per trigger, and `Check` alone fires on every push to every -# PR). +# Watch cron and publishing workflows. Ordinary CI stays on GitHub, avoiding +# both Discord noise and a skipped Alert run on every CI completion. # # This is the guard against the list going stale: adding a workflow, or adding # a push/schedule trigger to a PR-only one, fails here until alert.yml matches. check_coverage() { local expected actual - # Not `non_pr_workflow_names | sort -u`: a failure inside a pipeline would + # Not `alertable_workflow_names | sort -u`: a failure inside a pipeline would # be swallowed and read as "no workflows need watching", which is the # silent pass this whole guard exists to prevent. Capture, propagate, then # sort. - expected=$(non_pr_workflow_names) || return 2 + expected=$(alertable_workflow_names) || return 2 expected=$(sort -u <<<"$expected") actual=$(watched_workflow_names) || return 2 actual=$(sort -u <<<"$actual") if [[ "$expected" == "$actual" ]]; then - echo "alert.yml watches all $(wc -l <<<"$expected" | tr -d ' ') non-PR workflows" + echo "alert.yml watches all $(wc -l <<<"$expected" | tr -d ' ') alertable workflows" return 0 fi echo "alert.yml's workflow list is out of sync with .github/workflows/" >&2 comm -23 <(echo "$expected") <(echo "$actual") | sed 's/^/ missing (add to alert.yml): /' >&2 - comm -13 <(echo "$expected") <(echo "$actual") | sed 's/^/ stale (remove, PR-only): /' >&2 + comm -13 <(echo "$expected") <(echo "$actual") | sed 's/^/ stale (remove, CI-only): /' >&2 return 1 } -# Names of workflows with a trigger that starts a run outside a pull request. +# Names of cron and publishing workflows. # # The YAML is parsed rather than pattern-matched. GitHub accepts `on:` as a # block map, block sequence, flow sequence or bare scalar, any of which may wrap @@ -102,7 +100,7 @@ check_coverage() { # A workflow with no `name:` is an error, not a skip: GitHub falls back to the # file path, which alert.yml cannot reference, and skipping it would put the # workflow past this check unwatched. -non_pr_workflow_names() { +alertable_workflow_names() { local f files=() shopt -s nullglob for f in "$WORKFLOWS_DIR"/*.yml "$WORKFLOWS_DIR"/*.yaml; do @@ -113,10 +111,12 @@ non_pr_workflow_names() { printf "%s\n" "${files[@]}" | bun -e ' const files = (await Bun.stdin.text()).split("\n").filter(Boolean); -// The PR events report their failure as a check on the PR itself, and +// PR and merge group events report their failure as a check on the PR itself, and // workflow_call runs inside the run that called it, so a workflow triggered // only by these needs no entry. -const UNWATCHED = new Set(["pull_request", "pull_request_target", "workflow_call"]); +const UNWATCHED = new Set(["pull_request", "pull_request_target", "merge_group", "workflow_call"]); +// These push workflows validate branches or warm caches; none publishes. +const CI_ONLY = new Set(["Cache", "OBS", "Platform", "Quest"]); const names = []; for (const file of files) { const doc = Bun.YAML.parse(await Bun.file(file).text()); @@ -140,6 +140,7 @@ for (const file of files) { console.error("alert.sh: " + file + " has no top-level name:, so alert.yml cannot reference it"); process.exit(2); } + if (CI_ONLY.has(name.trim()) && !triggers.includes("schedule")) continue; names.push(name.trim()); } if (names.length) console.log(names.join("\n")); diff --git a/sh/gh/alert.test.ts b/sh/gh/alert.test.ts new file mode 100644 index 00000000000..6888fb03dca --- /dev/null +++ b/sh/gh/alert.test.ts @@ -0,0 +1,79 @@ +import { expect, test } from "bun:test"; +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const workflow = Bun.YAML.parse(await Bun.file(".github/workflows/alert.yml").text()); +const evaluate = new Function("github", "contains", "fromJSON", `return (${workflow.jobs.discord.if});`); +const watched = workflow.on.workflow_run.workflows as string[]; +const condition = (run: { name: string; event: string; conclusion: string }) => + watched.includes(run.name) && + evaluate({ event: { workflow_run: run } }, (values: string[], value: string) => values.includes(value), JSON.parse); + +for (const event of [ + "pull_request", + "pull_request_target", + "merge_group", + "push", + "schedule", + "workflow_dispatch", + "workflow_run", +]) { + test(`Discord alert policy for ${event}`, () => { + for (const conclusion of ["failure", "timed_out", "success", "cancelled", "skipped"]) { + const expected = + !["pull_request", "pull_request_target", "merge_group"].includes(event) && + ["failure", "timed_out"].includes(conclusion); + for (const name of ["Release JS", "moq.sh", "release-brew", "apt-repo"]) { + expect(condition({ name, event, conclusion })).toBe(expected); + } + } + }); +} + +test("cron workflows only alert on scheduled failures", async () => { + for await (const file of new Bun.Glob(".github/workflows/*.yml").scan()) { + const config = Bun.YAML.parse(await Bun.file(file).text()); + if (!config.on?.schedule) continue; + for (const event of ["schedule", "workflow_dispatch", "pull_request"]) { + for (const conclusion of ["failure", "timed_out", "success", "cancelled", "skipped"]) { + expect(condition({ name: config.name, event, conclusion })).toBe( + event === "schedule" && ["failure", "timed_out"].includes(conclusion), + ); + } + } + } +}); + +test("ordinary CI never subscribes to Discord alerts", () => { + for (const name of ["Check", "Cache", "OBS", "Platform", "Quest"]) { + expect(watched).not.toContain(name); + } +}); + +test("alert coverage excludes queue-only workflows and retains release and nightly workflows", () => { + const root = mkdtempSync(join(tmpdir(), "moq-alert-")); + try { + mkdirSync(join(root, "sh/gh"), { recursive: true }); + mkdirSync(join(root, ".github/workflows"), { recursive: true }); + copyFileSync("sh/gh/alert.sh", join(root, "sh/gh/alert.sh")); + const configs = { + alert: { name: "Alert", on: { workflow_run: { workflows: ["Release", "Nightly"] } } }, + check: { name: "Check", on: { pull_request: null, merge_group: null } }, + queue: { name: "Queue", on: ["merge_group"] }, + quest: { name: "Quest", on: { push: { branches: ["quest/**"] } } }, + cache: { name: "Cache", on: { push: { branches: ["main"] } } }, + release: { name: "Release", on: { pull_request: null, push: null } }, + nightly: { name: "Nightly", on: "schedule" }, + }; + for (const [name, config] of Object.entries(configs)) { + writeFileSync(join(root, `.github/workflows/${name}.yml`), Bun.YAML.stringify(config)); + } + const result = Bun.spawnSync(["bash", "sh/gh/alert.sh", "check-coverage"], { cwd: root }); + expect(result.stderr.toString()).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString()).toContain("watches all 2"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +});