Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ set working-directory := '..'
check:
actionlint
sh/gh/alert.sh check-coverage
bun test sh/gh/alert.test.ts
bun sh/gh/check-runs.ts

# Tests of repository tooling; nightly, not on every pull request.
Expand Down Expand Up @@ -40,7 +41,7 @@ test:
release *args:
sh/gh/release.sh "$@"

# Post a failed run to Discord, or check that alert.yml covers every workflow.
# Post a failed run to Discord, or check cron and publishing alert coverage.
[positional-arguments]
alert *args:
sh/gh/alert.sh "$@"
Expand Down
34 changes: 11 additions & 23 deletions .github/workflows/alert.yml
Original file line number Diff line number Diff line change
@@ -1,33 +1,19 @@
name: Alert

# Posts to the Discord #alerts channel when a workflow fails outside of a pull
# request.
#
# Why this exists: GitHub only notifies the user who *triggered* a run, and the
# runs that matter most here are triggered by moq-bot (release tags) or by cron,
# so without this nobody is notified when a release breaks.
#
# Pull request failures are deliberately excluded, both pull_request and
# pull_request_target. The author already sees them as a check on the PR, and
# that noise is what drives people to mute Actions notifications wholesale.
# Posts cron, release, and deployment failures to the Discord #alerts channel.
# PRs, merge queues, branch validation, and cache warming stay on GitHub.
#
# Needs a DISCORD_WEBHOOK repo secret (Discord: Channel > Integrations >
# Webhooks). Nothing else reads it.

on:
workflow_run:
# Matched by workflow `name:`, not filename. Every workflow that can run
# outside a pull request belongs here; the pull-request-only ones (Android)
# are omitted so they don't spawn a skipped Alert run on every
# push to every PR. `just gh check` enforces both halves of that.
#
# Check and Platform also run for merge groups, and OBS and Platform can
# run on release pushes. Failures outside PRs need an explicit alert.
# Matched by workflow `name:`, not filename. Watch cron and publishing
# workflows only, avoiding a skipped Alert run on every CI completion.
# `just gh check` enforces coverage and the exclusions in alert.sh.
workflows:
- apt-repo
- Cache
- Cachix
- Check
- Docker
- Interop
- moq-c
Expand All @@ -36,9 +22,6 @@ on:
- moq-relay
- moq.sh
- Nightly
- OBS
- Platform
- Quest
- release-brew
- Release Cpp
- Release Dart
Expand Down Expand Up @@ -72,11 +55,16 @@ jobs:
#
# Note the chained cases (a release tag -> release-brew -> here) sit at the
# third and final level workflow_run supports; a fourth would never fire.
# Cron suites alert only for schedules; publishing workflows also alert
# when dispatched manually or chained after another release.
if: >-
(github.event.workflow_run.conclusion == 'failure' ||
github.event.workflow_run.conclusion == 'timed_out') &&
github.event.workflow_run.event != 'pull_request' &&
github.event.workflow_run.event != 'pull_request_target'
github.event.workflow_run.event != 'pull_request_target' &&
github.event.workflow_run.event != 'merge_group' &&
(github.event.workflow_run.event == 'schedule' ||
!contains(fromJSON('["Interop", "Nightly", "TS", "Video"]'), github.event.workflow_run.name))
runs-on: ubuntu-latest

steps:
Expand Down
29 changes: 15 additions & 14 deletions sh/gh/alert.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/usr/bin/env bash
#
# Discord alerting for workflow failures outside of pull requests.
# Discord alerting for cron, release, and deployment failures.
# Usage:
# alert.sh discord Post the current workflow_run failure to Discord.
# alert.sh check-coverage Verify alert.yml watches every non-PR workflow.
# alert.sh check-coverage Verify alert.yml watches every alertable workflow.
#
# Environment (discord):
# DISCORD_WEBHOOK Required. The #alerts channel webhook URL.
Expand Down Expand Up @@ -59,36 +59,34 @@ discord() {
"$DISCORD_WEBHOOK"
}

# Every workflow that can run outside a pull request must be watched by
# alert.yml, and pull-request-only workflows must not be (each entry costs a
# skipped Alert run per trigger, and `Check` alone fires on every push to every
# PR).
# Watch cron and publishing workflows. Ordinary CI stays on GitHub, avoiding
# both Discord noise and a skipped Alert run on every CI completion.
#
# This is the guard against the list going stale: adding a workflow, or adding
# a push/schedule trigger to a PR-only one, fails here until alert.yml matches.
check_coverage() {
local expected actual
# Not `non_pr_workflow_names | sort -u`: a failure inside a pipeline would
# Not `alertable_workflow_names | sort -u`: a failure inside a pipeline would
# be swallowed and read as "no workflows need watching", which is the
# silent pass this whole guard exists to prevent. Capture, propagate, then
# sort.
expected=$(non_pr_workflow_names) || return 2
expected=$(alertable_workflow_names) || return 2
expected=$(sort -u <<<"$expected")
actual=$(watched_workflow_names) || return 2
actual=$(sort -u <<<"$actual")

if [[ "$expected" == "$actual" ]]; then
echo "alert.yml watches all $(wc -l <<<"$expected" | tr -d ' ') non-PR workflows"
echo "alert.yml watches all $(wc -l <<<"$expected" | tr -d ' ') alertable workflows"
return 0
fi

echo "alert.yml's workflow list is out of sync with .github/workflows/" >&2
comm -23 <(echo "$expected") <(echo "$actual") | sed 's/^/ missing (add to alert.yml): /' >&2
comm -13 <(echo "$expected") <(echo "$actual") | sed 's/^/ stale (remove, PR-only): /' >&2
comm -13 <(echo "$expected") <(echo "$actual") | sed 's/^/ stale (remove, CI-only): /' >&2
return 1
}

# Names of workflows with a trigger that starts a run outside a pull request.
# Names of cron and publishing workflows.
#
# The YAML is parsed rather than pattern-matched. GitHub accepts `on:` as a
# block map, block sequence, flow sequence or bare scalar, any of which may wrap
Expand All @@ -102,7 +100,7 @@ check_coverage() {
# A workflow with no `name:` is an error, not a skip: GitHub falls back to the
# file path, which alert.yml cannot reference, and skipping it would put the
# workflow past this check unwatched.
non_pr_workflow_names() {
alertable_workflow_names() {
local f files=()
shopt -s nullglob
for f in "$WORKFLOWS_DIR"/*.yml "$WORKFLOWS_DIR"/*.yaml; do
Expand All @@ -113,10 +111,12 @@ non_pr_workflow_names() {

printf "%s\n" "${files[@]}" | bun -e '
const files = (await Bun.stdin.text()).split("\n").filter(Boolean);
// The PR events report their failure as a check on the PR itself, and
// PR and merge group events report their failure as a check on the PR itself, and
// workflow_call runs inside the run that called it, so a workflow triggered
// only by these needs no entry.
const UNWATCHED = new Set(["pull_request", "pull_request_target", "workflow_call"]);
const UNWATCHED = new Set(["pull_request", "pull_request_target", "merge_group", "workflow_call"]);
// These push workflows validate branches or warm caches; none publishes.
const CI_ONLY = new Set(["Cache", "OBS", "Platform", "Quest"]);
const names = [];
for (const file of files) {
const doc = Bun.YAML.parse(await Bun.file(file).text());
Expand All @@ -140,6 +140,7 @@ for (const file of files) {
console.error("alert.sh: " + file + " has no top-level name:, so alert.yml cannot reference it");
process.exit(2);
}
if (CI_ONLY.has(name.trim()) && !triggers.includes("schedule")) continue;
names.push(name.trim());
}
if (names.length) console.log(names.join("\n"));
Expand Down
79 changes: 79 additions & 0 deletions sh/gh/alert.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { expect, test } from "bun:test";
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

const workflow = Bun.YAML.parse(await Bun.file(".github/workflows/alert.yml").text());
const evaluate = new Function("github", "contains", "fromJSON", `return (${workflow.jobs.discord.if});`);
const watched = workflow.on.workflow_run.workflows as string[];
const condition = (run: { name: string; event: string; conclusion: string }) =>
watched.includes(run.name) &&
evaluate({ event: { workflow_run: run } }, (values: string[], value: string) => values.includes(value), JSON.parse);

for (const event of [
"pull_request",
"pull_request_target",
"merge_group",
"push",
"schedule",
"workflow_dispatch",
"workflow_run",
]) {
test(`Discord alert policy for ${event}`, () => {
for (const conclusion of ["failure", "timed_out", "success", "cancelled", "skipped"]) {
const expected =
!["pull_request", "pull_request_target", "merge_group"].includes(event) &&
["failure", "timed_out"].includes(conclusion);
for (const name of ["Release JS", "moq.sh", "release-brew", "apt-repo"]) {
expect(condition({ name, event, conclusion })).toBe(expected);
}
}
});
}

test("cron workflows only alert on scheduled failures", async () => {
for await (const file of new Bun.Glob(".github/workflows/*.yml").scan()) {
const config = Bun.YAML.parse(await Bun.file(file).text());
if (!config.on?.schedule) continue;
for (const event of ["schedule", "workflow_dispatch", "pull_request"]) {
for (const conclusion of ["failure", "timed_out", "success", "cancelled", "skipped"]) {
expect(condition({ name: config.name, event, conclusion })).toBe(
event === "schedule" && ["failure", "timed_out"].includes(conclusion),
);
}
}
}
});

test("ordinary CI never subscribes to Discord alerts", () => {
for (const name of ["Check", "Cache", "OBS", "Platform", "Quest"]) {
expect(watched).not.toContain(name);
}
});

test("alert coverage excludes queue-only workflows and retains release and nightly workflows", () => {
const root = mkdtempSync(join(tmpdir(), "moq-alert-"));
try {
mkdirSync(join(root, "sh/gh"), { recursive: true });
mkdirSync(join(root, ".github/workflows"), { recursive: true });
copyFileSync("sh/gh/alert.sh", join(root, "sh/gh/alert.sh"));
const configs = {
alert: { name: "Alert", on: { workflow_run: { workflows: ["Release", "Nightly"] } } },
check: { name: "Check", on: { pull_request: null, merge_group: null } },
queue: { name: "Queue", on: ["merge_group"] },
quest: { name: "Quest", on: { push: { branches: ["quest/**"] } } },
cache: { name: "Cache", on: { push: { branches: ["main"] } } },
release: { name: "Release", on: { pull_request: null, push: null } },
nightly: { name: "Nightly", on: "schedule" },
};
for (const [name, config] of Object.entries(configs)) {
writeFileSync(join(root, `.github/workflows/${name}.yml`), Bun.YAML.stringify(config));
}
const result = Bun.spawnSync(["bash", "sh/gh/alert.sh", "check-coverage"], { cwd: root });
expect(result.stderr.toString()).toBe("");
expect(result.exitCode).toBe(0);
expect(result.stdout.toString()).toContain("watches all 2");
} finally {
rmSync(root, { recursive: true, force: true });
}
});