diff --git a/.github/workflows/grafana-alertcheck-release.yml b/.github/workflows/grafana-alertcheck-release.yml deleted file mode 100644 index 4f946b18f..000000000 --- a/.github/workflows/grafana-alertcheck-release.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Grafana Alertcheck Release - -on: - push: - tags: - - grafana-alertcheck/v* - -jobs: - release: - name: Build and Release - runs-on: ubuntu-latest - environment: integration - permissions: - id-token: write - contents: write - steps: - - name: Checkout repo - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v7 - with: - go-version-file: ./grafana-alertcheck/go.mod - cache-dependency-path: ./grafana-alertcheck/go.mod - - name: Goreleaser Release - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 - with: - distribution: goreleaser-pro - version: "~> v2" - args: release --clean -f ./grafana-alertcheck/.goreleaser.yaml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/grafana-alertcheck/.goreleaser.yaml b/grafana-alertcheck/.goreleaser.yaml deleted file mode 100644 index 0890d9f82..000000000 --- a/grafana-alertcheck/.goreleaser.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json -version: 2 -project_name: grafana-alertcheck - -dist: grafana-alertcheck/dist - -monorepo: - tag_prefix: grafana-alertcheck/ - dir: grafana-alertcheck - -builds: - - id: grafana-alertcheck - main: ./cmd/grafana-alertcheck/main.go - ldflags: - - -s - - -w - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.version={{.Version}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.commit={{.ShortCommit}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.date={{.CommitDate}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.builtBy=goreleaser - goos: - - linux - - darwin - goarch: - - amd64 - - arm64 - binary: grafana-alertcheck - env: - - CGO_ENABLED=0 - -before: - hooks: - - sh -c "cd grafana-alertcheck && go mod tidy" diff --git a/grafana-alertcheck/README.md b/grafana-alertcheck/README.md index 43cc03fc4..96aa0e999 100644 --- a/grafana-alertcheck/README.md +++ b/grafana-alertcheck/README.md @@ -1,6 +1,46 @@ # grafana-alertcheck -A CD quality gate for Grafana alerts: bookend a release with `watch` (record) and `check` (classify) to -answer whether any watched alert was in a bad state during the release window. +A CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and +`check` (classify) — and answers whether any watched alert was in a bad state during the release window. -Under construction. +``` +watch → your work → check +``` + +`watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a +`from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state +timeline, and exits `0`, `1`, or `2`. + +It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window. + +## Quickstart + +```bash +export GRAFANA_URL=https://grafana.example.com +export GRAFANA_TOKEN=… + +grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt +./deploy.sh # emits deployed_at= +./verify.sh # emits finished_at= +grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" +``` + +Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is +never a flag. + +## Documentation + +| Doc | Covers | +| --- | ------ | +| [`docs/index.md`](./docs/index.md) | Overview, quickstarts, exit codes, common surprises | +| [`docs/how-alerts-are-evaluated.md`](./docs/how-alerts-are-evaluated.md) | Verdict model, coverage proof, health/liveness | +| [`docs/advanced.md`](./docs/advanced.md) | Check budget, scheduling, why history isn't queried | +| [`docs/architecture.md`](./docs/architecture.md) | Design invariants, the pure-function seam, recorder lifecycle | +| [`docs/reference/cli.md`](./docs/reference/cli.md) | Full CLI reference — subcommands, flags, naming | +| [`docs/reference/log-format.md`](./docs/reference/log-format.md) | The JSONL log schema, for debugging artifacts | + +## Build + +```bash +go build ./... && go test ./... +``` diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/check.go b/grafana-alertcheck/cmd/check.go similarity index 94% rename from grafana-alertcheck/cmd/grafana-alertcheck/check.go rename to grafana-alertcheck/cmd/check.go index 3f2d295d6..e95fce376 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/check.go +++ b/grafana-alertcheck/cmd/check.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -40,6 +41,13 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`) if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(stderr, "check: unexpected arguments %v\n", fs.Args()) return 2 } if *output != "" && *output != "json" { @@ -82,7 +90,7 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { PidFile: *pidfile, Concurrency: *common.concurrency, Clock: gate.SystemClock{}, - Notes: stderr, + Notes: newNoteStyler(stderr), } if *to == "" { fmt.Fprintln(stderr, "check: --to is required") @@ -113,11 +121,10 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { result, checkErr := gate.Check(ctx, cfg) - if err := renderTable(stderr, result); err != nil { - fmt.Fprintln(stderr, err) - } if checkErr != nil { fmt.Fprintln(stderr, checkErr) + } else if err := renderTable(stderr, result); err != nil { + fmt.Fprintln(stderr, err) } if *output == "json" { enc := json.NewEncoder(stdout) diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/check_test.go b/grafana-alertcheck/cmd/check_test.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/check_test.go rename to grafana-alertcheck/cmd/check_test.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/common.go b/grafana-alertcheck/cmd/common.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/common.go rename to grafana-alertcheck/cmd/common.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/env.go b/grafana-alertcheck/cmd/env.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/env.go rename to grafana-alertcheck/cmd/env.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/version.go b/grafana-alertcheck/cmd/grafana-alertcheck/version.go deleted file mode 100644 index 5cf68b6af..000000000 --- a/grafana-alertcheck/cmd/grafana-alertcheck/version.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "fmt" - "io" -) - -// Build metadata. These are package-level variables so goreleaser's ldflags -// (-X) can stamp them at build time; left unstamped they fall back to the -// "dev" defaults below, which is what a plain `go build` produces. -var ( - version = "dev" - commit = "unknown" - date = "unknown" - builtBy = "unknown" -) - -// runVersion prints the build metadata to stdout. Unlike list/watch/check it -// needs no Grafana connection, so it never touches the environment or the -// network; it exists purely so operators can answer "what am I running?" -// against a deployed binary. -func runVersion(args []string, stdout, stderr io.Writer) int { - if len(args) != 0 { - fmt.Fprintf(stderr, "version takes no arguments, got %v\n", args) - return 2 - } - fmt.Fprintf(stdout, "version: %s\ncommit: %s\ndate: %s\nbuiltBy: %s\n", version, commit, date, builtBy) - return 0 -} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/version_test.go b/grafana-alertcheck/cmd/grafana-alertcheck/version_test.go deleted file mode 100644 index 9463e990c..000000000 --- a/grafana-alertcheck/cmd/grafana-alertcheck/version_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestRunVersion(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run([]string{"version"}, &stdout, &stderr) - require.Equal(t, 0, code) - out := stdout.String() - for _, want := range []string{"version:", "commit:", "date:", "builtBy:"} { - require.Contains(t, out, want) - } - require.Empty(t, stderr.String()) -} - -func TestRunVersion_RejectsArgs(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run([]string{"version", "extra"}, &stdout, &stderr) - require.Equal(t, 2, code) -} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/list.go b/grafana-alertcheck/cmd/list.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/list.go rename to grafana-alertcheck/cmd/list.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/list_test.go b/grafana-alertcheck/cmd/list_test.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/list_test.go rename to grafana-alertcheck/cmd/list_test.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/main.go b/grafana-alertcheck/cmd/main.go similarity index 91% rename from grafana-alertcheck/cmd/grafana-alertcheck/main.go rename to grafana-alertcheck/cmd/main.go index 2672de1a3..7ab5d6397 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/main.go +++ b/grafana-alertcheck/cmd/main.go @@ -12,7 +12,7 @@ func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } -const usage = "usage: grafana-alertcheck " +const usage = "usage: grafana-alertcheck " // run is the whole of main's testable surface: parse the subcommand, dispatch, // return the process exit code. Exit codes below 2 (pass/violations) belong to @@ -37,8 +37,6 @@ func run(args []string, stdout, stderr io.Writer) int { return runWatch(args[1:], os.Stdin, stdout, stderr) case "check": return runCheck(args[1:], os.Stdin, stdout, stderr) - case "version": - return runVersion(args[1:], stdout, stderr) case "-h", "-help", "--help": fmt.Fprintln(stdout, usage) return 0 diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/main_test.go b/grafana-alertcheck/cmd/main_test.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/main_test.go rename to grafana-alertcheck/cmd/main_test.go diff --git a/grafana-alertcheck/cmd/style.go b/grafana-alertcheck/cmd/style.go new file mode 100644 index 000000000..bb46fc2ec --- /dev/null +++ b/grafana-alertcheck/cmd/style.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "io" + "os" + "strings" +) + +// ANSI SGR codes for the human-facing notes and table footer. The colours are +// applied only when the destination is a terminal (see colorEnabled); a pipe, +// file or CI log gets plain text, so stdout stays reserved for --output json +// and no machine reader ever sees escape sequences. +const ( + ansiReset = "\x1b[0m" + ansiRed = "\x1b[31m" + ansiGreen = "\x1b[32m" + ansiYellow = "\x1b[33m" + ansiCyan = "\x1b[36m" + // Orange has no entry in the base-16 palette; 256-colour 208 is a legible + // orange used for warnings, distinct from the yellow used for notes. + ansiOrange = "\x1b[38;5;208m" +) + +// colorEnabled reports whether ANSI colour should be written to w. Colour is +// written only when three things hold: NO_COLOR is unset, w is a real *os.File +// (so text/tabwriter buffers, strings.Builder and bytes.Buffer tests all stay +// plain), and that file is a character device (a terminal, not a redirect). +func colorEnabled(w io.Writer) bool { + if os.Getenv("NO_COLOR") != "" { + return false + } + f, ok := w.(*os.File) + if !ok { + return false + } + fi, err := f.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +// styleLine applies the note vocabulary's colour to one line when enabled. The +// colour wraps the text only; the terminating newline is written uncoloured so +// the terminal's line discipline is never inside the escape sequence. +func styleLine(line string, enabled bool) string { + if !enabled { + return line + } + content := strings.TrimRight(line, "\n") + var color string + switch { + case strings.HasPrefix(content, "warning:"): + color = ansiOrange + case strings.HasPrefix(content, "note:"): + color = ansiYellow + case strings.HasPrefix(content, "drain wait:"): + color = ansiCyan + } + if color == "" { + return line + } + return color + content + ansiReset + "\n" +} + +// noteStyler wraps the gate package's Notes stream — a presentation seam that +// keeps colour out of the library. It colourises each line by its known prefix +// and separates the collection countdown from the setup phase with a single +// blank line before the first "collecting:" line. The gate keeps emitting plain +// prose; only the CLI lays it out. +type noteStyler struct { + w io.Writer + enabled bool + pending []byte + sawCollecting bool +} + +func newNoteStyler(w io.Writer) *noteStyler { + return ¬eStyler{w: w, enabled: colorEnabled(w)} +} + +// startsSection reports whether a line opens a new phase of the stream and so +// deserves a blank line above it. "collecting:" opens the countdown (once — +// later countdown lines follow on from the first), and "drain wait:" opens the +// drain phase. The setup lines (planned run time, warning, min-observed, notes) +// are one contiguous block and are not separated from each other. +func (s *noteStyler) startsSection(line string) bool { + switch { + case strings.HasPrefix(line, "warning:"): + return true + case strings.HasPrefix(line, "drain wait:"): + return true + case strings.HasPrefix(line, "collecting:"): + if s.sawCollecting { + return false + } + s.sawCollecting = true + return true + } + return false +} + +func (s *noteStyler) Write(p []byte) (int, error) { + n := len(p) + s.pending = append(s.pending, p...) + for { + i := bytes.IndexByte(s.pending, '\n') + if i < 0 { + break + } + line := string(s.pending[:i+1]) + s.pending = s.pending[i+1:] + + if s.startsSection(line) { + if _, err := io.WriteString(s.w, "\n"); err != nil { + return n, err + } + } + if _, err := io.WriteString(s.w, styleLine(line, s.enabled)); err != nil { + return n, err + } + } + return n, nil +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/table.go b/grafana-alertcheck/cmd/table.go similarity index 55% rename from grafana-alertcheck/cmd/grafana-alertcheck/table.go rename to grafana-alertcheck/cmd/table.go index 11b72a074..39ec75cbc 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/table.go +++ b/grafana-alertcheck/cmd/table.go @@ -14,29 +14,39 @@ import ( // which the caller (runCheck) always points at stderr — stdout is reserved for // the machine-readable --output json. // -// Three sections, in order: +// Three titled tables, in order (the name column is RULE in all of them — one +// row is one resolved alert rule, never a firing instance): // -// 1. one line per rule: outcome, BadFor, pollEvery, proved-or-not with the -// largest gap; -// 2. one line per Violation: a rule's worst-of outcome does not carry the -// State/Health of the instance that actually caused it — Violation does — -// so this is also where those two columns appear, sorted after the rule -// table rather than folded into it, and it is the only place an operator -// running WITHOUT --output json sees the --allow-paused hint that -// Violation.Note already carries (classify.go); -// 3. a footer with the numbers that answer "why" on exit 2: each non-skipped -// rule's maxGap/healthGrace/evalStaleAfter, the global transitionGrace and -// drainTimeout, and the largest measured clock skew alongside its own -// error bound (RTT/2) — SkewHardLimit is a separate, fixed input threshold -// and is reported next to it, never as if it were that bound. +// 1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not +// with the largest gap; +// 2. VIOLATIONS, one line per Violation (only when any): a rule's worst-of +// outcome does not carry the State/Health of the instance that actually +// caused it — Violation does — so this is also where those two columns +// appear, sorted after the result table rather than folded into it, and it +// is the only place an operator running WITHOUT --output json sees the +// --allow-paused hint that Violation.Note already carries (classify.go); +// 3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped +// rule's maxGap/healthGrace/evalStaleAfter, followed by the global +// transitionGrace and drainTimeout, and the largest measured clock skew +// alongside its own error bound (RTT/2) — SkewHardLimit is a separate, +// fixed input threshold and is reported next to it, never as if it were +// that bound. func renderTable(w io.Writer, res gate.Result) error { alertOf := make(map[string]string, len(res.Verdicts)) for _, v := range res.Verdicts { alertOf[v.RuleUID] = v.Alert } + enabled := colorEnabled(w) + // A blank line separates the result table from the notes the gate streamed + // before it (planned run time, warning, min-observed, collecting, drain + // wait), so the verdict reads as its own section rather than the tail of a + // wall of progress text. + fmt.Fprintln(w) + + fmt.Fprintln(w, "RESULTS") tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(tw, "ALERT\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE") + fmt.Fprintln(tw, "RULE\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE") for _, v := range sortedVerdicts(res.Verdicts) { fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", v.Alert, v.Outcome, v.BadFor.Round(time.Second), v.PollEvery.Round(time.Second), @@ -49,7 +59,7 @@ func renderTable(w io.Writer, res gate.Result) error { if len(res.Violations) > 0 { fmt.Fprintln(w, "\nVIOLATIONS") vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(vtw, "ALERT\tOUTCOME\tSTATE\tHEALTH\tNOTE") + fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tNOTE") for _, v := range sortedViolations(res.Violations) { fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\n", alertLabel(v, alertOf), v.Outcome, v.State, v.Health, v.Note) } @@ -58,20 +68,49 @@ func renderTable(w io.Writer, res gate.Result) error { } } + // The per-rule thresholds answer "why" on exit 2: a table, not the prose + // "rule NAME: maxGap=... healthGrace=... evalStaleAfter=..." that repeated + // the rule name a fourth time. It is separated from the result above by a + // blank line. fmt.Fprintln(w) + fmt.Fprintln(w, "THRESHOLDS") + ttw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(ttw, "RULE\tMAXGAP\tHEALTHGRACE\tEVALSTALEAFTER") for _, uid := range sortedThresholdUIDs(res.Thresholds, alertOf) { t := res.Thresholds[uid] - fmt.Fprintf(w, "rule %s: maxGap=%s healthGrace=%s evalStaleAfter=%s\n", + fmt.Fprintf(ttw, "%s\t%s\t%s\t%s\n", alertOr(uid, alertOf), t.MaxGap, t.HealthGrace, t.EvalStaleAfter) } + if err := ttw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + + fmt.Fprintln(w) fmt.Fprintf(w, "global: transitionGrace=%s (source: %s) drainTimeout=%s\n", res.Global.TransitionGrace, res.Global.GraceSource, res.Global.DrainTimeout) - fmt.Fprintf(w, "violations: %d, largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n", - len(res.Violations), res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond), + fmt.Fprintf(w, "largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n", + res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond), gate.SkewHardLimit, res.GrafanaVersion) + // The verdict — the single number a terminal operator reads last — sits on + // its own line at the very bottom, separated from the diagnostics above and + // from the shell prompt below. + fmt.Fprintf(w, "\n%s\n\n", violationsLabel(len(res.Violations), enabled)) return nil } +// violationsLabel colours the "violations: N" prefix of the footer: green for a +// clean run, red otherwise. The rest of the line is written uncoloured. +func violationsLabel(n int, enabled bool) string { + s := fmt.Sprintf("violations: %d", n) + if !enabled { + return s + } + if n == 0 { + return ansiGreen + s + ansiReset + } + return ansiRed + s + ansiReset +} + // provedLabel is the table's PROVED column: "yes" for a clean coverage // proof, "no" with the reason and largest gap for an unobservable rule, and // "-" for a rule decide never asked proveCoverage about at all (skipped — diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go b/grafana-alertcheck/cmd/table_test.go similarity index 90% rename from grafana-alertcheck/cmd/grafana-alertcheck/table_test.go rename to grafana-alertcheck/cmd/table_test.go index 9da9ea3ba..86c80a19b 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go +++ b/grafana-alertcheck/cmd/table_test.go @@ -67,11 +67,13 @@ func TestRenderTable(t *testing.T) { require.Contains(t, out, string(gate.StateFiring)) require.Contains(t, out, "error") - // The footer: per-rule thresholds, global thresholds, and skew with its own - // bound rather than the fixed hard limit. - require.Contains(t, out, "Ape Alert: maxGap=1m0s healthGrace=2m0s evalStaleAfter=1m0s") - require.Contains(t, out, "Zebra Alert: maxGap=1m0s healthGrace=1m0s evalStaleAfter=1m0s") - require.NotContains(t, out, "Paused Alert: maxGap") + // The footer: per-rule thresholds are a table (RULE/MAXGAP/HEALTHGRACE/ + // EVALSTALEAFTER) rather than prose, followed by the global thresholds and + // the violations count with the skew and its own bound rather than the + // fixed hard limit. + require.Contains(t, out, "MAXGAP") + require.Contains(t, out, "HEALTHGRACE") + require.Contains(t, out, "EVALSTALEAFTER") require.Contains(t, out, "global: transitionGrace=5m0s (source: Ape Alert (for=5m)) drainTimeout=2m0s") require.Contains(t, out, "largest measured clock skew: 1.5s (bound ±250ms, hard limit 1m0s)") require.Contains(t, out, "violations: 2") diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/watch.go b/grafana-alertcheck/cmd/watch.go similarity index 95% rename from grafana-alertcheck/cmd/grafana-alertcheck/watch.go rename to grafana-alertcheck/cmd/watch.go index df2c50029..69b585298 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/watch.go +++ b/grafana-alertcheck/cmd/watch.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "flag" "fmt" "io" @@ -49,6 +50,13 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { readyFD := fs.Int(gate.ReadyFDFlag[2:], 0, "") if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(stderr, "watch: unexpected arguments %v\n", fs.Args()) return 2 } @@ -78,7 +86,7 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { DaemonLog: *daemonLog, Concurrency: *common.concurrency, Clock: gate.SystemClock{}, - Notes: stderr, + Notes: newNoteStyler(stderr), } if *until != "" { t, err := time.Parse(time.RFC3339, *until) diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go b/grafana-alertcheck/cmd/watch_test.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go rename to grafana-alertcheck/cmd/watch_test.go diff --git a/grafana-alertcheck/docs/_category_.yaml b/grafana-alertcheck/docs/_category_.yaml new file mode 100644 index 000000000..3cbc431c4 --- /dev/null +++ b/grafana-alertcheck/docs/_category_.yaml @@ -0,0 +1,8 @@ +position: 1 +label: 'Grafana Alertcheck' +collapsible: true +collapsed: false +link: + type: generated-index + slug: /platform-services/devex/cicd/grafana-alertcheck/index + description: 'CD quality gate for Grafana alerts: watch, classify, and gate releases.' diff --git a/grafana-alertcheck/docs/advanced.md b/grafana-alertcheck/docs/advanced.md new file mode 100644 index 000000000..073292cd6 --- /dev/null +++ b/grafana-alertcheck/docs/advanced.md @@ -0,0 +1,41 @@ +--- +id: grafana-alertcheck-advanced +title: Check budget and scheduling +sidebar_label: Budget and scheduling +sidebar_position: 2 +description: Why grafana-alertcheck schedules per rule, how the request budget works, and why it never queries state history. +--- + +# Check budget and scheduling + +## Per-rule schedules, never a global cycle + +Each rule polls at its **own** cadence, `--poll-interval` (default: half the rule's own evaluation interval). There is deliberately no single global minimum-interval cycle. + +One rule at `intervalSeconds=10` beside twenty at `300` keeps a 5 s cadence for itself and 150 s for the other twenty — not a 5 s cycle for all of them, which would be a 60× request bloat at ~1.8 s per request and would fail to start on a reasonable fleet. + +The scheduler staggers each rule's initial next-due time across its cadence, and serves due rules **earliest-due-first**, so a tight rule never queues behind slack ones. + +## The check budget + +The gate records one observation of every rule up front and checks the schedule against those **measured** latencies (payload sizes vary ~230× across rules, so a fixed estimate is meaningless). It errors at start — before waiting — if any of three conditions hold: + +- **Utilization** — total request rate exceeds `--concurrency`. +- **Per-rule** — one rule's request can't fit its own cadence. +- **Burst bound** — the slowest request exceeds the fleet's tightest cadence, which can open a mid-run gap. + +The error names the three levers only: raise `--concurrency`, raise `--poll-interval`, or watch fewer alerts. It never prescribes a single interval. + +## Why the gate never queries state history + +Querying Grafana's alert state history after the fact fails closed *in the wrong direction* — it returns "pass" when the truth is unknown: + +- History stores **transitions**, not states. An alert firing through the whole window has its only record *before* the window. +- The annotations API **does not serve Loki-backed history** at all. +- An empty result is indistinguishable from a healthy one: no alert fired, the backend differs, retention removed data, the token lacked permission — all look identical. +- There is **no coverage signal** — nothing proves the history is complete to time T. +- Artifact transitions (`Paused`, `RuleDeleted`, `Updated`, `MissingSeries`) look like recoveries. + +Instead, `watch` records its own evidence live and the log becomes the source of truth. The trade-off: the gate can miss an episode shorter than a rule's poll interval, though `activeAt` still surfaces sub-interval onsets for instances still active at a poll. + +A corollary of recording fresh: there is no replay. Re-running a failed job is a new deploy with a new `from` and a new recording — never a re-classification of old evidence. diff --git a/grafana-alertcheck/docs/architecture.md b/grafana-alertcheck/docs/architecture.md new file mode 100644 index 000000000..ca493220e --- /dev/null +++ b/grafana-alertcheck/docs/architecture.md @@ -0,0 +1,68 @@ +--- +id: grafana-alertcheck-architecture +title: Architecture +sidebar_label: Architecture +sidebar_position: 3 +description: The design invariants, pure-function seam, and recorder lifecycle of grafana-alertcheck, for maintainers. +--- + +# Architecture + +This page documents the invariants and seams a maintainer must not break. It exists because most of them are the difference between a gate that fails closed and one that silently passes broken windows. + +## Fail-closed invariants + +The gate must stop the release if it cannot get an answer. Every rule below is a specific instance of that: + +- **An error is never a pass.** A pass is exactly `len(Violations) == 0 && err == nil`. Every error path leaves `err` non-nil, and the CLI maps that to exit `2` unconditionally. +- **Inability beats violation.** Any `unobservable` rule is exit `2`, even alongside a real violation found first. +- **Absent never means normal.** An instance that leaves the bad set is looked up in the *same* response: present as `normal` → cleared; absent (or `MissingSeries`) → vanished (a discontinuity, not a recovery). +- **Staleness is absolute.** `grafana_now − lastEvaluation` is compared against a threshold, never "did it increase since the last poll" — a delta check reports stale on ~half the polls of a healthy rule. +- **`grafana_now` is the response `Date` header.** Never the runner clock, in any comparison against a Grafana timestamp. +- **No early exit.** `check` collects to `to + transitionGrace` before classifying once. +- **No replay.** No run-id key, no artifact download, no state between attempts. A retry is a new deploy. + +## The pure-function seam + +All correctness lives in two phases written as **pure functions** over a flat list of polls — no HTTP, no files, no clock, no goroutines: + +``` +HTTP ──> Source ──> []StateRule ──> reduce ──> []Poll ──> proveCoverage ──> decide ──> Result + │ + JSONL log ──> ReadLog ──┘ +``` + +- `proveCoverage` (the nine coverage checks) and `decide` (the instance timelines and outcomes) are pure; tests drive them with `[]Poll` literals and a fake `Clock`, with no sleeping or fixture server. +- `Check`/`Watch` are I/O shells: HTTP, signals, the pidfile, file reads, the countdown print. The only test doubles needed are the `Source` and `Clock` interfaces. +- `Policy` is the narrowed view of `Config` that reaches the pure layer — classification knobs and the window, no URL and no token. The token must never cross that line, which is the cheapest guarantee it never lands in an error string or a result. + +## Strict parsing as the version guard + +Both API responses are parsed strictly: a missing or unparseable **required** field (`health`, `state`, `lastEvaluation`, `interval`) is an error, never a zero value. Optional keys (`alerts`, `totals`, `labels`, `keepFiringFor`) are absent-tolerant, and unknown keys are ignored — so Grafana can add fields without breaking the parser, but removing one fails loudly. + +This, plus the declared supported range (Grafana >= 13.0.0, < 14.0.0), is how a deprecation or schema change is caught instead of silently misread. + +## The recorder lifecycle + +`watch` detaches a background recorder so observation survives the step boundary: + +1. Parent resolves names, writes the header, observes every non-paused rule once, checks the budget. +2. Parent re-execs itself as the child (`--daemon-child`) under a new session/process group, stdout/stderr to the daemon log. +3. Child re-reads the header, reopens the log `O_APPEND`, takes the exclusive `flock`, and writes one readiness byte on `--ready-fd`. +4. Parent writes the pidfile **after** the readiness report, then returns. + +Two authorities, only one of which is evidence: + +- The **pidfile** says a recording ever started (written only after ready, removed on failure). It can go stale — a pid gets reused. +- The **flock** says a writer exists *now*. The kernel drops it on exit, so the lock is always authoritative. + +On a clean stop (SIGTERM/SIGINT/`--until`) the child finishes the in-flight write, appends the `stopped` sentinel, fsyncs, and exits. A hard error writes no sentinel — so a recorder that died reads exactly like a coverage gap, because it is one. + +`check` signals via the pidfile, waits for the **lock** to release (never the pid), and only then reads the log once. Reading while a writer can still append can only produce a shorter window than was recorded. + +## The log is the source of truth + +`watch` records raw evidence, so nothing trusts a state that could become unreachable. Two consequences a maintainer must preserve: + +- The **header is authoritative for recording facts** (the cadence actually used, the URL, the alert set); the ruler API is authoritative for **rule facts** (`for`, `intervalSeconds`, kind). `check` always re-resolves definitions fresh and never reconstructs them from the header — the header duplicates `for`/`interval` only so the uploaded artifact is self-describing. +- The **cadence authority** is the header's `poll_every_seconds`, not the definitions. Re-deriving it would compare gaps recorded at an override cadence against default-cadence thresholds — fail-open in the faster-override direction. diff --git a/grafana-alertcheck/docs/how-alerts-are-evaluated.md b/grafana-alertcheck/docs/how-alerts-are-evaluated.md new file mode 100644 index 000000000..7c9cb3b6a --- /dev/null +++ b/grafana-alertcheck/docs/how-alerts-are-evaluated.md @@ -0,0 +1,85 @@ +--- +id: grafana-alertcheck-evaluation +title: How alerts are evaluated +sidebar_label: How alerts are evaluated +sidebar_position: 1 +description: The verdict model, instance timelines, and coverage proof behind grafana-alertcheck. +--- + +# How alerts are evaluated + +Both `watch`+`check` (recorder mode) and `check` alone (single-step mode) converge on the same input: a flat list of polls. Everything below runs over that list; the mode only changes where the polls came from. + +## Instance states + +Grafana reports instance states in two vocabularies (`Alerting`/`Normal` at instance level, `firing`/`inactive` at rule level). The gate normalizes every instance to one canonical set: + +| Canonical | Meaning | +| --------- | ------- | +| `normal` | Healthy | +| `firing` | The condition is true and `for` has elapsed | +| `pending` | The condition is true, `for` has not elapsed | +| `nodata` | The query returned no series (synthetic instance) | +| `error` | The query failed (synthetic instance) | + +A rule's **rule-level** `state` and `health` are kept verbatim and only reported — they are never classified. The **instance** state is what the classifier reasons about. + +A "bad" instance is one whose canonical state is in `--states` (default `firing`). `pending` and `nodata` are excluded by default. + +## Verdict model + +For each instance the gate builds a timeline of bad spans over `[from, to]`, then takes the worst outcome across a rule's instances as the rule's outcome. + +| Outcome | Shape | Exit | +| ------- | ----- | ---- | +| `clean` | Good throughout, observed throughout | → 0 | +| `newly_bad` | Entered a bad state **inside** the window | → 1 | +| `persistently_bad` | Bad at `from`, still bad at `to` | → 1 | +| `recovered` | Bad at `from`, cleared before `to`, stayed clear | → 0 | +| `flapping` | Cleared, then became bad again | → 1 | +| `skipped` | Paused **before** the window opened | reported, not observable | +| `unobservable` | Coverage gap / sustained `health=error` / stale / absent | → 2 | + +`recovered` has **no deadline** — an alert that clears at minute 58 of a 60-minute window still passes. The total bad time is reported as `BadFor`; the removed deadline is replaced by that measured value rather than a derived limit. + +### Preexisting policy + +For an instance already bad when `from` opened, `--preexisting` decides: + +- `fail-unless-recovered` (default) — clears and stays clear → pass; never clears → fail. +- `fail` — any preexisting instance fails, recovered or not. +- `ignore` — preexisting instances are disregarded; only new episodes fail. + +## Cleared vs vanished + +When an instance leaves the bad set, the gate looks it up **in the same response**: + +- Present as `normal` → `cleared` (a real recovery). +- Absent, or present as `normal (MissingSeries)` → `vanished` (a discontinuity, **not** a recovery). + +A vanished instance that was bad stays `persistently_bad`. A metric that stops being emitted is not evidence of health — this is deliberate and can surprise users whose fix is to remove a metric rather than drive it to a good value. + +## Coverage proof + +Before classifying, `check` must **prove** continuous coverage of `[from, to]` for each alert. Nine checks run; any failure makes the rule `unobservable`: + +1. **Sentinel** — a clean recorder stop, timestamped at or after `to + transitionGrace`. A recorder that died mid-window looks exactly like a coverage gap and is one. +2. **`from` bounds** — `from` earlier than the recording start is unprovable. +3. **Heartbeat gap** — any gap larger than `maxGap` (= 2 × poll cadence) inside the window. Data at both ends with a hole between is not enough. +4. **`health=error`** — a contiguous run longer than `healthGrace` consumes coverage; a short blip is a note. +5. **`health=nodata`** — a note, never fatal (unless `--nodata-is-unobservable`). +6. **Liveness** — `grafana_now − lastEvaluation` must not exceed `evalStaleAfter`. This is an **absolute** check, never a "did it increase since the last poll" delta. +7. **In-window pause** — a poll reporting `isPaused` mid-window is `unobservable` (the primary pause detector). +8. **Rule absent** — an authoritative `2xx` with no matching rule. +9. **`KeepLast`** — a note naming a stale-state blind spot. + +## Health: `error` vs `nodata` + +- `health=error` means the query **failed** — a malfunction. Sustained past `healthGrace`, it makes the rule `unobservable`. +- `health=nodata` means the query **ran and returned no series** — indistinguishable from a quiet system. It is not fatal by default; most of a fleet runs `no_data_state: OK`. + +## The drain wait and `transitionGrace` + +A condition that arises just before `to` becomes `firing` only at the first evaluation after its `for` elapses. `transitionGrace` (derived from the watched rules' `for` values) extends the classification bound past `to` so such a surfacing condition is caught. After collection, a **drain wait** polls until each rule has evaluated through `to + transitionGrace` (bounded by `drainTimeout`); a rule that never does is `unobservable`. + +Run time = `(to − from) + transitionGrace + drainTimeout`. This is printed at start, and the grace is warned about when it exceeds a quarter of the window — the window may be too short for the alert's `for`. diff --git a/grafana-alertcheck/docs/index.md b/grafana-alertcheck/docs/index.md new file mode 100644 index 000000000..95d413772 --- /dev/null +++ b/grafana-alertcheck/docs/index.md @@ -0,0 +1,87 @@ +--- +id: grafana-alertcheck-index +title: Grafana Alertcheck +sidebar_label: Overview +sidebar_position: 0 +description: A CD quality gate that bookends a release with alert-state observation and answers whether any watched Grafana alert was bad during the release window. +--- + +# Grafana Alertcheck + +`grafana-alertcheck` is a CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and `check` (classify) — and answers one question: + +> A release finished at time T. Was any of these Grafana alerts in a bad state during the next N minutes? + +The contract is `watch → your work → check`. Between the two you run whatever you want (deploy, tests, migration); the gate only observes, then classifies. + +It **fails closed**: if it cannot get an answer, it stops the release. It never passes an unproven window. + +## How it works, in one paragraph + +`watch` starts a background recorder that polls each named alert and appends snapshots to a JSONL log. Your work then emits two RFC3339 timestamps — `from` (when the change landed) and `to` (when the work ended). `check` proves continuous coverage of `[from, to]`, builds a state timeline per alert, classifies it, and exits `0`, `1`, or `2`. + +## Install + +```bash +go install github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd@latest +``` + +Connection details come from the environment — the token is env-only, never a flag: + +```bash +export GRAFANA_URL=https://grafana.example.com +export GRAFANA_TOKEN=… +``` + +Requires Grafana >= 13.0.0 and < 14.0.0. Outside that range the gate exits `2`. + +## Quickstart — recorder mode + +```bash +grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt +./deploy.sh # emits deployed_at= when the rollout is stable +./verify.sh # emits finished_at= when the work is done +grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" +``` + +`alerts.txt` holds one alert name per line. See [Naming alerts](./reference/cli#naming-alerts). + +`watch` returns only after the recorder has observed every named, non-paused alert once and reported ready — so auth, name-resolution, and parse failures surface **before** your deploy runs. + +## Quickstart — single-step mode + +Skip the recorder and observe the window inline, from inside `check` itself: + +```bash +grafana-alertcheck check --alerts alerts.txt --to "$finished_at" +``` + +In single-step mode the window starts at `check`'s first observation; if you give no `--from`, the interval before that first observation is declared as a blind spot with a warning (not an error). + +## Exit codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Pass — no violations | +| `1` | Violations (including a paused-rule-only `--min-observed` shortfall) | +| `2` | The gate could not check — config, auth, resolution, a coverage gap, health/staleness, the drain limit, transport, … | + +An error is never a pass: `2` wins over any violation found alongside it. + +## Common surprises + +- A **paused rule fails by default** — even one someone else paused. Use `--allow-paused`. +- A fix that **stops emitting a metric is not a recovery** — the instance vanishes, which is a discontinuity, not health. +- The gate checks alert **state and health**, not notification delivery — a silenced alert that still fires fails. +- `recovered` has **no deadline** — a bad-at-`from` alert that clears by `to` passes; set `--preexisting fail` to forbid it. +- A **retry is a new deploy**, not a replay — re-running the job re-records against a new `from`. +- `watch` and `check` must run in **one job, one runner, one filesystem** — nothing persists across jobs or attempts. +- The gate **never exits early** — a violation at minute 2 still holds the runner to `to + transitionGrace + drainTimeout`; size the job timeout to the planned run time the gate prints at start. + +## More + +- [How alerts are evaluated](./how-alerts-are-evaluated) — the verdict model and coverage proof +- [Check budget and scheduling](./advanced) — why the schedule and budget look the way they do, and why history isn't queried +- [Architecture](./architecture) — design invariants and the recorder lifecycle, for maintainers +- [CLI reference](./reference/cli) — every subcommand and flag +- [Log format](./reference/log-format) — the JSONL log schema, for debugging artifacts diff --git a/grafana-alertcheck/docs/reference/_category_.yaml b/grafana-alertcheck/docs/reference/_category_.yaml new file mode 100644 index 000000000..2e5d50946 --- /dev/null +++ b/grafana-alertcheck/docs/reference/_category_.yaml @@ -0,0 +1,8 @@ +position: 3 +label: Reference +collapsible: true +collapsed: false +link: + type: generated-index + slug: /platform-services/devex/cicd/grafana-alertcheck/reference + description: 'CLI reference for grafana-alertcheck.' diff --git a/grafana-alertcheck/docs/reference/cli.md b/grafana-alertcheck/docs/reference/cli.md new file mode 100644 index 000000000..1530c508d --- /dev/null +++ b/grafana-alertcheck/docs/reference/cli.md @@ -0,0 +1,91 @@ +--- +id: grafana-alertcheck-cli +title: CLI reference +sidebar_label: CLI reference +sidebar_position: 0 +description: Full reference for the grafana-alertcheck CLI: watch, check, list, environment, naming, and output. +--- + +# CLI reference + +``` +grafana-alertcheck +``` + +Connection details are always from the environment: `GRAFANA_URL` and `GRAFANA_TOKEN`. The token is never a flag and never logged. + +## `list` + +Lists every rule from the ruler endpoint — kind, folder, group, title, uid. Useful to check auth and to find `uid:` names. + +```bash +grafana-alertcheck list +``` + +## `watch` — record + +```bash +grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] \ + --alerts [--folder F] [--poll-interval D] [--concurrency N] [--until RFC3339] +``` + +| Flag | Default | Meaning | +| ---- | ------- | ------- | +| `--out` | — | JSONL log path (required) | +| `--pidfile` | `.pid` | Where the recorder's pid is written | +| `--daemon-log` | `.daemon.log` | stdout/stderr sink for the detached recorder | +| `--alerts` | — | File of alert names, one per line, or `-` for stdin (required) | +| `--folder` | — | Default folder to scope unqualified names | +| `--poll-interval` | half the rule's interval | Override every rule's cadence (never clamped) | +| `--concurrency` | `1` | Max concurrent requests to Grafana | +| `--until` | run until signalled | Optional hard stop | + +`watch` writes the header, observes every non-paused rule once, checks the budget, then detaches a background recorder and returns. Recording is **unfiltered** — there is no `--states` here, so the same log can be re-classified later under different `--states` without re-recording. + +## `check` — classify + +```bash +grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 \ + [--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] \ + [--allow-paused] [--nodata-is-unobservable] [--concurrency N] [--output json] +``` + +| Flag | Default | Meaning | +| ---- | ------- | ------- | +| `--in` | — | Log recorded by `watch`; empty selects single-step mode | +| `--pidfile` | `.pid` | Recorder to stop before reading `--in` | +| `--from` | see below | Moment the deploy finished | +| `--to` | — | End of the window (required) | +| `--alerts` | — | Required **without** `--in`; refused **with** `--in` | +| `--states` | `firing` | Comma-separated bad states: `firing,pending,nodata,error` | +| `--preexisting` | `fail-unless-recovered` | `fail-unless-recovered` \| `fail` \| `ignore` | +| `--min-observed` | every resolved rule | Minimum rules that must be observed | +| `--allow-paused` | `false` | Don't count pre-window-paused rules against `--min-observed` | +| `--nodata-is-unobservable` | `false` | Treat sustained `health=nodata` as unobservable | +| `--concurrency` | `1` | Max concurrent requests | +| `--output` | `table` | `json` also writes the machine-readable result to stdout | + +`--from` and `--to` are RFC3339 with an explicit offset and must come from your work — `from` from the deploy step, `to` from the step that finishes. In recorder mode an absent `--from` is a hard error; in single-step mode it falls back (with a warning) to the start of the step. + +## Naming alerts + +Alert names take one of four forms: + +| Form | Meaning | +| ---- | ------- | +| `HighErrorRate` | Title only, scoped by `--folder` | +| `Platform/HighErrorRate` | Folder + title | +| `Platform/api/HighErrorRate` | Folder + group + title (always unique) | +| `uid:abc123` | Exact uid (present on both endpoints) | + +Datasource-managed and recording rules are refused with a specific error. A name matching multiple rules errors listing every candidate with the copyable `Folder/Group/Title` and its `uid:` form. A no-match errors with case-insensitive substring suggestions and points at `list`. Duplicate names that resolve to the same uid collapse to one (a note, not an error). + +## Output and exit codes + +The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per violation), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout. + +| Code | Meaning | +| ---- | ------- | +| `0` | Pass | +| `1` | Violations | +| `2` | Could not check — every library error, never a pass | diff --git a/grafana-alertcheck/docs/reference/log-format.md b/grafana-alertcheck/docs/reference/log-format.md new file mode 100644 index 000000000..66e071770 --- /dev/null +++ b/grafana-alertcheck/docs/reference/log-format.md @@ -0,0 +1,98 @@ +--- +id: grafana-alertcheck-log-format +title: Log format +sidebar_label: Log format +sidebar_position: 1 +description: The JSONL log schema written by watch and read by check, for debugging the forensic artifact. +--- + +# Log format + +`watch` records evidence to a JSONL log — one JSON object per line. A poll record *is* the heartbeat; there is no separate heartbeat type. + +## Record types + +Exactly three: + +| `type` | Meaning | +| ------ | ------- | +| `header` | Line 1 — identity and the alert set | +| `poll` | One reduced observation of one rule | +| `stopped` | The sentinel, written on a clean stop only | + +The header must be line 1, appear once, and carry `schema_version` `1` (any other value is a read error). Any unparseable line — including the last, or one after the sentinel — makes the log unreadable: a truncated log is evidence the recorder was killed, and must not pass. + +## Header + +```json +{ + "type": "header", + "schema_version": 1, + "url": "https://grafana.example.com", + "grafana_version": "13.1.0", + "started_at": "2026-09-07T10:00:00Z", + "rules": [ + { + "uid": "rule0000001", + "title": "HighErrorRate", + "folder": "Platform", + "group": "api", + "for_seconds": 300, + "interval_seconds": 60, + "is_paused": false, + "no_data_state": "OK", + "exec_err_state": "OK", + "poll_every_seconds": 30 + } + ] +} +``` + +- `url` and `rules` are the log's identity — `check` validates them against the current environment and a fresh ruler read. +- `is_paused` records the pause state at record start (the moment `skipped` means). +- `poll_every_seconds` is the cadence the recording **actually used** (after any `--poll-interval` override). `check` derives `maxGap` from it, never from `interval_seconds`. +- `for_seconds`, `interval_seconds`, `no_data_state`, `exec_err_state` are forensic only — `check` re-resolves definitions and never reads them back. + +## Poll + +```json +{ + "type": "poll", + "rule_uid": "rule0000001", + "grafana_now": "2026-09-07T10:00:30Z", + "skew_ms": 20, + "skew_bound_ms": 40, + "latency_ms": 123, + "found": true, + "state": "inactive", + "health": "ok", + "last_evaluation": "2026-09-07T10:00:28Z", + "is_paused": false, + "histogram": { "alerting": 0, "normal": 2004 }, + "reasons": { "NoData": 1091 }, + "abnormal": [ { "labels": { "env": "prod" }, "state": "firing", "active_at": "2026-09-07T09:50:00Z", "value": "1.5" } ], + "cleared": [ "env=prod\u0001..." ], + "vanished": [] +} +``` + +Field notes: + +- `grafana_now` is the response's `Date` header — never the runner clock. +- `skew_ms`/`skew_bound_ms` are the per-poll clock-skew estimate and its uncertainty (RTT/2), in milliseconds for compactness only. +- `found: false` is an authoritative `2xx` in which this rule was absent — a transport failure is retried and never becomes a poll. +- `state`, `health`, `last_error` are raw rule-level strings, reporting-only. +- `histogram` is a verbatim copy of the response `totals`; written, never analysed. +- `reasons` counts non-empty instance reasons (`NoData`, `Error`, `KeepLast`, …); composite states stay visible only here. +- `abnormal` holds only instances whose **canonical** state is not `normal`. +- `cleared`/`vanished` are instance keys that left the bad set, resolved against the same response: `cleared` = a real recovery; `vanished` = a discontinuity, never a recovery. + +Instance keys are a sorted `k=v\n` join of labels, so they correlate across polls without hashing. + +## Stopped + +```json +{ "type": "stopped", "at": "2026-09-07T10:10:30Z" } +``` + +`at` is the recorder's own stop time. `check` compares it against `to + transitionGrace`; absent or earlier is `unobservable` — never a pass. diff --git a/grafana-alertcheck/internal/gate/check.go b/grafana-alertcheck/internal/gate/check.go index 4c4d6cd2f..ef6543b71 100644 --- a/grafana-alertcheck/internal/gate/check.go +++ b/grafana-alertcheck/internal/gate/check.go @@ -238,15 +238,10 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { logHasHdr = true resolved, notes, err = resolveFromLog(allDefs, earlyHdr, cfg) if err == nil { - // Fail fast on a statically-knowable bound violation. `from < - // StartedAt` makes coverage unprovable no matter how healthy the polls - // that DO exist look, and StartedAt is immutable (line 1, written - // first), so this cannot disagree with the authoritative header read - // later. proveCoverage's check 2 remains the backstop against the - // authoritative header, so a bad advisory read can only ever fail - // closed, never produce a false pass. This is recorder mode only: the - // single-step branch has no header, and its own `from < startedAt` is - // a warning-and-pass (see below), not an error. + // Fail fast on a bound violation that can't change: StartedAt is + // immutable (line 1), so check 2's backstop still catches any bad + // advisory read — fail closed, never false-pass. Recorder mode only; + // single-step warns-and-passes (see below). if from.Before(earlyHdr.StartedAt) { return Result{}, fmt.Errorf("check: `from` %s is before recording started at %s", from.Format(time.RFC3339), earlyHdr.StartedAt.Format(time.RFC3339)) @@ -286,6 +281,16 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { } summary, warning := StartupSummary(from, cfg.To, gt) fmt.Fprintln(cfg.Notes, summary) + // MinObserved is printed with the plan, beside "planned run time", rather + // than after it: it is a fact about the run, not a diagnostic. Its default + // is the resolved rule count AFTER duplicate names collapse, which is + // len(resolved) by construction; decide defaults it identically, and it is + // resolved here rather than inferred from the verdict afterwards. + minObserved := cfg.MinObserved + if minObserved == 0 { + minObserved = len(resolved) + } + fmt.Fprintf(cfg.Notes, "min-observed: %d of %d resolved rule(s)\n", minObserved, len(resolved)) if warning != "" { fmt.Fprintf(cfg.Notes, "warning: %s\n", warning) } @@ -336,17 +341,6 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { } } - // ---- Apply MinObserved. ----------------------------------------------- - // Its default is the resolved rule count AFTER duplicate names collapse, - // which is len(resolved) by construction. decide defaults it identically; - // it is resolved here as well so the value the run will judge against is - // printed before the wait rather than inferred from the verdict afterwards. - minObserved := cfg.MinObserved - if minObserved == 0 { - minObserved = len(resolved) - } - fmt.Fprintf(cfg.Notes, "min-observed: %d of %d resolved rule(s)\n", minObserved, len(resolved)) - // ---- Collect the evidence. -------------------------------------------- // Collect ONLY. No classification happens here and there is no early exit, // even once a violation is certain: the loop always runs to diff --git a/grafana-alertcheck/internal/gate/check_test.go b/grafana-alertcheck/internal/gate/check_test.go index 6029d5bb7..780184bd3 100644 --- a/grafana-alertcheck/internal/gate/check_test.go +++ b/grafana-alertcheck/internal/gate/check_test.go @@ -657,6 +657,32 @@ func TestCheckFailFastWhenFromPrecedesRecordStart(t *testing.T) { require.True(t, clock.Now().Equal(testNow), "it must fail before the wait") } +// A whole-second `from` in the same second as the recording's sub-second +// StartedAt is not a blind interval: the whole-second comparison lets the run +// proceed to a clean pass instead of the fail-fast above. +func TestCheckRecorderModeFromSameSecondAsStartedAtPasses(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + // StartedAt is 500ms after `from` (testNow via recorderConfig) — the same + // whole second. Polls still cover the whole window. + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(500*time.Millisecond), testNow.Add(-time.Minute), windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow.Add(time.Minute)) + cfg := recorderConfig(t, clock, logPath) + src := newCheckSource(func(title string, _ int) (Observation, error) { + require.Fail(t, fmt.Sprintf("the drain wait polled %q although the log already proves the evaluations", title)) + return Observation{}, errors.New("unexpected poll") + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) +} + // The coverage proof failed: a hole in the middle of the recording is not // saved by healthy data at both ends. func TestCheckFailClosedOnCoverageGap(t *testing.T) { diff --git a/grafana-alertcheck/internal/gate/classify.go b/grafana-alertcheck/internal/gate/classify.go index d5d9c8e12..d8d058a34 100644 --- a/grafana-alertcheck/internal/gate/classify.go +++ b/grafana-alertcheck/internal/gate/classify.go @@ -476,7 +476,7 @@ func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, Thresholds: make(map[string]RuleThresholds), Global: GlobalThresholds{ TransitionGrace: gt.transitionGrace, - GraceSource: gt.graceSource, + GraceSource: graceSourceOrNone(gt.graceSource), DrainTimeout: gt.drainTimeout, }, } diff --git a/grafana-alertcheck/internal/gate/coverage.go b/grafana-alertcheck/internal/gate/coverage.go index 527720bfe..603a7f7d8 100644 --- a/grafana-alertcheck/internal/gate/coverage.go +++ b/grafana-alertcheck/internal/gate/coverage.go @@ -88,12 +88,16 @@ func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, d // Check 2 — from bounds: from < StartedAt makes coverage unprovable, no // matter how healthy the polls that DO exist look. Both are runner-domain // clock reads (the recorder's own Clock.Now()), so no cross-domain - // translation applies here. The other half of the bound — from too far - // ahead of the runner's clock — is Check's input validation, once per run - // rather than per rule. - if from.Before(h.StartedAt) { + // translation applies here. The comparison is at whole-second granularity: + // `from` is supplied at second precision (--from RFC3339) while StartedAt + // carries the recorder's sub-second clock stamp, so an operator naming the + // exact second the recording opened must not be judged early for the + // sub-second sliver inside that same second. The other half of the bound — + // from too far ahead of the runner's clock — is Check's input validation, + // once per run rather than per rule. + if from.Truncate(time.Second).Before(h.StartedAt.Truncate(time.Second)) { fail(ReasonFromBeforeRecord, fmt.Sprintf( - "requested from %s is before recording started at %s", from.Format(time.RFC3339), h.StartedAt.Format(time.RFC3339))) + "requested from %s is before recording started at %s", from.Format(time.RFC3339Nano), h.StartedAt.Format(time.RFC3339Nano))) } // Filtered once and threaded through every remaining check. @@ -142,7 +146,7 @@ func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, d // corrupted or hand-edited data (ReadLog does no field validation); // GrafanaNow-LastEvaluation would go negative and silently read as // fresh — fail-open. Treat it as unobservable instead. - if p.LastEvaluation.After(p.GrafanaNow) { + if p.LastEvaluation.Truncate(time.Second).After(p.GrafanaNow) { fail(ReasonFutureEvaluation, fmt.Sprintf( "lastEvaluation %s is after grafana_now %s (corrupted poll)", p.LastEvaluation.Format(time.RFC3339), p.GrafanaNow.Format(time.RFC3339))) diff --git a/grafana-alertcheck/internal/gate/coverage_test.go b/grafana-alertcheck/internal/gate/coverage_test.go index a35d7c6fb..3507ee4fc 100644 --- a/grafana-alertcheck/internal/gate/coverage_test.go +++ b/grafana-alertcheck/internal/gate/coverage_test.go @@ -126,6 +126,63 @@ func TestProveCoverage_FromBeforeRecordIsUnobservable(t *testing.T) { require.Equal(t, OutcomeUnobservable, dres.Verdicts[0].Outcome) } +// The from-bounds check compares at whole-second granularity: a whole-second +// `from` may precede the recorder's sub-second StartedAt INSIDE the same second +// without being judged early. That one sliver is the --from truncation, not a +// blind interval, so the window is still proved. +func TestProveCoverage_FromSameSecondAsStartedAtIsProved(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + started := from.Add(500 * time.Millisecond) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", State: "inactive", LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: started}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved) + require.False(t, res.Unobservable) + require.Empty(t, res.Reason) +} + +// Exactly one whole second later is a different second: even at the boundary, +// the whole-second comparison reads it as before, however healthy the polls. +func TestProveCoverage_FromExactlyOneSecondBeforeStartedAtIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + started := from.Add(time.Second) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) +} + +// A sub-second sliver that straddles the second boundary is still "before": +// 900ms into one second vs 100ms into the next are distinct seconds, so the +// 200ms gap is a from_before_record, not rounding noise. +func TestProveCoverage_FromSubSecondEarlierAcrossSecondBoundaryIsUnobservable(t *testing.T) { + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + from := base.Add(900 * time.Millisecond) + started := base.Add(time.Second + 100*time.Millisecond) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) +} + // --- Check 3: heartbeat continuity --- // The core heartbeat regression: data at both ends with a hole between is not diff --git a/grafana-alertcheck/internal/gate/parse_ruler.go b/grafana-alertcheck/internal/gate/parse_ruler.go index ae878d4c9..5517fcce2 100644 --- a/grafana-alertcheck/internal/gate/parse_ruler.go +++ b/grafana-alertcheck/internal/gate/parse_ruler.go @@ -84,7 +84,7 @@ func ParseDefinitions(body []byte) ([]Definition, error) { func parseDefinition(raw json.RawMessage, folder, group string) (Definition, error) { var m map[string]json.RawMessage if err := json.Unmarshal(raw, &m); err != nil { - return Definition{}, fmt.Errorf("%w", err) + return Definition{}, err } var forStr string diff --git a/grafana-alertcheck/internal/gate/parse_state_test.go b/grafana-alertcheck/internal/gate/parse_state_test.go index 9e2ed899c..f55c10185 100644 --- a/grafana-alertcheck/internal/gate/parse_state_test.go +++ b/grafana-alertcheck/internal/gate/parse_state_test.go @@ -217,10 +217,6 @@ func TestInstanceKey(t *testing.T) { require.Equal(t, "null", instanceKey(nil), "instanceKey(nil) should be \"null\"") } -<<<<<<< HEAD -// Label values may contain "\n" or "="; the JSON encoding must keep them distinct. -======= ->>>>>>> c276546b (chore: use testify's require in tests) func TestInstanceKey_NoCollision(t *testing.T) { require.NotEqual(t, instanceKey(map[string]string{"a": "1\nb=2"}), instanceKey(map[string]string{"a": "1", "b": "2"}), "instanceKey should not collide for sets {a:1\\nb=2} and {a:1,b:2}") diff --git a/grafana-alertcheck/internal/gate/schedule.go b/grafana-alertcheck/internal/gate/schedule.go index f2b9bd759..c2fe6e1c5 100644 --- a/grafana-alertcheck/internal/gate/schedule.go +++ b/grafana-alertcheck/internal/gate/schedule.go @@ -339,6 +339,16 @@ func CheckBudget(t map[string]ruleTimings, measured map[string]time.Duration, co return fmt.Errorf("%s", b.String()) } +// graceSourceOrNone is the single "none" default for the grace-source field: +// an empty source means no rule contributed a transitionGrace. Applied here so +// StartupSummary and the human table print the same thing. +func graceSourceOrNone(source string) string { + if source == "" { + return "none" + } + return source +} + // StartupSummary formats the pre-run print an operator sees before the wait: // the total planned run time and the rule (with its `for` value) that set // transitionGrace, plus a warning when the grace eats more than @@ -347,17 +357,14 @@ func CheckBudget(t map[string]ruleTimings, measured map[string]time.Duration, co func StartupSummary(from, to time.Time, global globalTimings) (summary, warning string) { window := to.Sub(from) total := window + global.transitionGrace + global.drainTimeout - source := global.graceSource - if source == "" { - source = "none" - } + source := graceSourceOrNone(global.graceSource) summary = fmt.Sprintf( - "planned run time: %s (window %s + transitionGrace %s [source: %s] + drainTimeout %s)", - total, window, global.transitionGrace, source, global.drainTimeout) + "planned run time: %s\n window %s + transitionGrace %s + drainTimeout %s\n transitionGrace source: %s", + total, window, global.transitionGrace, global.drainTimeout, source) if window > 0 && float64(global.transitionGrace) > float64(window)*graceWarnFraction { warning = fmt.Sprintf( - "transitionGrace %s is more than %.0f%% of the window %s (source: %s) — the window may be too short for this alert's `for`", + "transitionGrace %s is more than %.0f%% of the window %s — the window may be too short for this alert's `for`\n source: %s", global.transitionGrace, graceWarnFraction*100, window, source) } return summary, warning diff --git a/grafana-alertcheck/internal/gate/source.go b/grafana-alertcheck/internal/gate/source.go index bebb1839e..721151e21 100644 --- a/grafana-alertcheck/internal/gate/source.go +++ b/grafana-alertcheck/internal/gate/source.go @@ -44,9 +44,10 @@ type Observation struct { Latency time.Duration // t_send through the full body read — see requestResult.Latency } -// TransportError marks a failure worth retrying: a non-2xx response, a network -// failure, or a body that failed to parse. Not a deleted rule (an authoritative -// 2xx) and not a clock problem (a hard error — see doRequest). +// TransportError marks a failure worth retrying: a 5xx/429 response, a network +// failure, or a body that failed to parse. Not a 4xx (wrong auth, missing +// resource), not a deleted rule (an authoritative 2xx) and not a clock problem +// (a hard error — see doRequest). type TransportError struct { Err error Status int // 0 when the failure never got a status (network/transport failure) @@ -111,17 +112,7 @@ func parseGrafanaVersion(s string) (grafanaVersion, error) { var v grafanaVersion fields := [3]*int{&v.major, &v.minor, &v.patch} for i, field := range fields { - // Trim any trailing non-digit suffix (prerelease/build metadata, e.g. - // "0+security") rather than requiring an exact numeric match. - digits := parts[i] - j := 0 - for j < len(digits) && digits[j] >= '0' && digits[j] <= '9' { - j++ - } - if j == 0 { - return grafanaVersion{}, fmt.Errorf("unparseable version %q", s) - } - n, err := strconv.Atoi(digits[:j]) + n, err := strconv.Atoi(parts[i]) if err != nil { return grafanaVersion{}, fmt.Errorf("unparseable version %q: %w", s, err) } @@ -259,10 +250,11 @@ type requestResult struct { Latency time.Duration } -// doRequest performs one HTTP GET and classifies the outcome: network failure, -// non-2xx, or body-read failure is retryable (*TransportError); a missing or -// unparseable Date header or a skew beyond SkewHardLimit is a hard error — -// retrying can never fix either, so neither enters the backoff loop. +// doRequest performs one HTTP GET and classifies the outcome: a 5xx/429, a +// network failure, or a body-read failure is retryable (*TransportError); a +// 4xx (wrong auth, missing resource — retrying cannot fix it), a missing or +// unparseable Date header, or a skew beyond SkewHardLimit is a hard error, so +// none of those enters the backoff loop. // // The Date/skew check runs on every endpoint (even /api/health): a skew only // noticed once RuleState starts polling has already masked earlier reads, so it @@ -298,7 +290,11 @@ func (s *httpSource) doRequest(ctx context.Context, path string) (requestResult, latency := tBodyRead.Sub(tSend) if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return requestResult{}, &TransportError{Err: fmt.Errorf("unexpected status %d", resp.StatusCode), Status: resp.StatusCode} + err := fmt.Errorf("unexpected status %d", resp.StatusCode) + if resp.StatusCode >= 400 && resp.StatusCode < 500 && resp.StatusCode != http.StatusTooManyRequests { + return requestResult{}, err + } + return requestResult{}, &TransportError{Err: err, Status: resp.StatusCode} } dateHeader := resp.Header.Get("Date")