chore: add centralized docs

grafana-alertcheck/README.md

@@ -1,6 +1,46 @@
 # grafana-alertcheck
 
-A CD quality gate for Grafana alerts: bookend a release with `watch` (record) and `check` (classify) to
-answer whether any watched alert was in a bad state during the release window.
+A CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and
+`check` (classify) — and answers whether any watched alert was in a bad state during the release window.
 
-Under construction.
+```
+watch  → your work → check
+```
+
+`watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a
+`from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state
+timeline, and exits `0`, `1`, or `2`.
+
+It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window.
+
+## Quickstart
+
+```bash
+export GRAFANA_URL=https://grafana.example.com
+export GRAFANA_TOKEN=…
+
+grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt
+./deploy.sh   # emits deployed_at=<RFC3339>
+./verify.sh   # emits finished_at=<RFC3339>
+grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at"
+```
+
+Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is
+never a flag.
+
+## Documentation
+
+| Doc | Covers |
+| --- | ------ |
+| [`docs/index.md`](./docs/index.md) | Overview, quickstarts, exit codes, common surprises |
+| [`docs/how-alerts-are-evaluated.md`](./docs/how-alerts-are-evaluated.md) | Verdict model, coverage proof, health/liveness |
+| [`docs/advanced.md`](./docs/advanced.md) | Check budget, scheduling, why history isn't queried |
+| [`docs/architecture.md`](./docs/architecture.md) | Design invariants, the pure-function seam, recorder lifecycle |
+| [`docs/reference/cli.md`](./docs/reference/cli.md) | Full CLI reference — subcommands, flags, naming |
+| [`docs/reference/log-format.md`](./docs/reference/log-format.md) | The JSONL log schema, for debugging artifacts |
+
+## Build
+
+```bash
+go build ./... && go test ./...
+```

grafana-alertcheck/cmd/grafana-alertcheck/check.go → grafana-alertcheck/cmd/check.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -40,6 +41,13 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`)
 
 	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 0 {
+		fmt.Fprintf(stderr, "check: unexpected arguments %v\n", fs.Args())
 		return 2
 	}
 	if *output != "" && *output != "json" {
@@ -113,11 +121,10 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 
 	result, checkErr := gate.Check(ctx, cfg)
 
-	if err := renderTable(stderr, result); err != nil {
-		fmt.Fprintln(stderr, err)
-	}
 	if checkErr != nil {
 		fmt.Fprintln(stderr, checkErr)
+	} else if err := renderTable(stderr, result); err != nil {
+		fmt.Fprintln(stderr, err)
 	}
 	if *output == "json" {
 		enc := json.NewEncoder(stdout)

grafana-alertcheck/cmd/grafana-alertcheck/main.go → grafana-alertcheck/cmd/main.go

@@ -12,7 +12,7 @@ func main() {
 	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
 }
 
-const usage = "usage: grafana-alertcheck <list|watch|check|version>"
+const usage = "usage: grafana-alertcheck <list|watch|check>"
 
 // run is the whole of main's testable surface: parse the subcommand, dispatch,
 // return the process exit code. Exit codes below 2 (pass/violations) belong to
@@ -37,8 +37,6 @@ func run(args []string, stdout, stderr io.Writer) int {
 		return runWatch(args[1:], os.Stdin, stdout, stderr)
 	case "check":
 		return runCheck(args[1:], os.Stdin, stdout, stderr)
-	case "version":
-		return runVersion(args[1:], stdout, stderr)
 	case "-h", "-help", "--help":
 		fmt.Fprintln(stdout, usage)
 		return 0

grafana-alertcheck/cmd/grafana-alertcheck/watch.go → grafana-alertcheck/cmd/watch.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -49,6 +50,13 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	readyFD := fs.Int(gate.ReadyFDFlag[2:], 0, "")
 
 	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 0 {
+		fmt.Fprintf(stderr, "watch: unexpected arguments %v\n", fs.Args())
 		return 2
 	}
 

grafana-alertcheck/docs/_category_.yaml

@@ -0,0 +1,8 @@
+position: 1
+label: 'Grafana Alertcheck'
+collapsible: true
+collapsed: false
+link:
+  type: generated-index
+  slug: /platform-services/devex/cicd/grafana-alertcheck/index
+  description: 'CD quality gate for Grafana alerts: watch, classify, and gate releases.'

grafana-alertcheck/docs/advanced.md

@@ -0,0 +1,41 @@
+---
+id: grafana-alertcheck-advanced
+title: Check budget and scheduling
+sidebar_label: Budget and scheduling
+sidebar_position: 2
+description: Why grafana-alertcheck schedules per rule, how the request budget works, and why it never queries state history.
+---
+
+# Check budget and scheduling
+
+## Per-rule schedules, never a global cycle
+
+Each rule polls at its **own** cadence, `--poll-interval` (default: half the rule's own evaluation interval). There is deliberately no single global minimum-interval cycle.
+
+One rule at `intervalSeconds=10` beside twenty at `300` keeps a 5 s cadence for itself and 150 s for the other twenty — not a 5 s cycle for all of them, which would be a 60× request bloat at ~1.8 s per request and would fail to start on a reasonable fleet.
+
+The scheduler staggers each rule's initial next-due time across its cadence, and serves due rules **earliest-due-first**, so a tight rule never queues behind slack ones.
+
+## The check budget
+
+The gate records one observation of every rule up front and checks the schedule against those **measured** latencies (payload sizes vary ~230× across rules, so a fixed estimate is meaningless). It errors at start — before waiting — if any of three conditions hold:
+
+- **Utilization** — total request rate exceeds `--concurrency`.
+- **Per-rule** — one rule's request can't fit its own cadence.
+- **Burst bound** — the slowest request exceeds the fleet's tightest cadence, which can open a mid-run gap.
+
+The error names the three levers only: raise `--concurrency`, raise `--poll-interval`, or watch fewer alerts. It never prescribes a single interval.
+
+## Why the gate never queries state history
+
+Querying Grafana's alert state history after the fact fails closed *in the wrong direction* — it returns "pass" when the truth is unknown:
+
+- History stores **transitions**, not states. An alert firing through the whole window has its only record *before* the window.
+- The annotations API **does not serve Loki-backed history** at all.
+- An empty result is indistinguishable from a healthy one: no alert fired, the backend differs, retention removed data, the token lacked permission — all look identical.
+- There is **no coverage signal** — nothing proves the history is complete to time T.
+- Artifact transitions (`Paused`, `RuleDeleted`, `Updated`, `MissingSeries`) look like recoveries.
+
+Instead, `watch` records its own evidence live and the log becomes the source of truth. The trade-off: the gate can miss an episode shorter than a rule's poll interval, though `activeAt` still surfaces sub-interval onsets for instances still active at a poll.
+
+A corollary of recording fresh: there is no replay. Re-running a failed job is a new deploy with a new `from` and a new recording — never a re-classification of old evidence.

grafana-alertcheck/docs/architecture.md

@@ -0,0 +1,68 @@
+---
+id: grafana-alertcheck-architecture
+title: Architecture
+sidebar_label: Architecture
+sidebar_position: 3
+description: The design invariants, pure-function seam, and recorder lifecycle of grafana-alertcheck, for maintainers.
+---
+
+# Architecture
+
+This page documents the invariants and seams a maintainer must not break. It exists because most of them are the difference between a gate that fails closed and one that silently passes broken windows.
+
+## Fail-closed invariants
+
+The gate must stop the release if it cannot get an answer. Every rule below is a specific instance of that:
+
+- **An error is never a pass.** A pass is exactly `len(Violations) == 0 && err == nil`. Every error path leaves `err` non-nil, and the CLI maps that to exit `2` unconditionally.
+- **Inability beats violation.** Any `unobservable` rule is exit `2`, even alongside a real violation found first.
+- **Absent never means normal.** An instance that leaves the bad set is looked up in the *same* response: present as `normal` → cleared; absent (or `MissingSeries`) → vanished (a discontinuity, not a recovery).
+- **Staleness is absolute.** `grafana_now − lastEvaluation` is compared against a threshold, never "did it increase since the last poll" — a delta check reports stale on ~half the polls of a healthy rule.
+- **`grafana_now` is the response `Date` header.** Never the runner clock, in any comparison against a Grafana timestamp.
+- **No early exit.** `check` collects to `to + transitionGrace` before classifying once.
+- **No replay.** No run-id key, no artifact download, no state between attempts. A retry is a new deploy.
+
+## The pure-function seam
+
+All correctness lives in two phases written as **pure functions** over a flat list of polls — no HTTP, no files, no clock, no goroutines:
+
+```
+HTTP ──> Source ──> []StateRule ──> reduce ──> []Poll ──> proveCoverage ──> decide ──> Result
+                            │
+                       JSONL log ──> ReadLog ──┘
+```
+
+- `proveCoverage` (the nine coverage checks) and `decide` (the instance timelines and outcomes) are pure; tests drive them with `[]Poll` literals and a fake `Clock`, with no sleeping or fixture server.
+- `Check`/`Watch` are I/O shells: HTTP, signals, the pidfile, file reads, the countdown print. The only test doubles needed are the `Source` and `Clock` interfaces.
+- `Policy` is the narrowed view of `Config` that reaches the pure layer — classification knobs and the window, no URL and no token. The token must never cross that line, which is the cheapest guarantee it never lands in an error string or a result.
+
+## Strict parsing as the version guard
+
+Both API responses are parsed strictly: a missing or unparseable **required** field (`health`, `state`, `lastEvaluation`, `interval`) is an error, never a zero value. Optional keys (`alerts`, `totals`, `labels`, `keepFiringFor`) are absent-tolerant, and unknown keys are ignored — so Grafana can add fields without breaking the parser, but removing one fails loudly.
+
+This, plus the declared supported range (Grafana >= 13.0.0, < 14.0.0), is how a deprecation or schema change is caught instead of silently misread.
+
+## The recorder lifecycle
+
+`watch` detaches a background recorder so observation survives the step boundary:
+
+1. Parent resolves names, writes the header, observes every non-paused rule once, checks the budget.
+2. Parent re-execs itself as the child (`--daemon-child`) under a new session/process group, stdout/stderr to the daemon log.
+3. Child re-reads the header, reopens the log `O_APPEND`, takes the exclusive `flock`, and writes one readiness byte on `--ready-fd`.
+4. Parent writes the pidfile **after** the readiness report, then returns.
+
+Two authorities, only one of which is evidence:
+
+- The **pidfile** says a recording ever started (written only after ready, removed on failure). It can go stale — a pid gets reused.
+- The **flock** says a writer exists *now*. The kernel drops it on exit, so the lock is always authoritative.
+
+On a clean stop (SIGTERM/SIGINT/`--until`) the child finishes the in-flight write, appends the `stopped` sentinel, fsyncs, and exits. A hard error writes no sentinel — so a recorder that died reads exactly like a coverage gap, because it is one.
+
+`check` signals via the pidfile, waits for the **lock** to release (never the pid), and only then reads the log once. Reading while a writer can still append can only produce a shorter window than was recorded.
+
+## The log is the source of truth
+
+`watch` records raw evidence, so nothing trusts a state that could become unreachable. Two consequences a maintainer must preserve:
+
+- The **header is authoritative for recording facts** (the cadence actually used, the URL, the alert set); the ruler API is authoritative for **rule facts** (`for`, `intervalSeconds`, kind). `check` always re-resolves definitions fresh and never reconstructs them from the header — the header duplicates `for`/`interval` only so the uploaded artifact is self-describing.
+- The **cadence authority** is the header's `poll_every_seconds`, not the definitions. Re-deriving it would compare gaps recorded at an override cadence against default-cadence thresholds — fail-open in the faster-override direction.