diff --git a/grafana-alertcheck/.changeset/v0.1.8.md b/grafana-alertcheck/.changeset/v0.1.8.md new file mode 100644 index 000000000..3bc7c1278 --- /dev/null +++ b/grafana-alertcheck/.changeset/v0.1.8.md @@ -0,0 +1,5 @@ +- Add `--exclude-alerts `: an enumerated list subtracted from the selected alert set, combinable with both `--alerts` and label selection. A name that does not resolve is an error, and excluding the whole selection exits `2`. +- Rename `--no-fail-fast` to `--fail-fast`, on by default. Pass `--fail-fast=false` to wait for the full window and its coverage proof. +- When alerts are selected by labels, print each matched rule on its own line before the planned run time, so it is clear what was actually selected. +- Round an observation window with a subsecond part up to the next whole second (by extending `to`), so a plan never reads `window 9m59.99445781s`. +- Drop the warning that `transitionGrace` exceeds a quarter of the window; it was noise, not an actionable condition. diff --git a/grafana-alertcheck/README.md b/grafana-alertcheck/README.md index 19d68ee26..ebecbe23f 100644 --- a/grafana-alertcheck/README.md +++ b/grafana-alertcheck/README.md @@ -27,7 +27,8 @@ grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finish ``` Or select alerts by label instead of a file: `--include-labels team=bcm,env=stage` (optionally -`--exclude-labels`). See the [CLI reference](./docs/reference/cli.md#selecting-alerts-by-labels). +`--exclude-labels`). Either selection can be refined with `--exclude-alerts `. See the +[CLI reference](./docs/reference/cli.md#selecting-alerts-by-labels). Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is never a flag. diff --git a/grafana-alertcheck/cmd/check.go b/grafana-alertcheck/cmd/check.go index 7afcd7214..57f64996b 100644 --- a/grafana-alertcheck/cmd/check.go +++ b/grafana-alertcheck/cmd/check.go @@ -15,9 +15,9 @@ import ( ) const checkUsage = "usage: grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 " + - "[--alerts ... [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]] " + + "[--alerts ... [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]] [--exclude-alerts ...] " + "[--states ...] [--preexisting ...] [--min-observed N] [--allow-paused] " + - "[--nodata-is-unobservable] [--no-fail-fast] [--concurrency N] [--output json]" + "[--nodata-is-unobservable] [--fail-fast=false] [--concurrency N] [--output json]" // runCheck is the classify step's CLI surface: parse flags into a gate.Config, // run gate.Check, and translate its (Result, error) into output and an exit @@ -39,7 +39,7 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { minObserved := fs.Int("min-observed", 0, "minimum rules that must be observed (default: every resolved rule)") allowPaused := fs.Bool("allow-paused", false, "do not count a rule paused before the window against --min-observed") nodataIsUnobservable := fs.Bool("nodata-is-unobservable", false, "treat a sustained health=nodata as unobservable rather than a note") - noFailFast := fs.Bool("no-fail-fast", false, "collect to to+transitionGrace even after a certain failure, for a full-window coverage proof instead of the fastest feedback") + failFast := fs.Bool("fail-fast", true, "stop as soon as a failure that cannot become a pass is observed; --fail-fast=false waits for the full window and its coverage proof") output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`) if err := fs.Parse(args); err != nil { @@ -60,6 +60,15 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "check: --alerts cannot be combined with label selection") return 2 } + if *common.alerts == "-" && *common.excludeAlerts == "-" { + fmt.Fprintln(stderr, "check: --alerts and --exclude-alerts cannot both read from stdin") + return 2 + } + // Refused with a log before reading: `--exclude-alerts -` would block on stdin. + if *in != "" && *common.excludeAlerts != "" { + fmt.Fprintf(stderr, "check: --exclude-alerts is refused with a recorded log: %s already names the alert set it recorded\n", *in) + return 2 + } includeLabels, err := parseLabelPairs("--include-labels", *common.includeLabels) if err != nil { fmt.Fprintln(stderr, err) @@ -76,7 +85,12 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, err) return 2 } - alerts, err := readAlerts(stdin, *common.alerts) + alerts, err := readAlerts(stdin, "--alerts", *common.alerts) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + excludeAlerts, err := readAlerts(stdin, "--exclude-alerts", *common.excludeAlerts) if err != nil { fmt.Fprintln(stderr, err) return 2 @@ -99,12 +113,13 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { Folder: *common.folder, IncludeLabels: includeLabels, ExcludeLabels: excludeLabels, + ExcludeAlerts: excludeAlerts, States: stateList, Preexisting: preexistingPolicy, MinObserved: *minObserved, AllowPaused: *allowPaused, NodataIsUnobservable: *nodataIsUnobservable, - NoFailFast: *noFailFast, + NoFailFast: !*failFast, Log: *in, PidFile: *pidfile, Concurrency: *common.concurrency, diff --git a/grafana-alertcheck/cmd/check_test.go b/grafana-alertcheck/cmd/check_test.go index 9172f98f4..0748da265 100644 --- a/grafana-alertcheck/cmd/check_test.go +++ b/grafana-alertcheck/cmd/check_test.go @@ -81,12 +81,21 @@ func TestRunCheck_FlagValidation(t *testing.T) { {"alerts with in", true, func(t *testing.T) []string { return []string{"--to", "2026-01-01T00:00:00Z", "--in", "some.jsonl", "--alerts", writeTempAlerts(t)} }, "refused"}, + {"excluded alerts with in", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--in", "some.jsonl", "--exclude-alerts", t.TempDir() + "/missing.txt"} + }, "refused with a recorded log"}, {"no alerts no in", true, func(t *testing.T) []string { return []string{"--to", "2026-01-01T00:00:00Z"} }, "no alert names"}, {"alerts and labels", true, func(t *testing.T) []string { return []string{"--to", "2026-01-01T00:00:00Z", "--alerts", writeTempAlerts(t), "--include-labels", "team=bcm"} }, "cannot be combined with label selection"}, + {"alerts and excluded alerts both stdin", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--alerts", "-", "--exclude-alerts", "-"} + }, "cannot both read from stdin"}, + {"removed no-fail-fast flag", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--alerts", writeTempAlerts(t), "--no-fail-fast"} + }, "flag provided but not defined"}, {"bad label pair", true, func(t *testing.T) []string { return []string{"--to", "2026-01-01T00:00:00Z", "--include-labels", "team"} }, "--include-labels"}, diff --git a/grafana-alertcheck/cmd/common.go b/grafana-alertcheck/cmd/common.go index a9d8feae2..443b986be 100644 --- a/grafana-alertcheck/cmd/common.go +++ b/grafana-alertcheck/cmd/common.go @@ -21,6 +21,7 @@ type commonFlags struct { folder *string concurrency *int alerts *string + excludeAlerts *string includeLabels *string excludeLabels *string } @@ -30,6 +31,8 @@ func registerCommon(fs *flag.FlagSet) *commonFlags { folder: fs.String("folder", "", "default folder to scope an unqualified alert name to"), concurrency: fs.Int("concurrency", 1, "maximum concurrent requests to Grafana"), alerts: fs.String("alerts", "", "path to a file of alert names, one per line, or - for stdin"), + excludeAlerts: fs.String("exclude-alerts", "", + "path to a file of alert names to subtract from the selected set, one per line, or - for stdin"), includeLabels: fs.String("include-labels", "", "comma-separated key=value pairs selecting rules by label, e.g. team=bcm,env=stage (cannot be combined with --alerts)"), excludeLabels: fs.String("exclude-labels", "", @@ -66,11 +69,10 @@ func parseLabelPairs(flagName, s string) ([]gate.LabelMatcher, error) { return out, nil } -// readAlerts reads alert names, one per line, from a file or from -// stdin when path is "-". An empty path is not an error here — watch and -// check each decide for themselves whether an empty list is allowed -// (log mode never wants one; single-step / record mode always does). -func readAlerts(stdin io.Reader, path string) ([]string, error) { +// readAlerts reads alert names, one per line, from a file or stdin ("-"). +// flagName names the caller's flag so errors point at the right input. An +// empty path is not an error: callers decide whether an empty list is allowed. +func readAlerts(stdin io.Reader, flagName, path string) ([]string, error) { if path == "" { return nil, nil } @@ -80,7 +82,7 @@ func readAlerts(stdin io.Reader, path string) ([]string, error) { } else { f, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("read --alerts %s: %w", path, err) + return nil, fmt.Errorf("read %s %s: %w", flagName, path, err) } defer f.Close() r = f @@ -91,7 +93,7 @@ func readAlerts(stdin io.Reader, path string) ([]string, error) { lines = append(lines, sc.Text()) } if err := sc.Err(); err != nil { - return nil, fmt.Errorf("read --alerts %s: %w", path, err) + return nil, fmt.Errorf("read %s %s: %w", flagName, path, err) } return lines, nil } diff --git a/grafana-alertcheck/cmd/common_test.go b/grafana-alertcheck/cmd/common_test.go index 228d59125..7716994ad 100644 --- a/grafana-alertcheck/cmd/common_test.go +++ b/grafana-alertcheck/cmd/common_test.go @@ -8,6 +8,17 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" ) +func TestReadAlerts_ErrorNamesTheFlag(t *testing.T) { + missing := t.TempDir() + "/missing.txt" + for _, flagName := range []string{"--alerts", "--exclude-alerts"} { + t.Run(flagName, func(t *testing.T) { + _, err := readAlerts(nil, flagName, missing) + require.Error(t, err) + require.Contains(t, err.Error(), flagName) + }) + } +} + func TestParseLabelPairs(t *testing.T) { tests := []struct { name string diff --git a/grafana-alertcheck/cmd/watch.go b/grafana-alertcheck/cmd/watch.go index f97362738..3a12c5ce4 100644 --- a/grafana-alertcheck/cmd/watch.go +++ b/grafana-alertcheck/cmd/watch.go @@ -12,7 +12,7 @@ import ( ) const watchUsage = "usage: grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] " + - "(--alerts [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]) " + + "(--alerts [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]) [--exclude-alerts ] " + "[--poll-interval D] [--concurrency N] [--until RFC3339]" // runWatch is the record step's entire CLI surface, split in two by one flag @@ -69,6 +69,10 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "watch: --alerts cannot be combined with label selection") return 2 } + if *common.alerts == "-" && *common.excludeAlerts == "-" { + fmt.Fprintln(stderr, "watch: --alerts and --exclude-alerts cannot both read from stdin") + return 2 + } includeLabels, err := parseLabelPairs("--include-labels", *common.includeLabels) if err != nil { fmt.Fprintln(stderr, err) @@ -86,7 +90,12 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { return 2 } - alerts, err := readAlerts(stdin, *common.alerts) + alerts, err := readAlerts(stdin, "--alerts", *common.alerts) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + excludeAlerts, err := readAlerts(stdin, "--exclude-alerts", *common.excludeAlerts) if err != nil { fmt.Fprintln(stderr, err) return 2 @@ -99,6 +108,7 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { Folder: *common.folder, IncludeLabels: includeLabels, ExcludeLabels: excludeLabels, + ExcludeAlerts: excludeAlerts, Out: *out, PidFile: *pidfile, DaemonLog: *daemonLog, diff --git a/grafana-alertcheck/cmd/watch_test.go b/grafana-alertcheck/cmd/watch_test.go index 861f0c4af..e434e88c7 100644 --- a/grafana-alertcheck/cmd/watch_test.go +++ b/grafana-alertcheck/cmd/watch_test.go @@ -38,6 +38,9 @@ func TestRunWatch_FlagValidation(t *testing.T) { {"alerts and labels", true, func(t *testing.T) []string { return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--include-labels", "team=bcm"} }, "cannot be combined with label selection"}, + {"alerts and excluded alerts both stdin", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", "-", "--exclude-alerts", "-"} + }, "cannot both read from stdin"}, {"bad label pair", true, func(t *testing.T) []string { return []string{"--out", t.TempDir() + "/log.jsonl", "--include-labels", "team"} }, "--include-labels"}, diff --git a/grafana-alertcheck/docs/architecture.md b/grafana-alertcheck/docs/architecture.md index 1636f7c54..e8d62f442 100644 --- a/grafana-alertcheck/docs/architecture.md +++ b/grafana-alertcheck/docs/architecture.md @@ -19,7 +19,7 @@ The gate must fail if it cannot get an answer. Every rule below is a specific in - **Absent never means normal.** An instance that leaves the bad set is looked up in the *same* response: present as `normal` → cleared; absent (or `MissingSeries`) → vanished (a discontinuity, not a recovery). - **Staleness is absolute.** `grafana_now − lastEvaluation` is compared against a threshold, never "did it increase since the last poll" — a delta check reports stale on ~half the polls of a healthy rule (we poll at half of `intervalSeconds` of each rule). - **`grafana_now` is the response `Date` header.** Never the runner clock, in any comparison against a Grafana timestamp. -- **An early exit can never be a pass.** `check` may stop collecting before `to + transitionGrace` (fail-fast), but only on a *monotone* terminal verdict: an inability that has already happened, or a post-`from` bad onset (which the full classifier would call `new_failure`/`unstable`). The one outcome that forgives an observed bad state, `recovered`, is reserved for bad-at-`from`, so a preexisting condition is never terminal. `--no-fail-fast` removes the guard entirely. +- **An early exit can never be a pass.** `check` may stop collecting before `to + transitionGrace` (fail-fast), but only on a *monotone* terminal verdict: an inability that has already happened, or a post-`from` bad onset (which the full classifier would call `new_failure`/`unstable`). The one outcome that forgives an observed bad state, `recovered`, is reserved for bad-at-`from`, so a preexisting condition is never terminal. `--fail-fast=false` removes the guard entirely. - **No replay.** No run-id key, no artifact download, no state between attempts. A retry is a new piece of work and observation. ## The pure-function seam @@ -43,7 +43,7 @@ By default `check` stops as soon as it knows the run cannot pass, rather than ho - In single-step mode `check` evaluates the guard after each in-process poll batch. - In recorder mode the evidence lives in another process, so `check` **tails the recorder's log** while it waits, consuming complete newline-terminated records only. This is the one place the project reads a log a writer can still append to, and only because fail-fast wants an early answer, not the authoritative one — the strict, whole-file `ReadLog` still runs after the writer exits, and only its result is classified. -On a terminal verdict the run is classified over `[from, At]` by the **same `decide`**, with the policy window clamped to `At` and the transition grace zeroed; `decide` is not forked. The requested window and the real thresholds are restored on the `Result`, which carries a `TerminatedEarly` marker. The drain wait is skipped — its question no longer applies. `--no-fail-fast` leaves the guard unset and restores the full-window behavior exactly. +On a terminal verdict the run is classified over `[from, At]` by the **same `decide`**, with the policy window clamped to `At` and the transition grace zeroed; `decide` is not forked. The requested window and the real thresholds are restored on the `Result`, which carries a `TerminatedEarly` marker. The drain wait is skipped — its question no longer applies. `--fail-fast=false` leaves the guard unset and restores the full-window behavior exactly. ## Strict parsing as the version guard diff --git a/grafana-alertcheck/docs/how-alerts-are-evaluated.md b/grafana-alertcheck/docs/how-alerts-are-evaluated.md index 58ac89c14..60c2c28f9 100644 --- a/grafana-alertcheck/docs/how-alerts-are-evaluated.md +++ b/grafana-alertcheck/docs/how-alerts-are-evaluated.md @@ -89,10 +89,10 @@ Before classifying, `check` must **prove** continuous coverage of `[from, to]` f A preexisting bad instance is deliberately **not** terminal: if it clears before `to` the full run would call it `recovered`, which passes. -Fail-fast is on by default and always preserves the failure: an early run can exit `1` or `2`, never `0`. The one difference from a full run is that an early exit may report `1` before an inability surfaces that would have made it `2`. `--no-fail-fast` disables the guard and always waits for the full window and its coverage proof. +Fail-fast is on by default and always preserves the failure: an early run can exit `1` or `2`, never `0`. The one difference from a full run is that an early exit may report `1` before an inability surfaces that would have made it `2`. `--fail-fast=false` disables the guard and always waits for the full window and its coverage proof. ## The drain wait and `transitionGrace` A condition that arises just before `to` becomes `firing` only at the first evaluation after its `for` elapses. `transitionGrace` (derived from the watched rules' `for` values) extends the classification bound past `to` so such a surfacing condition is caught. After collection, a **drain wait** polls until each rule has evaluated through `to + transitionGrace` (bounded by `drainTimeout`); a rule that never does is `not_verified`. -Run time = `(to − from) + transitionGrace + drainTimeout`. This is printed at start, and the grace is warned about when it exceeds a quarter of the window — the window may be too short for the alert's `for`. +Run time = `(to − from) + transitionGrace + drainTimeout`. This is printed at start. A requested window with a subsecond part is rounded up to the next whole second by extending `to`, so the plan never reads a window like `9m59.99445781s`. diff --git a/grafana-alertcheck/docs/index.md b/grafana-alertcheck/docs/index.md index b5acc57e6..ef97473e7 100644 --- a/grafana-alertcheck/docs/index.md +++ b/grafana-alertcheck/docs/index.md @@ -46,7 +46,7 @@ grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" ``` -`alerts.txt` holds one alert name per line. See [Naming alerts](./reference/cli#naming-alerts). Alerts can also be selected by label instead of by name: `--include-labels team=bcm,env=stage` (optionally `--exclude-labels`). +`alerts.txt` holds one alert name per line. See [Naming alerts](./reference/cli#naming-alerts). Alerts can also be selected by label instead of by name: `--include-labels team=bcm,env=stage` (optionally `--exclude-labels`). Either selection can be refined with `--exclude-alerts `, an enumerated list subtracted from what was selected. `watch` returns only after the recorder has observed every selected, non-paused alert once and reported ready — so auth, alert-selection, and parse failures surface **before** your deploy runs. @@ -80,7 +80,7 @@ An error is never a pass: `2` wins over any violation found alongside it. - `recovered` has **no deadline** — a bad-at-`from` alert that clears by `to` passes; set `--preexisting fail` to forbid it. - If you retry the check, then the work also needs to be retried - there is no way to check the past. - `watch` and `check` must run in **one job, one runner, one filesystem** — nothing persists across jobs or attempts. -- The gate **stops early on a certain failure** — as soon as a post-`from` bad onset or an inability is observed, `check` returns instead of holding the runner to `to + transitionGrace + drainTimeout`. This can never turn into a pass, but it can report exit `1` where a full run would have reported exit `2` (inability beats violation only when the inability is observed). Pass `--no-fail-fast` to always wait for the full window and its coverage proof; size the job timeout to the planned run time the gate prints at start either way. +- The gate **stops early on a certain failure** — as soon as a post-`from` bad onset or an inability is observed, `check` returns instead of holding the runner to `to + transitionGrace + drainTimeout`. This can never turn into a pass, but it can report exit `1` where a full run would have reported exit `2` (inability beats violation only when the inability is observed). Pass `--fail-fast=false` to always wait for the full window and its coverage proof; size the job timeout to the planned run time the gate prints at start either way. ## More diff --git a/grafana-alertcheck/docs/reference/cli.md b/grafana-alertcheck/docs/reference/cli.md index 07c15602d..91ee6910f 100644 --- a/grafana-alertcheck/docs/reference/cli.md +++ b/grafana-alertcheck/docs/reference/cli.md @@ -27,7 +27,7 @@ grafana-alertcheck list ```bash grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] \ (--alerts [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]) \ - [--poll-interval D] [--concurrency N] [--until RFC3339] + [--exclude-alerts ] [--poll-interval D] [--concurrency N] [--until RFC3339] ``` | Flag | Default | Meaning | @@ -39,6 +39,7 @@ grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] \ | `--folder` | — | Default folder to scope unqualified names (with `--alerts` only) | | `--include-labels` | — | Comma-separated exact-match `key=value` pairs selecting rules by label (cannot be combined with `--alerts`) | | `--exclude-labels` | — | Comma-separated exact-match `key=value` pairs; a rule carrying any of them is dropped (requires `--include-labels`) | +| `--exclude-alerts` | — | File of alert names, one per line, or `-` for stdin; subtracted from the selected set (works with `--alerts` and with labels) | | `--poll-interval` | half the rule's interval | Override every rule's cadence (never clamped) | | `--concurrency` | `1` | Max concurrent requests to Grafana | | `--until` | run until signalled | Optional hard stop | @@ -66,9 +67,9 @@ Use it when the work failed and the alert verdict no longer matters, but the rec ```bash grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 \ - [--alerts ... [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]] \ + [--alerts ... [--folder F] | --include-labels k=v,... [--exclude-labels k=v,...]] [--exclude-alerts ...] \ [--states ...] [--preexisting ...] [--min-observed N] \ - [--allow-paused] [--nodata-is-unobservable] [--no-fail-fast] [--concurrency N] [--output json] + [--allow-paused] [--nodata-is-unobservable] [--fail-fast=false] [--concurrency N] [--output json] ``` | Flag | Default | Meaning | @@ -81,18 +82,19 @@ grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 | `--folder` | — | Default folder to scope unqualified names (with `--alerts` only) | | `--include-labels` | — | Comma-separated exact-match `key=value` pairs selecting rules by label (cannot be combined with `--alerts`) | | `--exclude-labels` | — | Comma-separated exact-match `key=value` pairs; a rule carrying any of them is dropped (requires `--include-labels`) | +| `--exclude-alerts` | — | File of alert names, one per line, or `-` for stdin; subtracted from the selected set (works with `--alerts` and with labels; refused **with** `--in`) | | `--states` | `firing` | Comma-separated bad states: `firing,pending,nodata,error` | | `--preexisting` | `fail-unless-recovered` | `fail-unless-recovered` \| `fail` \| `ignore` | | `--min-observed` | every resolved rule | Minimum rules that must be observed | | `--allow-paused` | `false` | Don't count pre-window-paused rules against `--min-observed` | | `--nodata-is-unobservable` | `false` | Treat sustained `health=nodata` as unobservable | -| `--no-fail-fast` | `false` | Wait for the full window even after a certain failure | +| `--fail-fast` | `true` | Stop as soon as a failure that cannot become a pass is observed; `--fail-fast=false` waits for the full window and its coverage proof | | `--concurrency` | `1` | Max concurrent requests | | `--output` | `table` | `json` also writes the machine-readable result to stdout | -By default `check` **exits early** on a failure that cannot become a pass: a post-`from` bad onset, or an inability (a heartbeat gap, a sustained `health=error`, a stale evaluation, an in-window pause, an absent rule). This is a latency optimization, not a weaker gate — it never exits `0` early. The one observable difference is that an early exit can report `1` where a full run would have discovered an inability later and reported `2`. `--no-fail-fast` always waits for `to + transitionGrace` and the full coverage proof; the `Result` then carries no `terminated_early` marker. With early exit the JSON result includes `terminated_early` naming the rule, kind, reason and time. +By default `check` **exits early** on a failure that cannot become a pass: a post-`from` bad onset, or an inability (a heartbeat gap, a sustained `health=error`, a stale evaluation, an in-window pause, an absent rule). This is a latency optimization, not a weaker gate — it never exits `0` early. The one observable difference is that an early exit can report `1` where a full run would have discovered an inability later and reported `2`. `--fail-fast=false` always waits for `to + transitionGrace` and the full coverage proof; the `Result` then carries no `terminated_early` marker. With early exit the JSON result includes `terminated_early` naming the rule, kind, reason and time. -`--from` and `--to` are RFC3339 with an explicit offset and must come from your work — `from` from the deploy step, `to` from the step that finishes. In recorder mode an absent `--from` is a hard error, and a `from` before the recording's first-observation pass is refused; in single-step mode an absent `from`, or one inside `check`'s own first-observation pass, is a declared blind interval — the window is classified from the pass completion, with a warning. +`--from` and `--to` are RFC3339 with an explicit offset and must come from your work — `from` from the deploy step, `to` from the step that finishes. In recorder mode an absent `--from` is a hard error; in single-step mode it falls back (with a warning) to the start of the step. A window with a subsecond part is rounded up to the next whole second by extending `to`, so the plan never reads `window 9m59.99445781s`. ## Naming alerts @@ -118,7 +120,9 @@ grafana-alertcheck check --to "$finished_at" --include-labels team=bcm --exclude `--include-labels` takes comma-separated exact-match `key=value` pairs; a rule must carry **all** of them. `--exclude-labels` is optional and drops any rule carrying **one** of its pairs. A rule that does not carry the label is never dropped, only never included. Values cannot contain commas; `key=` matches only rules that carry the label with an empty value. -The label flags cannot be combined with `--alerts` or `--folder`, and they are refused with `--in` — the recorded log names its own alert set. A selection that matches no rules, whose matches are all excluded, or that matches a recording rule exits `2`: an empty watch set must never pass. +`--exclude-alerts` is an enumerated list (a file, or `-` for stdin) subtracted from whichever set was selected — names or labels. It is the escape hatch for a rule that carries the include labels but must not be watched. + +The label flags cannot be combined with `--alerts` or `--folder`, and they are refused with `--in` — the recorded log names its own alert set. A selection that matches no rules, whose matches are all excluded, or that matches a recording rule exits `2`: an empty watch set must never pass. When rules are selected by labels, the resolved set is printed one rule per line before the run starts, so an operator can see exactly what matched. ## Output and exit codes diff --git a/grafana-alertcheck/internal/gate/check.go b/grafana-alertcheck/internal/gate/check.go index 43e03d138..ea07e3a88 100644 --- a/grafana-alertcheck/internal/gate/check.go +++ b/grafana-alertcheck/internal/gate/check.go @@ -37,10 +37,11 @@ type Config struct { Alerts []string Folder string - // IncludeLabels selects the watched rules by exact-match labels instead of - // names; ExcludeLabels drops matching rules from that set. Both are refused - // in log mode, where the header IS the alert set. + // IncludeLabels selects by labels instead of names; ExcludeLabels drops + // matching rules. Both are refused in log mode. ExcludeAlerts is an + // enumerated list subtracted from either selection. IncludeLabels, ExcludeLabels []LabelMatcher + ExcludeAlerts []string States []State Preexisting PreexistingPolicy @@ -50,7 +51,7 @@ type Config struct { // NoFailFast disables early exit, so the loop always runs to // to+transitionGrace for a full-window coverage proof. The CLI sets it - // with --no-fail-fast; the gate never exits 0 early either way. + // with --fail-fast=false; the gate never exits 0 early either way. NoFailFast bool // From is the moment the deploy finished and To is the end of the work. @@ -151,6 +152,8 @@ func (cfg Config) validate() error { return fmt.Errorf("check: --alerts is refused with a recorded log: %s already names the alert set it recorded", cfg.Log) } else if labeled { return fmt.Errorf("check: label selection is refused with a recorded log: %s already names the alert set it recorded", cfg.Log) + } else if len(cfg.ExcludeAlerts) > 0 { + return fmt.Errorf("check: --exclude-alerts is refused with a recorded log: %s already names the alert set it recorded", cfg.Log) } now := cfg.Clock.Now() @@ -201,6 +204,7 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { fmt.Fprintf(cfg.Notes, "note: no `from` given; the window starts at the start of this step, %s\n", from.Format(time.RFC3339)) } + cfg.To = roundWindowUp(from, cfg.To) // ---- Resolve the definitions from the ruler API. ---------------------- // Unconditional, in BOTH modes. A log's header supplies the alert set as @@ -255,7 +259,7 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { } } } else { - resolved, notes, err = resolveAlertSet(allDefs, cfg.namedAlerts(), cfg.IncludeLabels, cfg.ExcludeLabels, cfg.Folder) + resolved, notes, err = resolveAlertSet(allDefs, cfg.namedAlerts(), cfg.IncludeLabels, cfg.ExcludeLabels, cfg.ExcludeAlerts, cfg.Folder) } if err != nil { return Result{}, err @@ -269,6 +273,9 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { for _, n := range notes { fmt.Fprintf(cfg.Notes, "note: %s\n", n) } + if len(cfg.IncludeLabels) > 0 { + printLabelSelection(cfg.Notes, resolved, cfg.IncludeLabels, cfg.ExcludeLabels, len(cfg.ExcludeAlerts)) + } // ---- Derive the timings, print them, fit the request budget. ---------- if logHasHdr { @@ -286,8 +293,7 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { fmt.Fprintf(cfg.Notes, "note: %s\n", n) } } - summary, warning := StartupSummary(from, cfg.To, gt) - fmt.Fprintln(cfg.Notes, summary) + fmt.Fprintln(cfg.Notes, StartupSummary(from, cfg.To, gt)) // MinObserved is printed with the plan, beside "planned run time", rather // than after it: it is a fact about the run, not a diagnostic. Its default // is the resolved rule count AFTER duplicate names collapse, which is @@ -298,9 +304,6 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { minObserved = len(resolved) } fmt.Fprintf(cfg.Notes, "min-observed: %d of %d resolved rule(s)\n", minObserved, len(resolved)) - if warning != "" { - fmt.Fprintf(cfg.Notes, "warning: %s\n", warning) - } // The measurement pass and budget check are single-step only: in recorder // mode watch already measured and checked the budget before detaching. @@ -389,7 +392,7 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { // ---- Collect the evidence. -------------------------------------------- // The loop never classifies; its only access to the pure layer is the // fail-fast guard, which ends the run only on a condition that cannot - // become a pass. With --no-fail-fast there is no guard. + // become a pass. With --fail-fast=false there is no guard. windowEnd := cfg.To.Add(gt.transitionGrace) var ( diff --git a/grafana-alertcheck/internal/gate/check_test.go b/grafana-alertcheck/internal/gate/check_test.go index 210a427a4..b53b12c3a 100644 --- a/grafana-alertcheck/internal/gate/check_test.go +++ b/grafana-alertcheck/internal/gate/check_test.go @@ -200,6 +200,20 @@ func TestCheckValidateRejectsBadConfigurations(t *testing.T) { }, wantErr: "refused with a recorded log", }, + { + name: "log mode with excluded alerts", + mutate: func(c *Config) { + c.Log = "log.jsonl" + c.ExcludeAlerts = []string{"A"} + }, + wantErr: "--exclude-alerts is refused with a recorded log", + }, + { + // Exclusions subtract from a selection; they never name one. + name: "excluded alerts without a selection", + mutate: func(c *Config) { c.ExcludeAlerts = []string{"A"} }, + wantErr: "no alert names given", + }, { name: "alerts and labels", mutate: func(c *Config) { @@ -289,6 +303,20 @@ func TestCheckValidateAcceptsLabelSelection(t *testing.T) { Clock: newFakeClock(testNow), IncludeLabels: []LabelMatcher{{Key: "team", Value: "bcm"}}, ExcludeLabels: []LabelMatcher{{Key: "env", Value: "stage"}}, + ExcludeAlerts: []string{"uid:other"}, + }.withDefaults() + + require.NoError(t, cfg.validate()) +} + +func TestCheckValidateAcceptsExcludeAlertsWithNames(t *testing.T) { + cfg := Config{ + URL: "https://grafana.example.com", + Alerts: []string{"A"}, + ExcludeAlerts: []string{"B"}, + From: testNow, + To: testNow.Add(5 * time.Minute), + Clock: newFakeClock(testNow), }.withDefaults() require.NoError(t, cfg.validate()) @@ -371,6 +399,63 @@ func TestCheckSingleStepSelectsByLabels(t *testing.T) { require.Len(t, res.Verdicts, 1) require.Equal(t, checkUID, res.Verdicts[0].RuleUID) require.Empty(t, res.Violations) + + // The matched set is listed, by name, before the plan. + notes := notesOf(cfg) + require.Contains(t, notes, "alerts matching --include-labels team=bcm and --exclude-labels env=stage:\n") + require.Contains(t, notes, " - Rule One (rule-one)\n") + require.Less(t, strings.Index(notes, "alerts matching"), strings.Index(notes, "planned run time"), + "the matched alerts must be printed before the planned run time") +} + +// --exclude-alerts subtracts from the enumerated set end to end: the excluded +// rule is never polled and the run classifies only what remains. +func TestCheckSingleStepExcludesNamedAlerts(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.Alerts = []string{"uid:" + checkUID, "uid:other"} + cfg.ExcludeAlerts = []string{"uid:other"} + + excluded := checkDef() + excluded.UID, excluded.Title = "other", "Other" + + src := newCheckSource(func(title string, _ int) (Observation, error) { + uid := checkUID + if title == excluded.Title { + uid = excluded.UID + } + now := clock.Now() + return Observation{ + Rules: []StateRule{testStateRule(uid, title, time.Minute, now, testInstance(StateNormal, "", "a"))}, + GrafanaNow: now, + Latency: 200 * time.Millisecond, + }, nil + }) + src.defs = []Definition{checkDef(), excluded} + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Len(t, res.Verdicts, 1) + require.Equal(t, checkUID, res.Verdicts[0].RuleUID) + require.Zero(t, src.callCount(excluded.Title), "the excluded rule must never be polled") +} + +// A subsecond `to` is rounded up before the plan is printed, so a window never +// reads as 9m59.99445781s. +func TestCheckSingleStepRoundsWindowUp(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.To = cfg.From.Add(10*time.Minute - 5542190*time.Nanosecond) // 9m59.99445781s + + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + _, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Contains(t, notesOf(cfg), "window 10m0s") + require.False(t, clock.Now().Before(cfg.From.Add(10*time.Minute+checkGrace)), + "collection must run to the rounded-up window end + grace, not the subsecond one") } // A label selection matching nothing is exit 2, before any poll — never an @@ -505,8 +590,8 @@ func TestCheckSingleStepPreexistingBadDoesNotExitEarly(t *testing.T) { "exited early; collection must run to to+grace for a preexisting bad instance") } -// --no-fail-fast runs the same fresh onset to to+transitionGrace. -func TestCheckSingleStepNewOnsetNoFailFastRunsToTheEnd(t *testing.T) { +// --fail-fast=false runs the same fresh onset to to+transitionGrace. +func TestCheckSingleStepFailFastDisabledRunsToTheEnd(t *testing.T) { clock := newVirtualClock(testNow) cfg := baseConfig(t, clock) cfg.NoFailFast = true @@ -530,7 +615,7 @@ func TestCheckSingleStepNewOnsetNoFailFastRunsToTheEnd(t *testing.T) { require.Equal(t, OutcomeNewFailure, res.Violations[0].Outcome) require.Nil(t, res.TerminatedEarly) require.False(t, clock.Now().Before(cfg.To.Add(checkGrace)), - "with --no-fail-fast the loop must run to to+grace") + "with --fail-fast=false the loop must run to to+grace") } // An ABSENT `from` in single-step mode (as opposed to recorder mode, which @@ -909,8 +994,8 @@ func TestCheckRecorderModeExitsEarlyOnANewOnset(t *testing.T) { require.True(t, res.To.Equal(cfg.To), "the requested window is still reported") } -// --no-fail-fast over the same recording runs to to+transitionGrace. -func TestCheckRecorderModeNoFailFastRunsToTheEnd(t *testing.T) { +// --fail-fast=false over the same recording runs to to+transitionGrace. +func TestCheckRecorderModeFailFastDisabledRunsToTheEnd(t *testing.T) { onset := testNow.Add(time.Minute) logPath := recordedLogWithOnset(t, t.TempDir(), onset) writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) @@ -924,7 +1009,7 @@ func TestCheckRecorderModeNoFailFastRunsToTheEnd(t *testing.T) { require.Len(t, res.Violations, 1) require.Equal(t, OutcomeNewFailure, res.Violations[0].Outcome) require.False(t, clock.Now().Before(cfg.To.Add(checkGrace)), - "with --no-fail-fast the loop must run to to+grace") + "with --fail-fast=false the loop must run to to+grace") } // The identity of the log is not correct. The check runs against the header diff --git a/grafana-alertcheck/internal/gate/labels.go b/grafana-alertcheck/internal/gate/labels.go index d913f46c4..ce8e60810 100644 --- a/grafana-alertcheck/internal/gate/labels.go +++ b/grafana-alertcheck/internal/gate/labels.go @@ -3,6 +3,7 @@ package gate import ( "errors" "fmt" + "io" "strings" ) @@ -43,14 +44,75 @@ func SelectByLabels(defs []Definition, include, exclude []LabelMatcher) ([]Defin return selected, nil } -// resolveAlertSet picks the alert set for a run. Validation guarantees the two -// modes are never mixed; an empty include list means enumerated names. -func resolveAlertSet(defs []Definition, names []string, include, exclude []LabelMatcher, folder string) ([]Definition, []string, error) { +// resolveAlertSet picks the alert set: names or labels, minus --exclude-alerts. +func resolveAlertSet(defs []Definition, names []string, include, exclude []LabelMatcher, + excludeAlerts []string, folder string) ([]Definition, []string, error) { + + var ( + selected []Definition + notes []string + err error + ) if len(include) > 0 { - selected, err := SelectByLabels(defs, include, exclude) - return selected, nil, err + selected, err = SelectByLabels(defs, include, exclude) + } else { + selected, notes, err = Resolve(defs, names, folder) + } + if err != nil { + return nil, nil, err + } + selected, err = subtractExcluded(defs, selected, excludeAlerts, folder) + if err != nil { + return nil, nil, err + } + return selected, notes, nil +} + +// subtractExcluded drops every --exclude-alerts name from the selected set. +// Names resolve like --alerts, so a typo is an error; a name outside the +// selection removes nothing. +func subtractExcluded(all, selected []Definition, excludeAlerts []string, folder string) ([]Definition, error) { + if len(excludeAlerts) == 0 { + return selected, nil + } + excluded, _, err := Resolve(all, excludeAlerts, folder) + if err != nil { + return nil, fmt.Errorf("--exclude-alerts: %w", err) + } + drop := make(map[string]bool, len(excluded)) + for _, d := range excluded { + drop[d.UID] = true + } + out := make([]Definition, 0, len(selected)) + for _, d := range selected { + if !drop[d.UID] { + out = append(out, d) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("--exclude-alerts drops every selected rule; refusing to run with an empty alert set") + } + return out, nil +} + +// labelSelectionHeader names the include/exclude pairs and any --exclude-alerts. +func labelSelectionHeader(include, exclude []LabelMatcher, excludeCount int) string { + header := fmt.Sprintf("alerts matching --include-labels %s", formatLabelMatchers(include)) + if len(exclude) > 0 { + header += fmt.Sprintf(" and --exclude-labels %s", formatLabelMatchers(exclude)) + } + if excludeCount > 0 { + header += " minus --exclude-alerts" + } + return header +} + +// printLabelSelection lists the matched rules, one per line. +func printLabelSelection(w io.Writer, resolved []Definition, include, exclude []LabelMatcher, excludeCount int) { + fmt.Fprintf(w, "%s:\n", labelSelectionHeader(include, exclude, excludeCount)) + for _, d := range resolved { + fmt.Fprintf(w, " - %s (%s)\n", d.Title, d.UID) } - return Resolve(defs, names, folder) } func matchesAll(labels map[string]string, matchers []LabelMatcher) bool { diff --git a/grafana-alertcheck/internal/gate/labels_test.go b/grafana-alertcheck/internal/gate/labels_test.go index b97f15503..f7dcb422a 100644 --- a/grafana-alertcheck/internal/gate/labels_test.go +++ b/grafana-alertcheck/internal/gate/labels_test.go @@ -1,6 +1,7 @@ package gate import ( + "strings" "testing" "github.com/stretchr/testify/require" @@ -101,3 +102,61 @@ func TestSelectByLabels_EmptyIncludeIsNotAnError(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"rule0000002"}, selectedUIDs(selected)) } + +// --exclude-alerts subtracts from an enumerated set: the names resolve like +// --alerts, so uid: forms work, and the result keeps the input order. +func TestResolveAlertSet_SubtractsExcludedNames(t *testing.T) { + defs := rulerDefs(t) + selected, _, err := resolveAlertSet(defs, + []string{"uid:rule0000009", "uid:rule0000010"}, nil, nil, + []string{"uid:rule0000010"}, "") + require.NoError(t, err) + require.Equal(t, []string{"rule0000009"}, selectedUIDs(selected)) +} + +// --exclude-alerts combines with label selection too: the label match is +// computed first, then the named rules are subtracted from it. +func TestResolveAlertSet_SubtractsExcludedNamesFromLabelSelection(t *testing.T) { + defs := rulerDefs(t) + selected, _, err := resolveAlertSet(defs, nil, + []LabelMatcher{{Key: "team", Value: "example-team"}}, nil, + []string{"uid:rule0000009", "uid:rule0000010"}, "") + require.NoError(t, err) + require.Equal(t, []string{"rule0000006a", "rule0000006b"}, selectedUIDs(selected)) +} + +// A typo in the excluded list is an error, never a silent no-op: the operator +// asked to remove something and the gate cannot prove it did. +func TestResolveAlertSet_UnknownExcludedNameFails(t *testing.T) { + defs := rulerDefs(t) + _, _, err := resolveAlertSet(defs, []string{"uid:rule0000009"}, nil, nil, + []string{"Does Not Exist"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "--exclude-alerts") +} + +// Excluding the whole selection would leave nothing to observe; that is exit 2, +// not a vacuous pass. +func TestResolveAlertSet_ExcludingEverythingFails(t *testing.T) { + defs := rulerDefs(t) + _, _, err := resolveAlertSet(defs, []string{"uid:rule0000009"}, nil, nil, + []string{"uid:rule0000009"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "drops every selected rule") +} + +func TestPrintLabelSelection(t *testing.T) { + defs := rulerDefs(t) + selected, err := SelectByLabels(defs, []LabelMatcher{{Key: "severity", Value: "warning"}}, nil) + require.NoError(t, err) + + var b strings.Builder + printLabelSelection(&b, selected, + []LabelMatcher{{Key: "severity", Value: "warning"}}, + []LabelMatcher{{Key: "env", Value: "stage"}}, 1) + + out := b.String() + require.Contains(t, out, "alerts matching --include-labels severity=warning and --exclude-labels env=stage minus --exclude-alerts:\n") + require.Contains(t, out, " - Example Failure Ratio Above 10 Percent (rule0000009)\n") + require.Contains(t, out, " - Example Failure Ratio Above 10 Percent Weekly (rule0000010)\n") +} diff --git a/grafana-alertcheck/internal/gate/schedule.go b/grafana-alertcheck/internal/gate/schedule.go index 07e3f4572..65bb7d419 100644 --- a/grafana-alertcheck/internal/gate/schedule.go +++ b/grafana-alertcheck/internal/gate/schedule.go @@ -23,10 +23,6 @@ const fromFutureTolerance = 60 * time.Second // fleet of tight rules still lets a healthy in-flight poll land. const minDrainTimeout = 2 * time.Minute -// graceWarnFraction is the share of the window above which transitionGrace is -// worth warning about. -const graceWarnFraction = 0.25 - // RuleTimings groups the per-rule thresholds derived from a rule's poll // cadence and its own evaluation interval. title is carried for operator-facing // messages only — it is never compared or applied. @@ -413,21 +409,24 @@ func graceSourceOrNone(source string) string { // StartupSummary formats the pre-run print an operator sees before the wait: // the total planned run time and the rule (with its `for` value) that set -// transitionGrace, plus a warning when the grace eats more than -// graceWarnFraction of the requested window. from/to are the requested -// classification window. -func StartupSummary(from, to time.Time, global GlobalTimings) (summary, warning string) { +// transitionGrace. from/to are the requested classification window. +func StartupSummary(from, to time.Time, global GlobalTimings) string { window := to.Sub(from) total := window + global.transitionGrace + global.drainTimeout source := graceSourceOrNone(global.graceSource) - summary = fmt.Sprintf( + return fmt.Sprintf( "planned run time: %s\n window %s + transitionGrace %s + drainTimeout %s\n transitionGrace source: %s", total, window, global.transitionGrace, global.drainTimeout, source) +} - if window > 0 && float64(global.transitionGrace) > float64(window)*graceWarnFraction { - warning = fmt.Sprintf( - "transitionGrace %s is more than %.0f%% of the window %s — the window may be too short for this alert's `for`\n source: %s", - global.transitionGrace, graceWarnFraction*100, window, source) +// roundWindowUp extends to so the window is a whole number of seconds. from is +// never moved: it is the deploy-completion fact. +func roundWindowUp(from, to time.Time) time.Time { + if from.IsZero() || to.IsZero() { + return to + } + if r := to.Sub(from) % time.Second; r > 0 { + to = to.Add(time.Second - r) } - return summary, warning + return to } diff --git a/grafana-alertcheck/internal/gate/schedule_test.go b/grafana-alertcheck/internal/gate/schedule_test.go index e46fc33b5..b4ae02669 100644 --- a/grafana-alertcheck/internal/gate/schedule_test.go +++ b/grafana-alertcheck/internal/gate/schedule_test.go @@ -431,44 +431,44 @@ func uidN(i int) string { return "slack" + string(rune('a'+i)) // nolint:gosec // test-only uid generator } -func TestStartupSummary_WarningWhenGraceTooLarge(t *testing.T) { +func TestStartupSummary_PlansTheWholeRunTime(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(10 * time.Minute) global := GlobalTimings{transitionGrace: 5 * time.Minute, graceSource: "R (for=4m30s, interval=30s)", drainTimeout: time.Minute} - summary, warning := StartupSummary(from, to, global) - require.Contains(t, summary, "planned run time") - require.NotEmpty(t, warning, "transitionGrace (5m) > 1/4 of the 10m window") - require.Contains(t, warning, "R (for=4m30s, interval=30s)") + summary := StartupSummary(from, to, global) + require.Contains(t, summary, "planned run time: 16m0s") + require.Contains(t, summary, "window 10m0s") + require.Contains(t, summary, "transitionGrace 5m0s") + require.Contains(t, summary, "R (for=4m30s, interval=30s)") } -// The test above pins the warning formula with a hand-built globalTimings. -// This drives the same warning off the real ruler_rules.json fixture's for:1w -// rule instead, tying ParseDefinitions and DeriveTimings into the warning end -// to end. -func TestStartupSummary_RealForOneWeekRuleTriggersWarning(t *testing.T) { - defs := rulerDefs(t) - _, global, notes := DeriveTimings(defs, 0) - require.Empty(t, notes) - - from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - to := from.Add(10 * time.Minute) // transitionGrace (>1w) dwarfs 1/4 of this window - summary, warning := StartupSummary(from, to, global) - require.Contains(t, summary, "planned run time") - require.NotEmpty(t, warning) - require.Contains(t, warning, "Example Failure Ratio Above 10 Percent Weekly") -} - -func TestStartupSummary_NoWarningWhenGraceSmall(t *testing.T) { +func TestStartupSummary_NoGraceSourceReadsNone(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(time.Hour) - global := GlobalTimings{transitionGrace: time.Minute, graceSource: "R (for=30s, interval=30s)", drainTimeout: time.Minute} - _, warning := StartupSummary(from, to, global) - require.Empty(t, warning) + summary := StartupSummary(from, to, GlobalTimings{}) + require.Contains(t, summary, "none") } -func TestStartupSummary_NoGraceSourceReadsNone(t *testing.T) { +// A window with a subsecond part is rounded UP by moving `to`, never `from`: +// from is the deploy-completion fact and moving it earlier could fall before +// the recording started. +func TestRoundWindowUp(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - to := from.Add(time.Hour) - summary, _ := StartupSummary(from, to, GlobalTimings{}) - require.Contains(t, summary, "none") + tests := []struct { + name string + window time.Duration + want time.Duration + }{ + {"whole seconds unchanged", 10 * time.Minute, 10 * time.Minute}, + {"subsecond rounds up", 9*time.Minute + 59*time.Second + 994457810*time.Nanosecond, 10 * time.Minute}, + {"just over a second rounds up", time.Second + time.Nanosecond, 2 * time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := roundWindowUp(from, from.Add(tt.window)) + require.Equal(t, tt.want, got.Sub(from)) + }) + } + + require.Equal(t, from, roundWindowUp(time.Time{}, from), "a zero `from` leaves `to` alone") } diff --git a/grafana-alertcheck/internal/gate/watch.go b/grafana-alertcheck/internal/gate/watch.go index 5161d5dee..98e29145f 100644 --- a/grafana-alertcheck/internal/gate/watch.go +++ b/grafana-alertcheck/internal/gate/watch.go @@ -52,10 +52,11 @@ type WatchConfig struct { Alerts []string Folder string - // IncludeLabels selects the watched rules by exact-match labels instead of - // names; ExcludeLabels drops matching rules from that set. The two - // selection modes cannot be combined (validate refuses it). + // IncludeLabels selects by labels instead of names; ExcludeLabels drops + // matching rules. The two modes cannot be combined. ExcludeAlerts is an + // enumerated list subtracted from either selection. IncludeLabels, ExcludeLabels []LabelMatcher + ExcludeAlerts []string // Out is the JSONL log path. PidFile and DaemonLog default to // .pid and .daemon.log — the same convention check uses to find @@ -294,13 +295,16 @@ func prepareWatch(ctx context.Context, cfg WatchConfig, src Source) (*preparedWa if err != nil { return nil, fmt.Errorf("read rule definitions: %w", err) } - resolved, notes, err := resolveAlertSet(defs, cfg.Alerts, cfg.IncludeLabels, cfg.ExcludeLabels, cfg.Folder) + resolved, notes, err := resolveAlertSet(defs, cfg.Alerts, cfg.IncludeLabels, cfg.ExcludeLabels, cfg.ExcludeAlerts, cfg.Folder) if err != nil { return nil, err } for _, n := range notes { fmt.Fprintf(cfg.Notes, "note: %s\n", n) } + if len(cfg.IncludeLabels) > 0 { + printLabelSelection(cfg.Notes, resolved, cfg.IncludeLabels, cfg.ExcludeLabels, len(cfg.ExcludeAlerts)) + } rt, _, timingNotes := DeriveTimings(resolved, cfg.PollEvery) for _, n := range timingNotes { diff --git a/grafana-alertcheck/internal/gate/watch_test.go b/grafana-alertcheck/internal/gate/watch_test.go index 8d03ef302..576745da2 100644 --- a/grafana-alertcheck/internal/gate/watch_test.go +++ b/grafana-alertcheck/internal/gate/watch_test.go @@ -589,6 +589,7 @@ func TestWatchConfigValidation(t *testing.T) { {"no log path", func(c *WatchConfig) { c.Out = "" }, "no log path"}, {"no alerts", func(c *WatchConfig) { c.Alerts = nil }, "no alert names"}, {"blank alerts only", func(c *WatchConfig) { c.Alerts = []string{"", " "} }, "no alert names"}, + {"excluded alerts without a selection", func(c *WatchConfig) { c.Alerts = nil; c.ExcludeAlerts = []string{"A"} }, "no alert names"}, {"alerts and labels", func(c *WatchConfig) { c.IncludeLabels = []LabelMatcher{{Key: "team", Value: "bcm"}} }, "cannot be combined"}, {"exclude without include", func(c *WatchConfig) { c.Alerts = nil; c.ExcludeLabels = []LabelMatcher{{Key: "team", Value: "bcm"}} }, "requires --include-labels"}, {"labels with folder", func(c *WatchConfig) { @@ -618,6 +619,7 @@ func TestWatchConfigValidation(t *testing.T) { cfg := base() cfg.Alerts = nil cfg.IncludeLabels = []LabelMatcher{{Key: "team", Value: "bcm"}} + cfg.ExcludeAlerts = []string{"uid:other"} require.NoError(t, cfg.withDefaults().validate()) }) } @@ -644,6 +646,12 @@ func TestPrepareWatchSelectsByLabels(t *testing.T) { require.Equal(t, []string{watchActiveUID, weeklyUID}, []string{header.Rules[0].UID, header.Rules[1].UID}) require.Len(t, polls, 2) + + // The matched set is listed, by name, so an operator sees what was picked. + out := notes.String() + require.Contains(t, out, "alerts matching --include-labels severity=warning:\n") + require.Contains(t, out, " - Example Failure Ratio Above 10 Percent (rule0000009)\n") + require.Contains(t, out, " - Example Failure Ratio Above 10 Percent Weekly (rule0000010)\n") } func TestPrepareWatchLabelSelectionWithNoMatchFails(t *testing.T) {