Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 0 additions & 34 deletions .github/workflows/grafana-alertcheck-release.yml

This file was deleted.

33 changes: 0 additions & 33 deletions grafana-alertcheck/.goreleaser.yaml

This file was deleted.

46 changes: 43 additions & 3 deletions grafana-alertcheck/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,46 @@
# grafana-alertcheck

A CD quality gate for Grafana alerts: bookend a release with `watch` (record) and `check` (classify) to
answer whether any watched alert was in a bad state during the release window.
A CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and
`check` (classify) — and answers whether any watched alert was in a bad state during the release window.

Under construction.
```
watch → your work → check
```

`watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a
`from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state
timeline, and exits `0`, `1`, or `2`.

It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window.

## Quickstart

```bash
export GRAFANA_URL=https://grafana.example.com
export GRAFANA_TOKEN=…

grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt
./deploy.sh # emits deployed_at=<RFC3339>
./verify.sh # emits finished_at=<RFC3339>
grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at"
```

Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is
never a flag.

## Documentation

| Doc | Covers |
| --- | ------ |
| [`docs/index.md`](./docs/index.md) | Overview, quickstarts, exit codes, common surprises |
| [`docs/how-alerts-are-evaluated.md`](./docs/how-alerts-are-evaluated.md) | Verdict model, coverage proof, health/liveness |
| [`docs/advanced.md`](./docs/advanced.md) | Check budget, scheduling, why history isn't queried |
| [`docs/architecture.md`](./docs/architecture.md) | Design invariants, the pure-function seam, recorder lifecycle |
| [`docs/reference/cli.md`](./docs/reference/cli.md) | Full CLI reference — subcommands, flags, naming |
| [`docs/reference/log-format.md`](./docs/reference/log-format.md) | The JSONL log schema, for debugging artifacts |

## Build

```bash
go build ./... && go test ./...
```
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
Expand Down Expand Up @@ -40,6 +41,13 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`)

if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if fs.NArg() != 0 {
fmt.Fprintf(stderr, "check: unexpected arguments %v\n", fs.Args())
return 2
}
if *output != "" && *output != "json" {
Expand Down Expand Up @@ -82,7 +90,7 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
PidFile: *pidfile,
Concurrency: *common.concurrency,
Clock: gate.SystemClock{},
Notes: stderr,
Notes: newNoteStyler(stderr),
}
if *to == "" {
fmt.Fprintln(stderr, "check: --to is required")
Expand Down Expand Up @@ -113,11 +121,10 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {

result, checkErr := gate.Check(ctx, cfg)

if err := renderTable(stderr, result); err != nil {
fmt.Fprintln(stderr, err)
}
if checkErr != nil {
fmt.Fprintln(stderr, checkErr)
} else if err := renderTable(stderr, result); err != nil {
fmt.Fprintln(stderr, err)
}
if *output == "json" {
enc := json.NewEncoder(stdout)
Expand Down
29 changes: 0 additions & 29 deletions grafana-alertcheck/cmd/grafana-alertcheck/version.go

This file was deleted.

25 changes: 0 additions & 25 deletions grafana-alertcheck/cmd/grafana-alertcheck/version_test.go

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ func main() {
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
}

const usage = "usage: grafana-alertcheck <list|watch|check|version>"
const usage = "usage: grafana-alertcheck <list|watch|check>"

// run is the whole of main's testable surface: parse the subcommand, dispatch,
// return the process exit code. Exit codes below 2 (pass/violations) belong to
Expand All @@ -37,8 +37,6 @@ func run(args []string, stdout, stderr io.Writer) int {
return runWatch(args[1:], os.Stdin, stdout, stderr)
case "check":
return runCheck(args[1:], os.Stdin, stdout, stderr)
case "version":
return runVersion(args[1:], stdout, stderr)
case "-h", "-help", "--help":
fmt.Fprintln(stdout, usage)
return 0
Expand Down
125 changes: 125 additions & 0 deletions grafana-alertcheck/cmd/style.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package main

import (
"bytes"
"io"
"os"
"strings"
)

// ANSI SGR codes for the human-facing notes and table footer. The colours are
// applied only when the destination is a terminal (see colorEnabled); a pipe,
// file or CI log gets plain text, so stdout stays reserved for --output json
// and no machine reader ever sees escape sequences.
const (
ansiReset = "\x1b[0m"
ansiRed = "\x1b[31m"
ansiGreen = "\x1b[32m"
ansiYellow = "\x1b[33m"
ansiCyan = "\x1b[36m"
// Orange has no entry in the base-16 palette; 256-colour 208 is a legible
// orange used for warnings, distinct from the yellow used for notes.
ansiOrange = "\x1b[38;5;208m"
)

// colorEnabled reports whether ANSI colour should be written to w. Colour is
// written only when three things hold: NO_COLOR is unset, w is a real *os.File
// (so text/tabwriter buffers, strings.Builder and bytes.Buffer tests all stay
// plain), and that file is a character device (a terminal, not a redirect).
func colorEnabled(w io.Writer) bool {
if os.Getenv("NO_COLOR") != "" {
return false
}
f, ok := w.(*os.File)
if !ok {
return false
}
fi, err := f.Stat()
if err != nil {
return false
}
return fi.Mode()&os.ModeCharDevice != 0
}

// styleLine applies the note vocabulary's colour to one line when enabled. The
// colour wraps the text only; the terminating newline is written uncoloured so
// the terminal's line discipline is never inside the escape sequence.
func styleLine(line string, enabled bool) string {
if !enabled {
return line
}
content := strings.TrimRight(line, "\n")
var color string
switch {
case strings.HasPrefix(content, "warning:"):
color = ansiOrange
case strings.HasPrefix(content, "note:"):
color = ansiYellow
case strings.HasPrefix(content, "drain wait:"):
color = ansiCyan
}
if color == "" {
return line
}
return color + content + ansiReset + "\n"
}

// noteStyler wraps the gate package's Notes stream — a presentation seam that
// keeps colour out of the library. It colourises each line by its known prefix
// and separates the collection countdown from the setup phase with a single
// blank line before the first "collecting:" line. The gate keeps emitting plain
// prose; only the CLI lays it out.
type noteStyler struct {
w io.Writer
enabled bool
pending []byte
sawCollecting bool
}

func newNoteStyler(w io.Writer) *noteStyler {
return &noteStyler{w: w, enabled: colorEnabled(w)}
}

// startsSection reports whether a line opens a new phase of the stream and so
// deserves a blank line above it. "collecting:" opens the countdown (once —
// later countdown lines follow on from the first), and "drain wait:" opens the
// drain phase. The setup lines (planned run time, warning, min-observed, notes)
// are one contiguous block and are not separated from each other.
func (s *noteStyler) startsSection(line string) bool {
switch {
case strings.HasPrefix(line, "warning:"):
return true
case strings.HasPrefix(line, "drain wait:"):
return true
case strings.HasPrefix(line, "collecting:"):
if s.sawCollecting {
return false
}
s.sawCollecting = true
return true
}
return false
}

func (s *noteStyler) Write(p []byte) (int, error) {
n := len(p)
s.pending = append(s.pending, p...)
for {
i := bytes.IndexByte(s.pending, '\n')
if i < 0 {
break
}
line := string(s.pending[:i+1])
s.pending = s.pending[i+1:]

if s.startsSection(line) {
if _, err := io.WriteString(s.w, "\n"); err != nil {
return n, err
}
}
if _, err := io.WriteString(s.w, styleLine(line, s.enabled)); err != nil {
return n, err
}
}
return n, nil
}
Loading
Loading