feat(observability): add the alert render pipeline, alert templates and promtool tests

.github/workflows/test.yml

@@ -25,4 +25,29 @@ jobs:
           make test
 
       - name: Running Scaffold Gate
-        run: make test-scaffold
+        run: make test-scaffold
+  observability:
+    name: Alerts and dashboards
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone the code
+        uses: actions/checkout@v7
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+
+      # promtool is not part of `make all` so that contributors without it can
+      # still run everything else. Bump PROMETHEUS_VERSION by hand.
+      - name: Install promtool
+        env:
+          PROMETHEUS_VERSION: "3.14.0"
+        run: |
+          mkdir -p "$HOME/.local/bin"
+          curl -sSfL "https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz" \
+            | tar -xz --strip-components=1 -C "$HOME/.local/bin" "prometheus-$PROMETHEUS_VERSION.linux-amd64/promtool"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Test alerts
+        run: make test-alerts

.gitignore

@@ -49,3 +49,6 @@ docs/superpowers/
 
 # MkDocs build output
 site/
+
+# Rendered observability artifacts
+observability/generated/

Makefile

@@ -205,6 +205,128 @@ e2e-component: ginkgo kind-create kind-set-context ## Run component E2E tests on
 .PHONY: e2e-full
 e2e-full: kind-create kind-set-context e2e kind-delete ## Full E2E lifecycle: create cluster, test, teardown.
 
+##@ Observability
+
+OBS_DIR := observability
+# Render output directory. The dev stack overrides it to keep its render apart.
+OBS_OUT ?= $(OBS_DIR)/generated
+# Prometheus metric namespace the condition gauge was created with
+# (ocm.NewOperatorConditionsGauge("<namespace>")). Required for rendering.
+METRIC_NAMESPACE ?= unset
+# Label carrying the namespace of the custom resource. The pod scraping the
+# operator usually owns `namespace`, so the exported label arrives as
+# `exported_namespace`; override with NAMESPACE_LABEL=namespace if yours does not.
+NAMESPACE_LABEL ?= exported_namespace
+# Shape of the rendered alert files: prometheusrule (one PrometheusRule object
+# per rule file) or rules (plain files for prometheus' rule_files).
+ALERT_FORMAT ?= prometheusrule
+# Optional metadata for the PrometheusRule objects: the namespace to create them
+# in, and comma-separated key=value labels, for example the release label a
+# kube-prometheus-stack ruleSelector matches on (PROMETHEUSRULE_LABELS=release=kps).
+PROMETHEUSRULE_NAMESPACE ?=
+PROMETHEUSRULE_LABELS ?=
+# Metric namespace the alert unit tests are written against.
+ALERT_TEST_NAMESPACE := test_operator
+
+# Fail unless METRIC_NAMESPACE was given. $(1) is the target name for the hint.
+define require_metric_namespace
+@[ "$(METRIC_NAMESPACE)" != "unset" ] && [ -n "$(METRIC_NAMESPACE)" ] || { \
+	echo "Error: METRIC_NAMESPACE is required."; \
+	echo "Usage: make $(1) METRIC_NAMESPACE=my_operator"; \
+	exit 1; \
+}
+endef
+
+# Render a template to stdout. $(1) template path, $(2) metric namespace,
+# $(3) namespace label.
+define render_template
+sed -e 's/{{operator_namespace}}/$(2)_/g' -e 's/{{namespace_label}}/$(3)/g' $(1)
+endef
+
+.PHONY: dashboards
+dashboards: ## Render the Grafana dashboards for METRIC_NAMESPACE into observability/generated/dashboards.
+	$(call require_metric_namespace,dashboards)
+	@echo "Rendering dashboards for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL))..."
+	@mkdir -p $(OBS_OUT)/dashboards
+	@for file in $(OBS_DIR)/dashboards/*.tpl.json; do \
+	  [ -e "$$file" ] || continue; \
+	  name=$$(basename "$$file" .tpl.json); \
+	  $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$(OBS_OUT)/dashboards/$$name.json"; \
+	done
+
+.PHONY: alerts
+alerts: ## Render the Prometheus alert rules for METRIC_NAMESPACE into observability/generated/alerts.
+	$(call require_metric_namespace,alerts)
+	@echo "Rendering alerts for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL), format: $(ALERT_FORMAT))..."
+	@mkdir -p $(OBS_OUT)/alerts
+	@for file in $(OBS_DIR)/alerts/*.yaml; do \
+	  [ -e "$$file" ] || continue; \
+	  case "$$file" in \
+	    *.tpl.yaml) \
+	      name=$$(basename "$$file" .tpl.yaml); \
+	      rule_name="$(METRIC_NAMESPACE)-$$name" ;; \
+	    *) \
+	      name=$$(basename "$$file" .yaml); \
+	      rule_name="ocf-$$name" ;; \
+	  esac; \
+	  rule_name=$$(echo "$$rule_name" | tr '[:upper:]' '[:lower:]' | tr '_:' '--'); \
+	  out="$(OBS_OUT)/alerts/$$name.yaml"; \
+	  case "$(ALERT_FORMAT)" in \
+	    rules) \
+	      $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$$out" ;; \
+	    prometheusrule) \
+	      { \
+	        echo "apiVersion: monitoring.coreos.com/v1"; \
+	        echo "kind: PrometheusRule"; \
+	        echo "metadata:"; \
+	        echo "  name: $$rule_name"; \
+	        [ -z "$(PROMETHEUSRULE_NAMESPACE)" ] || echo "  namespace: $(PROMETHEUSRULE_NAMESPACE)"; \
+	        if [ -n "$(PROMETHEUSRULE_LABELS)" ]; then \
+	          echo "  labels:"; \
+	          for kv in $$(echo "$(PROMETHEUSRULE_LABELS)" | tr ',' ' '); do \
+	            echo "    $${kv%%=*}: \"$${kv#*=}\""; \
+	          done; \
+	        fi; \
+	        echo "spec:"; \
+	        $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) \
+	          | sed -e 's/^/  /' -e 's/[[:space:]]*$$//'; \
+	      } > "$$out" ;; \
+	    *) \
+	      echo "Error: ALERT_FORMAT must be prometheusrule or rules, got '$(ALERT_FORMAT)'."; exit 1 ;; \
+	  esac; \
+	done
+
+.PHONY: test-alerts
+test-alerts: ## Lint and unit test the alert rules with promtool.
+	@command -v promtool >/dev/null 2>&1 || { \
+		echo "Error: promtool is required to test the alert rules."; \
+		echo "It ships with prometheus: https://prometheus.io/download/"; \
+		exit 1; \
+	}
+	@set -e; \
+	tmpdir=$$(mktemp -d "$${TMPDIR:-/tmp}/ocf-alerts.XXXXXX"); \
+	trap 'rm -rf "$$tmpdir"' EXIT; \
+	mkdir -p "$$tmpdir/tests" "$$tmpdir/namespace-label"; \
+	for file in $(OBS_DIR)/alerts/*.yaml; do \
+	  [ -e "$$file" ] || continue; \
+	  case "$$file" in \
+	    *.tpl.yaml) \
+	      name=$$(basename "$$file" .tpl.yaml); \
+	      $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),exported_namespace) > "$$tmpdir/$$name.yaml"; \
+	      $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),namespace) > "$$tmpdir/namespace-label/$$name.yaml" ;; \
+	    *) \
+	      cp "$$file" "$$tmpdir/"; \
+	      cp "$$file" "$$tmpdir/namespace-label/" ;; \
+	  esac; \
+	done; \
+	cp $(OBS_DIR)/alerts/tests/*.yaml "$$tmpdir/tests/"; \
+	echo "Linting rules..."; \
+	promtool check rules --lint=all --lint-fatal "$$tmpdir"/*.yaml; \
+	echo "Linting rules with NAMESPACE_LABEL=namespace..."; \
+	promtool check rules --lint=all --lint-fatal "$$tmpdir"/namespace-label/*.yaml; \
+	echo "Running unit tests..."; \
+	promtool test rules --diff "$$tmpdir"/tests/*.yaml
+
 
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary

observability/README.md

@@ -0,0 +1,19 @@
+# Observability
+
+Grafana dashboards and Prometheus alert rules for operators built on the framework, plus a local stack to look at them.
+Full documentation: [docs/observability.md](../docs/observability.md).
+
+Render for your operator, where `METRIC_NAMESPACE` is the argument you gave `ocm.NewOperatorConditionsGauge`:
+
+    make dashboards METRIC_NAMESPACE=myoperator
+    make alerts METRIC_NAMESPACE=myoperator
+
+Output lands in `generated/`. `generated/alerts/` contains the per-operator condition rules, named after the metric
+namespace, plus the shared `ocf-*` rules for controller-runtime and the managed resource counters, which are installed
+once per cluster. Add `NAMESPACE_LABEL=namespace` if your scrape keeps the exported `namespace` label, and
+`ALERT_FORMAT=rules` for plain rule files instead of `PrometheusRule` objects. `PROMETHEUSRULE_NAMESPACE` and
+`PROMETHEUSRULE_LABELS` set the metadata of the `PrometheusRule` objects; for kube-prometheus-stack pass
+`PROMETHEUSRULE_LABELS=release=<name>`.
+
+Run the alert unit tests with `make test-alerts` (needs `promtool`), and bring up Prometheus and Grafana with the
+simulator behind them with `make observability-up`.

observability/alerts/controller_runtime.yaml

@@ -0,0 +1,124 @@
+# Alerting rules for the controller-runtime metrics every operator exposes:
+# reconcile results and latency, the workqueue, leader election.
+#
+# These rules carry no metric namespace placeholder: they are shared by every
+# operator in the cluster and installed once, as the PrometheusRule
+# `ocf-controller-runtime`.
+#
+# Thresholds are deliberately conservative and expressed as ratios or
+# quantiles, so that they hold whether the operator manages three owners or
+# three thousand. See docs/observability.md for the thresholds to tune.
+#
+# `namespace` is the operator's own namespace as stamped by the scrape job
+# (absent when scraping outside a cluster, which is harmless), so two installs
+# of one operator in a cluster are alerted on separately.
+groups:
+  - name: controller-runtime
+    rules:
+      - alert: ControllerReconcileErrors
+        for: 15m
+        expr: >
+          (
+            sum by (namespace, controller) (rate(controller_runtime_reconcile_total{result="error"}[10m]))
+            /
+            sum by (namespace, controller) (rate(controller_runtime_reconcile_total[10m]))
+          ) > 0.25
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Controller `{{ $labels.controller }}` fails {{ $value | humanizePercentage }} of its reconciles
+          description: |
+            Over the last 10 minutes, {{ $value | humanizePercentage }} of the reconciles of controller `{{ $labels.controller }}` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip.
+
+            Quick check with:
+            ```
+            kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep {{ $labels.controller }}
+            ```
+
+      - alert: ControllerReconcilePanics
+        expr: >
+          sum by (namespace, controller) (increase(controller_runtime_reconcile_panics_total[10m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Controller `{{ $labels.controller }}` panicked during reconcile
+          description: |
+            Controller `{{ $labels.controller }}` recovered from {{ $value | printf "%.0f" }} panic(s) in the last 10 minutes. The reconcile that panicked was requeued, so the same owner will likely panic again on its next turn.
+
+            Quick check with:
+            ```
+            kubectl logs -l control-plane=controller-manager --all-containers | grep -A 20 "Observed a panic"
+            ```
+
+      # Queue wait time rather than queue depth: depth has no threshold that is
+      # right for every operator, whereas items waiting minutes for a worker is
+      # wrong at any scale. controller-runtime labels every workqueue series
+      # with both `name` and `controller` (same value) in every version the
+      # framework supports (v0.22 and later), so aggregating by `controller`
+      # keeps one label across the reconcile rules, the workqueue rules and the
+      # dashboards. The workqueue histogram has one bucket per decade, so the
+      # threshold sits on a bucket bound: a p99 above 100 seconds means more
+      # than one percent of items waited longer than 100 seconds. The value in
+      # the description is interpolated within that bucket.
+      - alert: ControllerWorkqueueBacklog
+        for: 15m
+        expr: >
+          histogram_quantile(
+            0.99,
+            sum by (namespace, controller, le) (rate(workqueue_queue_duration_seconds_bucket[10m]))
+          ) > 100
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Controller `{{ $labels.controller }}` cannot keep up with its workqueue
+          description: |
+            More than one percent of the items in the workqueue of controller `{{ $labels.controller }}` waited longer than 100 seconds before a worker picked them up (p99 wait {{ $value | humanizeDuration }}, interpolated within the histogram bucket). The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb.
+
+            Quick check with:
+            ```
+            kubectl top pod -l control-plane=controller-manager
+            ```
+
+      - alert: ControllerReconcileLatencyHigh
+        for: 15m
+        expr: >
+          histogram_quantile(
+            0.99,
+            sum by (namespace, controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket[10m]))
+          ) > 30
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Controller `{{ $labels.controller }}` reconciles are slow, p99 above 30 seconds
+          description: |
+            The p99 reconcile time of controller `{{ $labels.controller }}` over the last 10 minutes is {{ $value | humanizeDuration }}, above the 30 second threshold. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog.
+
+            Quick check with:
+            ```
+            kubectl logs -l control-plane=controller-manager --all-containers | grep {{ $labels.controller }} | tail -50
+            ```
+
+      # Silent when leader election is off (the gauge is not exported) and when
+      # no replica is alive to export it, for example a crash loop before the
+      # elector starts; pair it with your platform's target-down alert for the
+      # operator job.
+      - alert: OperatorLeaderMissing
+        for: 5m
+        expr: >
+          max by (namespace, name) (leader_election_master_status) == 0
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Operator `{{ $labels.name }}` has no leader
+          description: |
+            No replica of operator `{{ $labels.name }}` has held the leader election lease for 5 minutes, so none of its controllers is reconciling.
+
+            Quick check with:
+            ```
+            kubectl get lease -A --field-selector metadata.name={{ $labels.name }}
+            ```